diff --git a/docs/screenshots/README.md b/docs/screenshots/README.md index 0045586..d33c536 100644 --- a/docs/screenshots/README.md +++ b/docs/screenshots/README.md @@ -35,3 +35,8 @@ Light and dark, 390 and 1440 px: - `p-provider-local-*`: Local GPU (Ollama), keyless LOCAL settings, discovery/test results and discovered models. - `p-model-*`: a discovered model's capabilities, compute prices and connection-test result. - `p-home-providers-*`: the home's provider summary and usage from the executed demo calls, including disabled OpenRouter with its key Not set. + +Answer presentation (owner priority, 2026-10-13), taken against `OpenAiDemoServer` with the Next UI build of QRun-IO/qqq-frontend-next#111 (`next dev` proxied to the demo). The demo's OpenAI-compatible mock answers "How many images do I have by type?" the way a model would through the real adapter: five `count_records` calls through the UI API, then `present_results`. + +Light and dark, 390 and 1440 px: +- `panel-presentation-*`: the summary line, an Images metric, the "Media by type" table (counts and the server-computed 80.2% and 19.8%, each row a filtered-list link), the pie chart, two filter links (Open JPEG media, Open PNG media) and a follow-up action. diff --git a/docs/screenshots/panel-presentation-dark-1440.png b/docs/screenshots/panel-presentation-dark-1440.png new file mode 100644 index 0000000..5ebe425 Binary files /dev/null and b/docs/screenshots/panel-presentation-dark-1440.png differ diff --git a/docs/screenshots/panel-presentation-dark-390.png b/docs/screenshots/panel-presentation-dark-390.png new file mode 100644 index 0000000..2d3a4d8 Binary files /dev/null and b/docs/screenshots/panel-presentation-dark-390.png differ diff --git a/docs/screenshots/panel-presentation-light-1440.png b/docs/screenshots/panel-presentation-light-1440.png new file mode 100644 index 0000000..83d9e1e Binary files /dev/null and b/docs/screenshots/panel-presentation-light-1440.png differ diff --git a/docs/screenshots/panel-presentation-light-390.png b/docs/screenshots/panel-presentation-light-390.png new file mode 100644 index 0000000..dc36904 Binary files /dev/null and b/docs/screenshots/panel-presentation-light-390.png differ diff --git a/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/OpenAiDemoServer.java b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/OpenAiDemoServer.java index 2673312..4930c21 100644 --- a/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/OpenAiDemoServer.java +++ b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/OpenAiDemoServer.java @@ -21,8 +21,13 @@ import java.io.IOException; import java.math.BigDecimal; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.kingsrook.qbits.assistant.AssistantQBitConfig; import com.kingsrook.qbits.assistant.metadata.AssistantMetaDataBuilder; import com.kingsrook.qbits.assistant.model.AssistantModel; @@ -35,6 +40,9 @@ import com.kingsrook.qbits.assistant.provider.AssistantProviderType; import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider; import com.kingsrook.qbits.assistant.provider.openai.OpenAiCompatibleProviderType; import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; +import com.kingsrook.qbits.assistant.testing.EvalSupport; +import com.kingsrook.qbits.assistant.testing.GoldenTasks; +import com.kingsrook.qbits.assistant.testing.PresentationGoldenTask; import com.kingsrook.qbits.assistant.testing.TurnRunner; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; @@ -61,6 +69,8 @@ public class OpenAiDemoServer extends AbstractQQQApplication { private static final String ADMIN = "admin-demo"; private static final String MODEL = "qwen3:30b-a3b"; + private static final String IMAGE_QUESTION = "How many images do I have by type?"; + private static final ObjectMapper JSON = new ObjectMapper(); private static QInstance instance; @@ -160,6 +170,8 @@ public class OpenAiDemoServer extends AbstractQQQApplication { throw (new QException("Demo assistant turn failed: " + outcome.errorCode())); } + TurnRunner.Outcome presentation = new TurnRunner(instance, config).run(ADMIN, IMAGE_QUESTION); + PresentationGoldenTask.check(presentation); instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK) .withCustomizer(new QCodeReference(DemoSessionCustomizer.class))); instance.setHomeAppName(AssistantMetaDataBuilder.APP_NAME); @@ -167,6 +179,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication System.out.println("Assistant home: http://localhost:" + port + "/app/" + AssistantMetaDataBuilder.APP_NAME); System.out.println("Local provider: http://localhost:" + port + "/app/assistant_provider/" + providerId); System.out.println("Local model: http://localhost:" + port + "/app/assistant_model/" + modelId); + System.out.println("Image presentation conversation id: " + presentation.conversationId()); } @@ -211,7 +224,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication { try(exchange) { - exchange.getRequestBody().readAllBytes(); + byte[] request = exchange.getRequestBody().readAllBytes(); String path = exchange.getRequestURI().getPath(); String body; if("GET".equals(exchange.getRequestMethod()) && "/v1/models".equals(path)) @@ -227,16 +240,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication else if("POST".equals(exchange.getRequestMethod()) && "/v1/chat/completions".equals(path)) { exchange.getResponseHeaders().set("Content-Type", "text/event-stream"); - body = """ - data: {"id":"demo-completion","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello from the local demo provider."}}]} - - data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} - - data: {"choices":[],"usage":{"prompt_tokens":24,"completion_tokens":8,"total_tokens":32}} - - data: [DONE] - - """; + body = completion(JSON.readTree(request)); } else { @@ -248,4 +252,84 @@ public class OpenAiDemoServer extends AbstractQQQApplication exchange.getResponseBody().flush(); } } + + + + /******************************************************************************* + ** Follow the latest user turn, so connection tests and repeated chats are + ** independent. Counts are read from the adapter's actual tool messages. + *******************************************************************************/ + private static String completion(JsonNode request) throws IOException + { + String question = ""; + Map counts = new LinkedHashMap<>(); + boolean presented = false; + for(JsonNode message : request.path("messages")) + { + if("user".equals(message.path("role").asText())) + { + question = message.path("content").asText(); + counts.clear(); + presented = false; + } + else if("tool".equals(message.path("role").asText()) && IMAGE_QUESTION.equals(question)) + { + String id = message.path("tool_call_id").asText(); + if(id.startsWith("count-image/")) + { + counts.put(id.substring("count-".length()), ((Number) EvalSupport.unwrap(message.path("content").asText()).get("count")).intValue()); + } + else if("present-images".equals(id)) + { + presented = true; + } + } + } + + Map delta = Map.of("role", "assistant", "content", "Hello from the local demo provider."); + String finish = "stop"; + if(IMAGE_QUESTION.equals(question) && !presented) + { + List> calls = new ArrayList<>(); + if(counts.isEmpty()) + { + for(String mime : List.of("image/jpeg", "image/png", "image/webp", "image/gif", "image/svg+xml")) + { + calls.add(toolCall(calls.size(), "count-" + mime, "count_records", Map.of("table", "media", "filter", + GoldenTasks.filter("AND", GoldenTasks.criterion("mimeType", "EQUALS", mime))))); + } + } + else + { + if(counts.size() != 5 || counts.get("image/webp") != 0 || counts.get("image/gif") != 0 || counts.get("image/svg+xml") != 0) + { + throw (new IOException("Unexpected demo image counts.")); + } + Map input = PresentationGoldenTask.input(counts.get("image/jpeg"), counts.get("image/png")); + input.put("actions", List.of(Map.of("label", "Show the PNG files", "prompt", "Show the PNG files"))); + calls.add(toolCall(0, "present-images", "present_results", input)); + } + delta = Map.of("role", "assistant", "tool_calls", calls); + finish = "tool_calls"; + } + else if(presented) + { + delta = Map.of("role", "assistant", "content", "You can open either image type above."); + } + return ("data: " + JSON.writeValueAsString(Map.of("id", "demo-completion", "object", "chat.completion.chunk", + "choices", List.of(Map.of("index", 0, "delta", delta)))) + "\n\n" + + "data: " + JSON.writeValueAsString(Map.of("choices", List.of(Map.of("index", 0, "delta", Map.of(), "finish_reason", finish)))) + "\n\n" + + "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":24,\"completion_tokens\":8,\"total_tokens\":32}}\n\n" + + "data: [DONE]\n\n"); + } + + + + /******************************************************************************* + ** Encode tool arguments as a JSON string, as OpenAI streaming requires. + *******************************************************************************/ + private static Map toolCall(Integer index, String id, String name, Map arguments) throws IOException + { + return (Map.of("index", index, "id", id, "type", "function", "function", Map.of("name", name, "arguments", JSON.writeValueAsString(arguments)))); + } } diff --git a/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/README.md b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/README.md index b05166f..328f9c8 100644 --- a/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/README.md +++ b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/demo/README.md @@ -29,6 +29,11 @@ Startup prints these URLs (with the selected port): - Local provider: `http://localhost:8099/app/assistant_provider/1` - Priced and tested model: `http://localhost:8099/app/assistant_model/1` +Startup also prints `Image presentation conversation id: `. Open that +conversation in the assistant panel history as `admin-demo` to capture the +answer to "How many images do I have by type?". The id is assigned on each run; +the conversation and its presentation are memory-backed and reset on restart. + Every browser session is `admin-demo`. Startup inserts an enabled, default, keyless Local GPU (Ollama) provider, marked LOCAL and HIPAA eligible for the synthetic demo, and a disabled CLOUD OpenRouter provider whose key is Not set. @@ -40,8 +45,14 @@ mock: `qwen3:30b-a3b`, `gpt-oss:20b`, `qwen3-coder:30b`, and `bge-m3`. A normal update sets qwen3's input/output compute prices to $0.05/$0.10 per million tokens. The provider and model Test connection processes then each make a real HTTP completion call, recording two TEST usage rows and the -last-test fields. One assistant turn records a third usage row with purpose -TURN. Discovery timestamps/status also come from the actual process. +last-test fields. The greeting turn records a third usage row with purpose +TURN. An image breakdown turn then makes three more model calls through the +real OpenAI adapter and engine: five MIME-filtered `count_records` calls, +`present_results` using the returned counts, and a short closing message. +The fixture seeds 81 JPEG and 20 PNG rows (no WebP, GIF or SVG). The presentation +includes a table with computed shares, filtered row links, a pie chart, JPEG +and PNG filter links, and a "Show the PNG files" prompt action. +Discovery timestamps/status also come from the actual process. Replies and token counts are synthetic mock API responses (24 input / 8 output tokens per completion), not live GPU measurements or billed cloud usage.