2026-10-14 01:29:46 +00:00
7 changed files with 113 additions and 13 deletions
@@ -35,3 +35,8 @@ Light and dark, 390 and 1440 px:
 - `p-provider-local-*`: Local GPU (Ollama), keyless LOCAL settings, discovery/test results and discovered models.
 - `p-model-*`: a discovered model's capabilities, compute prices and connection-test result.
 - `p-home-providers-*`: the home's provider summary and usage from the executed demo calls, including disabled OpenRouter with its key Not set.
+
+Answer presentation (owner priority, 2026-10-13), taken against `OpenAiDemoServer` with the Next UI build of QRun-IO/qqq-frontend-next#111 (`next dev` proxied to the demo). The demo's OpenAI-compatible mock answers "How many images do I have by type?" the way a model would through the real adapter: five `count_records` calls through the UI API, then `present_results`.
+
+Light and dark, 390 and 1440 px:
+- `panel-presentation-*`: the summary line, an Images metric, the "Media by type" table (counts and the server-computed 80.2% and 19.8%, each row a filtered-list link), the pie chart, two filter links (Open JPEG media, Open PNG media) and a follow-up action.
@@ -21,8 +21,13 @@ import java.io.IOException;
 import java.math.BigDecimal;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.kingsrook.qbits.assistant.AssistantQBitConfig;
 import com.kingsrook.qbits.assistant.metadata.AssistantMetaDataBuilder;
 import com.kingsrook.qbits.assistant.model.AssistantModel;
@@ -35,6 +40,9 @@ import com.kingsrook.qbits.assistant.provider.AssistantProviderType;
 import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider;
 import com.kingsrook.qbits.assistant.provider.openai.OpenAiCompatibleProviderType;
 import com.kingsrook.qbits.assistant.testing.AssistantTestFixture;
+import com.kingsrook.qbits.assistant.testing.EvalSupport;
+import com.kingsrook.qbits.assistant.testing.GoldenTasks;
+import com.kingsrook.qbits.assistant.testing.PresentationGoldenTask;
 import com.kingsrook.qbits.assistant.testing.TurnRunner;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
@@ -61,6 +69,8 @@ public class OpenAiDemoServer extends AbstractQQQApplication
 {
   private static final String ADMIN = "admin-demo";
   private static final String MODEL = "qwen3:30b-a3b";
+   private static final String IMAGE_QUESTION = "How many images do I have by type?";
+   private static final ObjectMapper JSON = new ObjectMapper();
   private static QInstance instance;


@@ -160,6 +170,8 @@ public class OpenAiDemoServer extends AbstractQQQApplication
      {
         throw (new QException("Demo assistant turn failed: " + outcome.errorCode()));
      }
+      TurnRunner.Outcome presentation = new TurnRunner(instance, config).run(ADMIN, IMAGE_QUESTION);
+      PresentationGoldenTask.check(presentation);
      instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK)
         .withCustomizer(new QCodeReference(DemoSessionCustomizer.class)));
      instance.setHomeAppName(AssistantMetaDataBuilder.APP_NAME);
@@ -167,6 +179,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication
      System.out.println("Assistant home: http://localhost:" + port + "/app/" + AssistantMetaDataBuilder.APP_NAME);
      System.out.println("Local provider: http://localhost:" + port + "/app/assistant_provider/" + providerId);
      System.out.println("Local model: http://localhost:" + port + "/app/assistant_model/" + modelId);
+      System.out.println("Image presentation conversation id: " + presentation.conversationId());
   }


@@ -211,7 +224,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication
   {
      try(exchange)
      {
-         exchange.getRequestBody().readAllBytes();
+         byte[] request = exchange.getRequestBody().readAllBytes();
         String path = exchange.getRequestURI().getPath();
         String body;
         if("GET".equals(exchange.getRequestMethod()) && "/v1/models".equals(path))
@@ -227,16 +240,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication
         else if("POST".equals(exchange.getRequestMethod()) && "/v1/chat/completions".equals(path))
         {
            exchange.getResponseHeaders().set("Content-Type", "text/event-stream");
-            body = """
-               data: {"id":"demo-completion","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello from the local demo provider."}}]}
-
-               data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
-
-               data: {"choices":[],"usage":{"prompt_tokens":24,"completion_tokens":8,"total_tokens":32}}
-
-               data: [DONE]
-
-               """;
+            body = completion(JSON.readTree(request));
         }
         else
         {
@@ -248,4 +252,84 @@ public class OpenAiDemoServer extends AbstractQQQApplication
         exchange.getResponseBody().flush();
      }
   }
+
+
+
+   /*******************************************************************************
+    ** Follow the latest user turn, so connection tests and repeated chats are
+    ** independent. Counts are read from the adapter's actual tool messages.
+    *******************************************************************************/
+   private static String completion(JsonNode request) throws IOException
+   {
+      String question = "";
+      Map<String, Integer> counts = new LinkedHashMap<>();
+      boolean presented = false;
+      for(JsonNode message : request.path("messages"))
+      {
+         if("user".equals(message.path("role").asText()))
+         {
+            question = message.path("content").asText();
+            counts.clear();
+            presented = false;
+         }
+         else if("tool".equals(message.path("role").asText()) && IMAGE_QUESTION.equals(question))
+         {
+            String id = message.path("tool_call_id").asText();
+            if(id.startsWith("count-image/"))
+            {
+               counts.put(id.substring("count-".length()), ((Number) EvalSupport.unwrap(message.path("content").asText()).get("count")).intValue());
+            }
+            else if("present-images".equals(id))
+            {
+               presented = true;
+            }
+         }
+      }
+
+      Map<String, Object> delta = Map.of("role", "assistant", "content", "Hello from the local demo provider.");
+      String finish = "stop";
+      if(IMAGE_QUESTION.equals(question) && !presented)
+      {
+         List<Map<String, Object>> calls = new ArrayList<>();
+         if(counts.isEmpty())
+         {
+            for(String mime : List.of("image/jpeg", "image/png", "image/webp", "image/gif", "image/svg+xml"))
+            {
+               calls.add(toolCall(calls.size(), "count-" + mime, "count_records", Map.of("table", "media", "filter",
+                  GoldenTasks.filter("AND", GoldenTasks.criterion("mimeType", "EQUALS", mime)))));
+            }
+         }
+         else
+         {
+            if(counts.size() != 5 || counts.get("image/webp") != 0 || counts.get("image/gif") != 0 || counts.get("image/svg+xml") != 0)
+            {
+               throw (new IOException("Unexpected demo image counts."));
+            }
+            Map<String, Object> input = PresentationGoldenTask.input(counts.get("image/jpeg"), counts.get("image/png"));
+            input.put("actions", List.of(Map.of("label", "Show the PNG files", "prompt", "Show the PNG files")));
+            calls.add(toolCall(0, "present-images", "present_results", input));
+         }
+         delta = Map.of("role", "assistant", "tool_calls", calls);
+         finish = "tool_calls";
+      }
+      else if(presented)
+      {
+         delta = Map.of("role", "assistant", "content", "You can open either image type above.");
+      }
+      return ("data: " + JSON.writeValueAsString(Map.of("id", "demo-completion", "object", "chat.completion.chunk",
+         "choices", List.of(Map.of("index", 0, "delta", delta)))) + "\n\n"
+         + "data: " + JSON.writeValueAsString(Map.of("choices", List.of(Map.of("index", 0, "delta", Map.of(), "finish_reason", finish)))) + "\n\n"
+         + "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":24,\"completion_tokens\":8,\"total_tokens\":32}}\n\n"
+         + "data: [DONE]\n\n");
+   }
+
+
+
+   /*******************************************************************************
+    ** Encode tool arguments as a JSON string, as OpenAI streaming requires.
+    *******************************************************************************/
+   private static Map<String, Object> toolCall(Integer index, String id, String name, Map<String, Object> arguments) throws IOException
+   {
+      return (Map.of("index", index, "id", id, "type", "function", "function", Map.of("name", name, "arguments", JSON.writeValueAsString(arguments))));
+   }
 }
@@ -29,6 +29,11 @@ Startup prints these URLs (with the selected port):
 - Local provider: `http://localhost:8099/app/assistant_provider/1`
 - Priced and tested model: `http://localhost:8099/app/assistant_model/1`

+Startup also prints `Image presentation conversation id: <id>`. Open that
+conversation in the assistant panel history as `admin-demo` to capture the
+answer to "How many images do I have by type?". The id is assigned on each run;
+the conversation and its presentation are memory-backed and reset on restart.
+
 Every browser session is `admin-demo`. Startup inserts an enabled, default,
 keyless Local GPU (Ollama) provider, marked LOCAL and HIPAA eligible for the
 synthetic demo, and a disabled CLOUD OpenRouter provider whose key is Not set.
@@ -40,8 +45,14 @@ mock: `qwen3:30b-a3b`, `gpt-oss:20b`, `qwen3-coder:30b`, and `bge-m3`.
 A normal update sets qwen3's input/output compute prices to $0.05/$0.10 per
 million tokens. The provider and model Test connection processes then each
 make a real HTTP completion call, recording two TEST usage rows and the
-last-test fields. One assistant turn records a third usage row with purpose
-TURN. Discovery timestamps/status also come from the actual process.
+last-test fields. The greeting turn records a third usage row with purpose
+TURN. An image breakdown turn then makes three more model calls through the
+real OpenAI adapter and engine: five MIME-filtered `count_records` calls,
+`present_results` using the returned counts, and a short closing message.
+The fixture seeds 81 JPEG and 20 PNG rows (no WebP, GIF or SVG). The presentation
+includes a table with computed shares, filtered row links, a pie chart, JPEG
+and PNG filter links, and a "Show the PNG files" prompt action.
+Discovery timestamps/status also come from the actual process.

 Replies and token counts are synthetic mock API responses (24 input / 8 output
 tokens per completion), not live GPU measurements or billed cloud usage.