Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87835961f4
|
||
|
|
c3a828c1c3
|
||
|
|
5e1c213fd6 | ||
|
|
1a65aa2257 | ||
|
|
6a26a7d4fa
|
No files matched your search
@@ -35,3 +35,8 @@ Light and dark, 390 and 1440 px:
|
||||
- `p-provider-local-*`: Local GPU (Ollama), keyless LOCAL settings, discovery/test results and discovered models.
|
||||
- `p-model-*`: a discovered model's capabilities, compute prices and connection-test result.
|
||||
- `p-home-providers-*`: the home's provider summary and usage from the executed demo calls, including disabled OpenRouter with its key Not set.
|
||||
|
||||
Answer presentation (owner priority, 2026-10-13), taken against `OpenAiDemoServer` with the Next UI build of QRun-IO/qqq-frontend-next#111 (`next dev` proxied to the demo). The demo's OpenAI-compatible mock answers "How many images do I have by type?" the way a model would through the real adapter: five `count_records` calls through the UI API, then `present_results`.
|
||||
|
||||
Light and dark, 390 and 1440 px:
|
||||
- `panel-presentation-*`: the summary line, an Images metric, the "Media by type" table (counts and the server-computed 80.2% and 19.8%, each row a filtered-list link), the pie chart, two filter links (Open JPEG media, Open PNG media) and a follow-up action.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 124 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 126 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
+95
-11
@@ -21,8 +21,13 @@ import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.assistant.AssistantQBitConfig;
|
||||
import com.kingsrook.qbits.assistant.metadata.AssistantMetaDataBuilder;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantModel;
|
||||
@@ -35,6 +40,9 @@ import com.kingsrook.qbits.assistant.provider.AssistantProviderType;
|
||||
import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider;
|
||||
import com.kingsrook.qbits.assistant.provider.openai.OpenAiCompatibleProviderType;
|
||||
import com.kingsrook.qbits.assistant.testing.AssistantTestFixture;
|
||||
import com.kingsrook.qbits.assistant.testing.EvalSupport;
|
||||
import com.kingsrook.qbits.assistant.testing.GoldenTasks;
|
||||
import com.kingsrook.qbits.assistant.testing.PresentationGoldenTask;
|
||||
import com.kingsrook.qbits.assistant.testing.TurnRunner;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
@@ -61,6 +69,8 @@ public class OpenAiDemoServer extends AbstractQQQApplication
|
||||
{
|
||||
private static final String ADMIN = "admin-demo";
|
||||
private static final String MODEL = "qwen3:30b-a3b";
|
||||
private static final String IMAGE_QUESTION = "How many images do I have by type?";
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
private static QInstance instance;
|
||||
|
||||
|
||||
@@ -160,6 +170,8 @@ public class OpenAiDemoServer extends AbstractQQQApplication
|
||||
{
|
||||
throw (new QException("Demo assistant turn failed: " + outcome.errorCode()));
|
||||
}
|
||||
TurnRunner.Outcome presentation = new TurnRunner(instance, config).run(ADMIN, IMAGE_QUESTION);
|
||||
PresentationGoldenTask.check(presentation);
|
||||
instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK)
|
||||
.withCustomizer(new QCodeReference(DemoSessionCustomizer.class)));
|
||||
instance.setHomeAppName(AssistantMetaDataBuilder.APP_NAME);
|
||||
@@ -167,6 +179,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication
|
||||
System.out.println("Assistant home: http://localhost:" + port + "/app/" + AssistantMetaDataBuilder.APP_NAME);
|
||||
System.out.println("Local provider: http://localhost:" + port + "/app/assistant_provider/" + providerId);
|
||||
System.out.println("Local model: http://localhost:" + port + "/app/assistant_model/" + modelId);
|
||||
System.out.println("Image presentation conversation id: " + presentation.conversationId());
|
||||
}
|
||||
|
||||
|
||||
@@ -211,7 +224,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication
|
||||
{
|
||||
try(exchange)
|
||||
{
|
||||
exchange.getRequestBody().readAllBytes();
|
||||
byte[] request = exchange.getRequestBody().readAllBytes();
|
||||
String path = exchange.getRequestURI().getPath();
|
||||
String body;
|
||||
if("GET".equals(exchange.getRequestMethod()) && "/v1/models".equals(path))
|
||||
@@ -227,16 +240,7 @@ public class OpenAiDemoServer extends AbstractQQQApplication
|
||||
else if("POST".equals(exchange.getRequestMethod()) && "/v1/chat/completions".equals(path))
|
||||
{
|
||||
exchange.getResponseHeaders().set("Content-Type", "text/event-stream");
|
||||
body = """
|
||||
data: {"id":"demo-completion","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello from the local demo provider."}}]}
|
||||
|
||||
data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
|
||||
|
||||
data: {"choices":[],"usage":{"prompt_tokens":24,"completion_tokens":8,"total_tokens":32}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
""";
|
||||
body = completion(JSON.readTree(request));
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -248,4 +252,84 @@ public class OpenAiDemoServer extends AbstractQQQApplication
|
||||
exchange.getResponseBody().flush();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Follow the latest user turn, so connection tests and repeated chats are
|
||||
** independent. Counts are read from the adapter's actual tool messages.
|
||||
*******************************************************************************/
|
||||
private static String completion(JsonNode request) throws IOException
|
||||
{
|
||||
String question = "";
|
||||
Map<String, Integer> counts = new LinkedHashMap<>();
|
||||
boolean presented = false;
|
||||
for(JsonNode message : request.path("messages"))
|
||||
{
|
||||
if("user".equals(message.path("role").asText()))
|
||||
{
|
||||
question = message.path("content").asText();
|
||||
counts.clear();
|
||||
presented = false;
|
||||
}
|
||||
else if("tool".equals(message.path("role").asText()) && IMAGE_QUESTION.equals(question))
|
||||
{
|
||||
String id = message.path("tool_call_id").asText();
|
||||
if(id.startsWith("count-image/"))
|
||||
{
|
||||
counts.put(id.substring("count-".length()), ((Number) EvalSupport.unwrap(message.path("content").asText()).get("count")).intValue());
|
||||
}
|
||||
else if("present-images".equals(id))
|
||||
{
|
||||
presented = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> delta = Map.of("role", "assistant", "content", "Hello from the local demo provider.");
|
||||
String finish = "stop";
|
||||
if(IMAGE_QUESTION.equals(question) && !presented)
|
||||
{
|
||||
List<Map<String, Object>> calls = new ArrayList<>();
|
||||
if(counts.isEmpty())
|
||||
{
|
||||
for(String mime : List.of("image/jpeg", "image/png", "image/webp", "image/gif", "image/svg+xml"))
|
||||
{
|
||||
calls.add(toolCall(calls.size(), "count-" + mime, "count_records", Map.of("table", "media", "filter",
|
||||
GoldenTasks.filter("AND", GoldenTasks.criterion("mimeType", "EQUALS", mime)))));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if(counts.size() != 5 || counts.get("image/webp") != 0 || counts.get("image/gif") != 0 || counts.get("image/svg+xml") != 0)
|
||||
{
|
||||
throw (new IOException("Unexpected demo image counts."));
|
||||
}
|
||||
Map<String, Object> input = PresentationGoldenTask.input(counts.get("image/jpeg"), counts.get("image/png"));
|
||||
input.put("actions", List.of(Map.of("label", "Show the PNG files", "prompt", "Show the PNG files")));
|
||||
calls.add(toolCall(0, "present-images", "present_results", input));
|
||||
}
|
||||
delta = Map.of("role", "assistant", "tool_calls", calls);
|
||||
finish = "tool_calls";
|
||||
}
|
||||
else if(presented)
|
||||
{
|
||||
delta = Map.of("role", "assistant", "content", "You can open either image type above.");
|
||||
}
|
||||
return ("data: " + JSON.writeValueAsString(Map.of("id", "demo-completion", "object", "chat.completion.chunk",
|
||||
"choices", List.of(Map.of("index", 0, "delta", delta)))) + "\n\n"
|
||||
+ "data: " + JSON.writeValueAsString(Map.of("choices", List.of(Map.of("index", 0, "delta", Map.of(), "finish_reason", finish)))) + "\n\n"
|
||||
+ "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":24,\"completion_tokens\":8,\"total_tokens\":32}}\n\n"
|
||||
+ "data: [DONE]\n\n");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Encode tool arguments as a JSON string, as OpenAI streaming requires.
|
||||
*******************************************************************************/
|
||||
private static Map<String, Object> toolCall(Integer index, String id, String name, Map<String, Object> arguments) throws IOException
|
||||
{
|
||||
return (Map.of("index", index, "id", id, "type", "function", "function", Map.of("name", name, "arguments", JSON.writeValueAsString(arguments))));
|
||||
}
|
||||
}
|
||||
+13
-2
@@ -29,6 +29,11 @@ Startup prints these URLs (with the selected port):
|
||||
- Local provider: `http://localhost:8099/app/assistant_provider/1`
|
||||
- Priced and tested model: `http://localhost:8099/app/assistant_model/1`
|
||||
|
||||
Startup also prints `Image presentation conversation id: <id>`. Open that
|
||||
conversation in the assistant panel history as `admin-demo` to capture the
|
||||
answer to "How many images do I have by type?". The id is assigned on each run;
|
||||
the conversation and its presentation are memory-backed and reset on restart.
|
||||
|
||||
Every browser session is `admin-demo`. Startup inserts an enabled, default,
|
||||
keyless Local GPU (Ollama) provider, marked LOCAL and HIPAA eligible for the
|
||||
synthetic demo, and a disabled CLOUD OpenRouter provider whose key is Not set.
|
||||
@@ -40,8 +45,14 @@ mock: `qwen3:30b-a3b`, `gpt-oss:20b`, `qwen3-coder:30b`, and `bge-m3`.
|
||||
A normal update sets qwen3's input/output compute prices to $0.05/$0.10 per
|
||||
million tokens. The provider and model Test connection processes then each
|
||||
make a real HTTP completion call, recording two TEST usage rows and the
|
||||
last-test fields. One assistant turn records a third usage row with purpose
|
||||
TURN. Discovery timestamps/status also come from the actual process.
|
||||
last-test fields. The greeting turn records a third usage row with purpose
|
||||
TURN. An image breakdown turn then makes three more model calls through the
|
||||
real OpenAI adapter and engine: five MIME-filtered `count_records` calls,
|
||||
`present_results` using the returned counts, and a short closing message.
|
||||
The fixture seeds 81 JPEG and 20 PNG rows (no WebP, GIF or SVG). The presentation
|
||||
includes a table with computed shares, filtered row links, a pie chart, JPEG
|
||||
and PNG filter links, and a "Show the PNG files" prompt action.
|
||||
Discovery timestamps/status also come from the actual process.
|
||||
|
||||
Replies and token counts are synthetic mock API responses (24 input / 8 output
|
||||
tokens per completion), not live GPU measurements or billed cloud usage.
|
||||
|
||||
@@ -133,6 +133,7 @@ public class AssistantQBitConfig implements QBitConfig
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
private String uiApiBaseUrl;
|
||||
private UiApiGateway uiApiGateway;
|
||||
private com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner bulkLoadRunner;
|
||||
private Integer pendingActionTtlSeconds = 900;
|
||||
private List<String> customRedactionPatterns = new ArrayList<>();
|
||||
private Integer killSwitchPollMillis = 2000;
|
||||
@@ -1202,6 +1203,38 @@ public class AssistantQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for bulkLoadRunner (null: the durable jobs launch,
|
||||
** JobsLaunchBulkLoadRunner) - how bulk_load_run starts a load.
|
||||
*******************************************************************************/
|
||||
public com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner getBulkLoadRunner()
|
||||
{
|
||||
return (this.bulkLoadRunner);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for bulkLoadRunner
|
||||
*******************************************************************************/
|
||||
public void setBulkLoadRunner(com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner bulkLoadRunner)
|
||||
{
|
||||
this.bulkLoadRunner = bulkLoadRunner;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for bulkLoadRunner
|
||||
*******************************************************************************/
|
||||
public AssistantQBitConfig withBulkLoadRunner(com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner bulkLoadRunner)
|
||||
{
|
||||
this.bulkLoadRunner = bulkLoadRunner;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for uiApiGateway (null: the loopback stand-in)
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -37,6 +37,7 @@ import com.kingsrook.qbits.assistant.rag.InMemoryVectorStore;
|
||||
import com.kingsrook.qbits.assistant.rag.RagService;
|
||||
import com.kingsrook.qbits.assistant.rag.VectorStore;
|
||||
import com.kingsrook.qbits.assistant.tools.KnowledgeTool;
|
||||
import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools;
|
||||
import com.kingsrook.qbits.assistant.tools.UiApiToolbox;
|
||||
import com.kingsrook.qbits.assistant.uiapi.LoopbackUiApiGateway;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiGateway;
|
||||
@@ -58,6 +59,7 @@ public class AssistantRuntime
|
||||
{
|
||||
private final RagService ragService;
|
||||
private final KnowledgeTool knowledgeTool;
|
||||
private final BulkLoadTools bulkLoadTools;
|
||||
|
||||
private final QInstance qInstance;
|
||||
private final AssistantQBitConfig config;
|
||||
@@ -116,7 +118,8 @@ public class AssistantRuntime
|
||||
VectorStore vectors = config.getVectorStore() == null ? new InMemoryVectorStore() : config.getVectorStore();
|
||||
this.ragService = new RagService(this, vectors);
|
||||
this.knowledgeTool = new KnowledgeTool(this, ragService, vectors);
|
||||
this.contextBuilder = new ContextBuilder(config, kernel, toolbox).withKnowledgeTool(knowledgeTool);
|
||||
this.bulkLoadTools = new BulkLoadTools(qInstance, dataAccessService, config.getBulkLoadRunner());
|
||||
this.contextBuilder = new ContextBuilder(config, kernel, toolbox).withKnowledgeTool(knowledgeTool).withBulkLoadTools(bulkLoadTools);
|
||||
}
|
||||
|
||||
|
||||
@@ -447,4 +450,14 @@ public class AssistantRuntime
|
||||
return (knowledgeTool);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The bulk-load tools (saved profiles, attached files, dry runs, loads).
|
||||
*******************************************************************************/
|
||||
public BulkLoadTools getBulkLoadTools()
|
||||
{
|
||||
return (bulkLoadTools);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -46,6 +46,7 @@ import com.kingsrook.qbits.assistant.model.AssistantTurn;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantTurnEvent;
|
||||
import com.kingsrook.qbits.assistant.persistence.AssistantStore;
|
||||
import com.kingsrook.qbits.assistant.tools.UiApiToolbox;
|
||||
import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.output.AiRedactor;
|
||||
@@ -598,7 +599,9 @@ public class AssistantApi
|
||||
}
|
||||
PolicyService.EffectivePolicy policy = runtime.getPolicyService().current(true);
|
||||
UiApiToolbox.Context context = new UiApiToolbox.Context(session.getUuid(), session.getValue(QSession.VALUE_KEY_USER_TIMEZONE), runtime.getUiApiGateway(), policy.tools(), null);
|
||||
UiApiToolbox.Outcome outcome = runtime.getToolbox().executeConfirmedWrite(context, action.getToolCallId(), action.getToolName(), action.getTableName(), action.getRecordId(), values);
|
||||
UiApiToolbox.Outcome outcome = BulkLoadTools.WRITE_TOOLS.contains(action.getToolName())
|
||||
? runtime.getBulkLoadTools().executeConfirmed(context, action.getToolCallId(), action.getToolName(), action.getTableName(), values)
|
||||
: runtime.getToolbox().executeConfirmedWrite(context, action.getToolCallId(), action.getToolName(), action.getTableName(), action.getRecordId(), values);
|
||||
AiToolResult result = outcome.getResult();
|
||||
boolean ok = result.getOutcome() == AiToolOutcome.OK;
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ import com.kingsrook.qbits.assistant.provider.ModelToolDefinition;
|
||||
import com.kingsrook.qbits.assistant.provider.SystemBlock;
|
||||
import com.kingsrook.qbits.assistant.tools.KnowledgeTool;
|
||||
import com.kingsrook.qbits.assistant.tools.PresentResultsTool;
|
||||
import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools;
|
||||
import com.kingsrook.qbits.assistant.tools.UiApiToolbox;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolDefinition;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolKernel;
|
||||
@@ -66,12 +67,14 @@ public class ContextBuilder
|
||||
|
||||
public static final int MAX_CONTEXT_CHARS = 512;
|
||||
public static final int MAX_SELECTED_IDS = 100;
|
||||
public static final int MAX_ATTACHMENTS = 5;
|
||||
|
||||
private final AssistantQBitConfig config;
|
||||
private final AiToolKernel kernel;
|
||||
private final UiApiToolbox toolbox;
|
||||
private volatile String appDigest;
|
||||
private KnowledgeTool knowledgeTool;
|
||||
private BulkLoadTools bulkLoadTools;
|
||||
|
||||
|
||||
|
||||
@@ -125,6 +128,13 @@ public class ContextBuilder
|
||||
byName.put(KnowledgeTool.NAME, knowledgeTool.definition());
|
||||
}
|
||||
byName.put(PresentResultsTool.NAME, PresentResultsTool.definition(isStrict(PresentResultsTool.NAME, true)));
|
||||
if(bulkLoadTools != null && bulkLoadTools.offered())
|
||||
{
|
||||
for(ModelToolDefinition definition : bulkLoadTools.definitions(policy, toolbox.anyWritableTable()))
|
||||
{
|
||||
byName.put(definition.getName(), definition);
|
||||
}
|
||||
}
|
||||
return (new ArrayList<>(byName.values()));
|
||||
}
|
||||
|
||||
@@ -352,6 +362,32 @@ public class ContextBuilder
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// files the user attached in the panel (already uploaded through //
|
||||
// the UI API as the user): only their opaque refs and names //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
if(raw.get("attachments") instanceof List<?> attachments)
|
||||
{
|
||||
List<Map<String, Object>> kept = new ArrayList<>();
|
||||
for(Object attachment : attachments)
|
||||
{
|
||||
if(attachment instanceof Map<?, ?> a && scalar(a.get("uploadRef")) != null && kept.size() < MAX_ATTACHMENTS)
|
||||
{
|
||||
Map<String, Object> one = new LinkedHashMap<>();
|
||||
one.put("uploadRef", scalar(a.get("uploadRef")));
|
||||
if(scalar(a.get("fileName")) != null)
|
||||
{
|
||||
one.put("fileName", scalar(a.get("fileName")));
|
||||
}
|
||||
kept.add(one);
|
||||
}
|
||||
}
|
||||
if(!kept.isEmpty())
|
||||
{
|
||||
clean.put("attachments", kept);
|
||||
}
|
||||
}
|
||||
|
||||
if(JsonUtils.toJson(clean).length() > 4000)
|
||||
{
|
||||
clean.remove("filter");
|
||||
@@ -390,4 +426,15 @@ public class ContextBuilder
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the bulk-load tools.
|
||||
*******************************************************************************/
|
||||
public ContextBuilder withBulkLoadTools(BulkLoadTools bulkLoadTools)
|
||||
{
|
||||
this.bulkLoadTools = bulkLoadTools;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -37,7 +37,14 @@ public class ToolLabels
|
||||
Map.entry("get_record_history", new String[] { "Reading the change history", "Read the change history" }),
|
||||
Map.entry("render_widget", new String[] { "Rendering a widget", "Rendered a widget" }),
|
||||
Map.entry("search_help", new String[] { "Searching help", "Searched help" }),
|
||||
Map.entry("navigate", new String[] { "Opening a screen", "Opened a screen" })
|
||||
Map.entry("navigate", new String[] { "Opening a screen", "Opened a screen" }),
|
||||
Map.entry("bulk_profiles_list", new String[] { "Listing saved bulk load profiles", "Listed saved bulk load profiles" }),
|
||||
Map.entry("bulk_profile_describe", new String[] { "Reading a bulk load profile", "Read a bulk load profile" }),
|
||||
Map.entry("bulk_file_attach", new String[] { "Reading the attached file", "Read the attached file" }),
|
||||
Map.entry("bulk_mapping_suggest", new String[] { "Suggesting a column mapping", "Suggested a column mapping" }),
|
||||
Map.entry("bulk_load_dry_run", new String[] { "Checking the mapping (dry run)", "Checked the mapping (dry run)" }),
|
||||
Map.entry("bulk_profile_save", new String[] { "Preparing to save the profile", "Prepared to save the profile" }),
|
||||
Map.entry("bulk_load_run", new String[] { "Preparing the load", "Prepared the load" })
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ import com.kingsrook.qbits.assistant.provider.ModelUsage;
|
||||
import com.kingsrook.qbits.assistant.provider.StreamHandle;
|
||||
import com.kingsrook.qbits.assistant.tools.KnowledgeTool;
|
||||
import com.kingsrook.qbits.assistant.tools.PresentResultsTool;
|
||||
import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools;
|
||||
import com.kingsrook.qbits.assistant.tools.UiApiToolbox;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiRecordRef;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolCall;
|
||||
@@ -716,6 +717,11 @@ public class TurnEngine implements Runnable
|
||||
{
|
||||
results[index] = runtime.getKnowledgeTool().execute(toolContext(), call.getId(), call.getInput(), classification.getClassification(), conversation.getId(), turn.getId());
|
||||
}
|
||||
else if(BulkLoadTools.NAMES.contains(call.getName()))
|
||||
{
|
||||
outcomes[index] = runtime.getBulkLoadTools().execute(toolContext(), call.getId(), call.getName(), call.getInput(), hipaaRoute());
|
||||
results[index] = outcomes[index].getResult();
|
||||
}
|
||||
else
|
||||
{
|
||||
outcomes[index] = runtime.getToolbox().execute(toolContext(), call.getId(), call.getName(), call.getInput());
|
||||
@@ -856,7 +862,7 @@ public class TurnEngine implements Runnable
|
||||
.withRowCount(result.getRowCount())
|
||||
.withDurationMs(result.getDurationMs())
|
||||
.withResultSummary(summary)
|
||||
.withResultJson(truncate(result.getModelContent(), MAX_RESULT_JSON_CHARS))
|
||||
.withResultJson(truncate(BulkLoadTools.NAMES.contains(use.getToolName()) ? BulkLoadTools.forLog(result.getModelContent()) : result.getModelContent(), MAX_RESULT_JSON_CHARS))
|
||||
.withRedactionsJson(result.getRedactions() == null || result.getRedactions().isEmpty() ? null : JsonUtils.toJson(result.getRedactions()))
|
||||
.withUiApiOperation(extras == null ? ("search_knowledge".equals(use.getToolName()) ? "GET /table/{tableName}/{primaryKey}" : null) : extras.getOperation())
|
||||
.withHttpStatus(extras == null ? null : extras.getHttpStatus())
|
||||
@@ -876,6 +882,21 @@ public class TurnEngine implements Runnable
|
||||
** API), the gateway, the policy's tool settings, and the pending-action
|
||||
** store for writes that need confirmation.
|
||||
*******************************************************************************/
|
||||
private boolean hipaaRoute()
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// the X-QQQ-AI-Route the bulk-load tools send: the turn's provider's //
|
||||
// HIPAA eligibility, never anything the model says //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
return (active != null && active.row() != null && Boolean.TRUE.equals(active.row().getHipaaEligible()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The UI API tool context: the user's session, the gateway, the policy's
|
||||
** tools and the pending-action hook.
|
||||
*******************************************************************************/
|
||||
private UiApiToolbox.Context toolContext()
|
||||
{
|
||||
return (new UiApiToolbox.Context(session.getUuid(), userTimezone(), runtime.getUiApiGateway(), policy.tools(), this::proposeWrite));
|
||||
|
||||
@@ -203,6 +203,21 @@ public class UiApiToolbox
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An outcome built by another tool family (e.g. the bulk-load tools), so
|
||||
** TurnEngine and the confirm route log it the same way.
|
||||
*******************************************************************************/
|
||||
public static Outcome of(AiToolResult result, String operation, Integer httpStatus, Integer pendingActionId)
|
||||
{
|
||||
Outcome outcome = new Outcome(result);
|
||||
outcome.operation = operation;
|
||||
outcome.httpStatus = httpStatus;
|
||||
outcome.pendingActionId = pendingActionId;
|
||||
return (outcome);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for result
|
||||
*******************************************************************************/
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.assistant.tools.bulkload;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiGateway;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** How bulk_load_run starts a load (design spec #1171 section 3.4): one seam,
|
||||
** so the run can move from today's path to durable jobs without the tool
|
||||
** changing. Every implementation goes through the /qqq/v1 UI API as the user.
|
||||
*******************************************************************************/
|
||||
public interface BulkLoadRunner
|
||||
{
|
||||
/*******************************************************************************
|
||||
** A load to start: the target table, the upload, and the saved profile.
|
||||
*******************************************************************************/
|
||||
record Request(String sessionUUID, String userTimezone, UiApiGateway gateway, String tableName, String uploadRef, Integer savedBulkLoadProfileId)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What started: status uses the durable job status names (jobs_job.status:
|
||||
** AWAITING_CONFIRMATION, QUEUED, RUNNING, COMPLETED,
|
||||
** COMPLETED_WITH_ERRORS, FAILED, ...); jobId when it runs as a job,
|
||||
** processUUID when it ran synchronously; a short summary; the HTTP status.
|
||||
*******************************************************************************/
|
||||
record RunHandle(String status, Long jobId, String processUUID, String summary, String operation, Integer httpStatus)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start the load. Throws when it cannot start at all (the handle's status
|
||||
** reports a load that started and failed).
|
||||
*******************************************************************************/
|
||||
RunHandle start(Request request) throws QException;
|
||||
}
|
||||
+883
@@ -0,0 +1,883 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.assistant.tools.bulkload;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.assistant.access.DataAccessService;
|
||||
import com.kingsrook.qbits.assistant.engine.PolicyService;
|
||||
import com.kingsrook.qbits.assistant.provider.ModelToolDefinition;
|
||||
import com.kingsrook.qbits.assistant.tools.UiApiToolbox;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiCall;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.output.AiDataEnvelope;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.tools.AiJsonSchema;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.tools.AiToolException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The assistant's bulk-load tools (design spec #1171 section 3.4): use and
|
||||
** create saved bulk load profiles from a file the user attached in the chat.
|
||||
** Every call goes through the /qqq/v1 UI API with the user's own session, so
|
||||
** table and process permissions, record locks and the PHI rule are enforced
|
||||
** by the server, never by the model:
|
||||
** - bulk_profiles_list / bulk_profile_describe: the user's (and shared)
|
||||
** saved profiles, through the savedBulkLoadProfile table (record locks);
|
||||
** - bulk_file_attach: an upload the panel already made (by its ref), with
|
||||
** its column profiles (samples only where the server allows);
|
||||
** - bulk_mapping_suggest: core's suggester, then the model refines it;
|
||||
** - bulk_load_dry_run: the table's real bulk-insert validation, no store;
|
||||
** - bulk_profile_save (write): storeSavedBulkLoadProfile, and shares through
|
||||
** insertSharedRecord, after the user confirms;
|
||||
** - bulk_load_run (write): through {@link BulkLoadRunner}, after the user
|
||||
** confirms (and, as a durable job, again on its job record).
|
||||
**
|
||||
** The AI route header (X-QQQ-AI-Route: hipaa|standard) is set here from the
|
||||
** turn's provider, never from the model's input. Tool-call logs keep
|
||||
** profiles and counts, never sample values (see {@link #forLog}).
|
||||
**
|
||||
** STAND-IN(core=#1179, removeBy=RC.8): the profile, suggestMapping and
|
||||
** validate operations ship in a QQQ RC after 4.1.0-RC.5; until then the
|
||||
** tools report that the server does not offer them.
|
||||
*******************************************************************************/
|
||||
public class BulkLoadTools
|
||||
{
|
||||
public static final String LIST = "bulk_profiles_list";
|
||||
public static final String DESCRIBE = "bulk_profile_describe";
|
||||
public static final String ATTACH = "bulk_file_attach";
|
||||
public static final String SUGGEST = "bulk_mapping_suggest";
|
||||
public static final String DRY_RUN = "bulk_load_dry_run";
|
||||
public static final String SAVE = "bulk_profile_save";
|
||||
public static final String RUN = "bulk_load_run";
|
||||
|
||||
public static final Set<String> NAMES = Set.of(LIST, DESCRIBE, ATTACH, SUGGEST, DRY_RUN, SAVE, RUN);
|
||||
public static final Set<String> WRITE_TOOLS = Set.of(SAVE, RUN);
|
||||
|
||||
public static final String PROFILE_TABLE = "savedBulkLoadProfile";
|
||||
public static final String ROUTE_HEADER = "X-QQQ-AI-Route";
|
||||
|
||||
private static final Set<String> LOG_STRIPPED_KEYS = Set.of("sampleValues", "bulletsOfText", "valueMappings");
|
||||
|
||||
private final QInstance qInstance;
|
||||
private final DataAccessService dataAccess;
|
||||
private final BulkLoadRunner runner;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public BulkLoadTools(QInstance qInstance, DataAccessService dataAccess, BulkLoadRunner runner)
|
||||
{
|
||||
this.qInstance = qInstance;
|
||||
this.dataAccess = dataAccess;
|
||||
this.runner = runner == null ? new JobsLaunchBulkLoadRunner() : runner;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Offered when the instance has saved bulk load profiles.
|
||||
*******************************************************************************/
|
||||
public boolean offered()
|
||||
{
|
||||
return (qInstance != null && qInstance.getTable(PROFILE_TABLE) != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The definitions the policy allows (writes only when anyWritable).
|
||||
*******************************************************************************/
|
||||
public List<ModelToolDefinition> definitions(PolicyService.ToolPolicy policy, boolean anyWritable)
|
||||
{
|
||||
List<ModelToolDefinition> definitions = new ArrayList<>();
|
||||
if(!offered())
|
||||
{
|
||||
return (definitions);
|
||||
}
|
||||
|
||||
Map<String, Object> table = AiJsonSchema.string("The table to load into (from list_tables).");
|
||||
Map<String, Object> upload = AiJsonSchema.string("The upload ref of the file the user attached (from the conversation's attachments).");
|
||||
Map<String, Object> profile = new LinkedHashMap<>(Map.of("type", "object", "description", """
|
||||
A v1 bulk load profile: {"version":"v1","layout":"FLAT","hasHeaderRow":true,"fieldList":[{"fieldName":"...","columnIndex":0, \
|
||||
"headerName":"...","defaultValue":null,"doValueMapping":false,"valueMappings":{"file value":"field value"}}]}."""));
|
||||
|
||||
add(definitions, policy, LIST, "List the saved bulk load profiles (the user's own and those shared with them) for a table.",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("tableName", table), List.of("tableName")));
|
||||
add(definitions, policy, DESCRIBE, "Read one saved bulk load profile: its table, owner and column mapping.",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("id", AiJsonSchema.integer("The profile id (from bulk_profiles_list).")), List.of("id")));
|
||||
add(definitions, policy, ATTACH, """
|
||||
Look at a file the user attached for a bulk load: whether row 1 is a header, and each column's type, blanks, distinct count, \
|
||||
pattern and (only where the server allows) a few sample values.""",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("uploadRef", upload, "tableName", table), List.of("uploadRef", "tableName")));
|
||||
add(definitions, policy, SUGGEST, """
|
||||
Suggest a bulk load profile for an attached file: the server's header matching, with the columns it left unmapped, required \
|
||||
fields it did not fill, and fields whose values need mapping. Refine the profile yourself from the headers and column \
|
||||
profiles, then check it with bulk_load_dry_run.""",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("uploadRef", upload, "tableName", table), List.of("uploadRef", "tableName")));
|
||||
add(definitions, policy, DRY_RUN, """
|
||||
Dry-run a bulk load profile over an attached file through the table's real bulk insert validation. Nothing is stored. \
|
||||
Returns counts, summary lines and error groups with row numbers. SAMPLE checks the first 100 rows; FULL the whole file. \
|
||||
Fix the profile and run it again until there are no errors (or explain the rest to the user).""",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("uploadRef", upload, "tableName", table, "profile", profile,
|
||||
"mode", AiJsonSchema.enumOf(List.of("SAMPLE", "FULL"), "SAMPLE (default) or FULL.")), List.of("uploadRef", "tableName", "profile")));
|
||||
|
||||
if(anyWritable)
|
||||
{
|
||||
Map<String, Object> share = AiJsonSchema.object(AiJsonSchema.properties(
|
||||
"audienceType", AiJsonSchema.enumOf(List.of("user", "group"), "Share with a user (email) or a group."),
|
||||
"audienceId", AiJsonSchema.string("The user's email or the group's name."),
|
||||
"scope", AiJsonSchema.enumOf(List.of("READ_ONLY", "READ_WRITE"), "The share's scope (default READ_ONLY).")), List.of("audienceType", "audienceId"));
|
||||
add(definitions, policy, SAVE, """
|
||||
Save a bulk load profile (after a clean dry run) under a name, optionally sharing it. The user confirms before it is saved; \
|
||||
it is saved as the user, and shares are limited to what the user may share.""",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("tableName", table, "label", AiJsonSchema.string("The profile's name."), "profile", profile,
|
||||
"shares", AiJsonSchema.array(share, "Optional shares.")), List.of("tableName", "label", "profile")));
|
||||
add(definitions, policy, RUN, """
|
||||
Load an attached file into a table with a saved profile. The user confirms first; a large load runs as a background job that \
|
||||
the user starts from its job record.""",
|
||||
AiJsonSchema.object(AiJsonSchema.properties("tableName", table, "uploadRef", upload, "profileId", AiJsonSchema.integer("The saved profile id.")),
|
||||
List.of("tableName", "uploadRef", "profileId")));
|
||||
}
|
||||
return (definitions);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void add(List<ModelToolDefinition> definitions, PolicyService.ToolPolicy policy, String name, String description, Map<String, Object> schema)
|
||||
{
|
||||
if(policy == null || policy.allows(name))
|
||||
{
|
||||
definitions.add(new ModelToolDefinition().withName(name).withDescription(description).withInputSchema(schema).withStrict(false));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a tool. hipaaRoute is the turn's provider's HIPAA eligibility (sent
|
||||
** as X-QQQ-AI-Route; the model cannot set it).
|
||||
*******************************************************************************/
|
||||
public UiApiToolbox.Outcome execute(UiApiToolbox.Context context, String toolCallId, String toolName, Map<String, Object> input, boolean hipaaRoute)
|
||||
{
|
||||
AiToolResult result = new AiToolResult().withToolCallId(toolCallId).withToolName(toolName);
|
||||
Map<String, Object> in = input == null ? Map.of() : input;
|
||||
Call call = new Call(context, hipaaRoute);
|
||||
Integer actionId = null;
|
||||
long started = System.nanoTime();
|
||||
try
|
||||
{
|
||||
if(context.policy() != null && !context.policy().allows(toolName))
|
||||
{
|
||||
deny(result, "policy:tool", "The administrator has not allowed the " + toolName + " tool.");
|
||||
return (UiApiToolbox.Outcome.of(result, null, null, null));
|
||||
}
|
||||
|
||||
String tableName = in.get("tableName") instanceof String t ? t.strip() : null;
|
||||
if(tableName != null)
|
||||
{
|
||||
result.setTableName(tableName);
|
||||
if(!dataAccess.forTable(tableName).mode().allowsRead())
|
||||
{
|
||||
deny(result, "dataAccess:" + dataAccess.forTable(tableName).mode(), "The assistant may not use the " + tableName + " table.");
|
||||
return (UiApiToolbox.Outcome.of(result, null, null, null));
|
||||
}
|
||||
}
|
||||
|
||||
switch(toolName)
|
||||
{
|
||||
case LIST -> list(call, result, required(in, "tableName"));
|
||||
case DESCRIBE -> describe(call, result, requiredInteger(in, "id"));
|
||||
case ATTACH -> attach(call, result, required(in, "tableName"), required(in, "uploadRef"));
|
||||
case SUGGEST -> suggest(call, result, required(in, "tableName"), required(in, "uploadRef"));
|
||||
case DRY_RUN -> dryRun(call, result, required(in, "tableName"), required(in, "uploadRef"), requiredMap(in, "profile"), in.get("mode") instanceof String m ? m : "SAMPLE");
|
||||
case SAVE, RUN -> actionId = propose(context, result, toolName, required(in, "tableName"), in);
|
||||
default -> throw (AiToolException.invalidInput("Unknown tool \"" + toolName + "\"."));
|
||||
}
|
||||
}
|
||||
catch(AiToolException e)
|
||||
{
|
||||
result.withOutcome(e.getOutcome() == null ? AiToolOutcome.INVALID_INPUT : e.getOutcome()).withSummary(shortText(e.getMessage())).withModelContent(e.getMessage());
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
result.withOutcome(AiToolOutcome.ERROR).withSummary("Failed").withModelContent("The tool failed: " + shortText(e.getMessage()));
|
||||
}
|
||||
finally
|
||||
{
|
||||
result.setDurationMs((System.nanoTime() - started) / 1_000_000);
|
||||
}
|
||||
return (UiApiToolbox.Outcome.of(result, call.operation, call.httpStatus, actionId));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a confirmed write (from a pending action) with the confirming user's
|
||||
** session.
|
||||
*******************************************************************************/
|
||||
public UiApiToolbox.Outcome executeConfirmed(UiApiToolbox.Context context, String toolCallId, String toolName, String tableName, Map<String, Object> values)
|
||||
{
|
||||
AiToolResult result = new AiToolResult().withToolCallId(toolCallId).withToolName(toolName).withTableName(tableName);
|
||||
Call call = new Call(context, false);
|
||||
long started = System.nanoTime();
|
||||
try
|
||||
{
|
||||
if(!WRITE_TOOLS.contains(toolName))
|
||||
{
|
||||
throw (AiToolException.invalidInput("Not a write tool: " + toolName));
|
||||
}
|
||||
DataAccessService.Decision decision = dataAccess.forTable(SAVE.equals(toolName) ? PROFILE_TABLE : tableName);
|
||||
if(!decision.mode().allowsWrite())
|
||||
{
|
||||
deny(result, "dataAccess:" + decision.mode(), "The assistant may not write here (data access " + decision.mode() + ", " + decision.source() + ").");
|
||||
return (UiApiToolbox.Outcome.of(result, null, null, null));
|
||||
}
|
||||
if(SAVE.equals(toolName))
|
||||
{
|
||||
save(call, result, tableName, values);
|
||||
}
|
||||
else
|
||||
{
|
||||
run(call, result, context, tableName, values);
|
||||
}
|
||||
}
|
||||
catch(AiToolException e)
|
||||
{
|
||||
result.withOutcome(e.getOutcome() == null ? AiToolOutcome.INVALID_INPUT : e.getOutcome()).withSummary(shortText(e.getMessage())).withModelContent(e.getMessage());
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
result.withOutcome(AiToolOutcome.ERROR).withSummary("Failed").withModelContent("The write failed: " + shortText(e.getMessage()));
|
||||
}
|
||||
finally
|
||||
{
|
||||
result.setDurationMs((System.nanoTime() - started) / 1_000_000);
|
||||
}
|
||||
return (UiApiToolbox.Outcome.of(result, call.operation, call.httpStatus, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// read tools //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** POST /table/savedBulkLoadProfile/query: the profiles the record locks let
|
||||
** the user see, for one table.
|
||||
*******************************************************************************/
|
||||
private void list(Call call, AiToolResult result, String tableName) throws QException
|
||||
{
|
||||
Map<String, Object> filter = Map.of(
|
||||
"criteria", List.of(Map.of("fieldName", "tableName", "operator", "EQUALS", "values", List.of(tableName))),
|
||||
"orderBys", List.of(Map.of("fieldName", "label", "isAscending", true)));
|
||||
UiApiResult response = call.invoke(new UiApiCall("POST", "/table/{tableName}/query").withPathParam("tableName", PROFILE_TABLE)
|
||||
.withJsonBody(Map.of("filter", filter)));
|
||||
if(response == null || !response.isSuccess())
|
||||
{
|
||||
fail(result, call, response);
|
||||
return;
|
||||
}
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
for(Object record : list(response.json().get("records")))
|
||||
{
|
||||
Map<String, Object> values = map(map(record).get("values"));
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("id", values.get("id"));
|
||||
row.put("label", values.get("label"));
|
||||
row.put("owner", values.get("userId"));
|
||||
row.put("fieldCount", fieldList(values.get("mappingJson")).size());
|
||||
rows.add(row);
|
||||
}
|
||||
result.withOutcome(AiToolOutcome.OK).withRowCount(rows.size()).withSummary("Found " + rows.size() + " saved profile" + (rows.size() == 1 ? "" : "s"))
|
||||
.withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(AiDataEnvelope.wrap(LIST, PROFILE_TABLE, JsonUtils.toJson(Map.of("tableName", tableName, "profiles", rows))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** GET /table/savedBulkLoadProfile/{id}: one profile (when the locks allow).
|
||||
*******************************************************************************/
|
||||
private void describe(Call call, AiToolResult result, Integer id) throws QException
|
||||
{
|
||||
UiApiResult response = call.invoke(new UiApiCall("GET", "/table/{tableName}/{primaryKey}").withPathParam("tableName", PROFILE_TABLE).withPathParam("primaryKey", id));
|
||||
if(response == null || !response.isSuccess())
|
||||
{
|
||||
fail(result, call, response);
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = map(map(response.json().get("record")).get("values"));
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
out.put("id", values.get("id"));
|
||||
out.put("label", values.get("label"));
|
||||
out.put("tableName", values.get("tableName"));
|
||||
out.put("owner", values.get("userId"));
|
||||
out.put("profile", values.get("mappingJson") == null ? null : new JSONObject(String.valueOf(values.get("mappingJson"))).toMap());
|
||||
result.setTableName(String.valueOf(values.get("tableName")));
|
||||
result.withOutcome(AiToolOutcome.OK).withRowCount(1).withSummary("Read profile " + values.get("label"))
|
||||
.withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(AiDataEnvelope.wrap(DESCRIBE, PROFILE_TABLE, JsonUtils.toJson(out)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** GET /bulkLoad/uploads/{uploadRef}/profile.
|
||||
*******************************************************************************/
|
||||
private void attach(Call call, AiToolResult result, String tableName, String uploadRef) throws QException
|
||||
{
|
||||
UiApiResult response = call.invoke(new UiApiCall("GET", "/bulkLoad/uploads/{uploadRef}/profile").withPathParam("uploadRef", uploadRef)
|
||||
.withQueryParam("tableName", tableName).withQueryParam("sampleCount", 5));
|
||||
if(response == null || !response.isSuccess())
|
||||
{
|
||||
fail(result, call, response);
|
||||
return;
|
||||
}
|
||||
Map<String, Object> json = response.json();
|
||||
int columns = list(json.get("columns")).size();
|
||||
result.withOutcome(AiToolOutcome.OK).withRowCount(columns).withSummary("Read " + json.get("fileName") + ": " + columns + " columns")
|
||||
.withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(AiDataEnvelope.wrap(ATTACH, tableName, response.body()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** POST /bulkLoad/{tableName}/suggestMapping.
|
||||
*******************************************************************************/
|
||||
private void suggest(Call call, AiToolResult result, String tableName, String uploadRef) throws QException
|
||||
{
|
||||
UiApiResult response = call.invoke(new UiApiCall("POST", "/bulkLoad/{tableName}/suggestMapping").withPathParam("tableName", tableName)
|
||||
.withJsonBody(Map.of("uploadRef", uploadRef)));
|
||||
if(response == null || !response.isSuccess())
|
||||
{
|
||||
fail(result, call, response);
|
||||
return;
|
||||
}
|
||||
Map<String, Object> json = response.json();
|
||||
int mapped = list(map(json.get("profile")).get("fieldList")).size();
|
||||
result.withOutcome(AiToolOutcome.OK).withRowCount(mapped).withSummary("Suggested a mapping of " + mapped + " field" + (mapped == 1 ? "" : "s"))
|
||||
.withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(AiDataEnvelope.wrap(SUGGEST, tableName, response.body()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** POST /bulkLoad/{tableName}/validate (nothing is stored).
|
||||
*******************************************************************************/
|
||||
private void dryRun(Call call, AiToolResult result, String tableName, String uploadRef, Map<String, Object> profile, String mode) throws QException
|
||||
{
|
||||
UiApiResult response = call.invoke(new UiApiCall("POST", "/bulkLoad/{tableName}/validate").withPathParam("tableName", tableName)
|
||||
.withJsonBody(Map.of("uploadRef", uploadRef, "profile", profile, "mode", "FULL".equalsIgnoreCase(mode) ? "FULL" : "SAMPLE"))
|
||||
.withTimeout(Duration.ofSeconds(120)));
|
||||
if(response == null || !response.isSuccess())
|
||||
{
|
||||
fail(result, call, response);
|
||||
return;
|
||||
}
|
||||
Map<String, Object> json = response.json();
|
||||
Integer ok = ValueUtils.getValueAsInteger(json.get("okCount"));
|
||||
Integer errors = ValueUtils.getValueAsInteger(json.get("errorCount"));
|
||||
result.withOutcome(AiToolOutcome.OK).withRowCount(ok)
|
||||
.withSummary("Dry run (" + json.get("mode") + "): " + ok + " ok, " + errors + " with errors")
|
||||
.withPermissionCheck(call.permissionCheck())
|
||||
.withRender(render("bulk_load_dry_run", tableName, json))
|
||||
.withModelContent(AiDataEnvelope.wrap(DRY_RUN, tableName, response.body()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// write tools //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Propose a save or a run as a pending action (the user confirms in the
|
||||
** panel; nothing happens until then).
|
||||
*******************************************************************************/
|
||||
private Integer propose(UiApiToolbox.Context context, AiToolResult result, String toolName, String tableName, Map<String, Object> in) throws QException
|
||||
{
|
||||
DataAccessService.Decision decision = dataAccess.forTable(SAVE.equals(toolName) ? PROFILE_TABLE : tableName);
|
||||
if(!decision.mode().allowsWrite())
|
||||
{
|
||||
deny(result, "dataAccess:" + decision.mode(), "The assistant may only read here (data access " + decision.mode() + ", " + decision.source() + "). Writes need READ_WRITE.");
|
||||
return (null);
|
||||
}
|
||||
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
String summary;
|
||||
if(SAVE.equals(toolName))
|
||||
{
|
||||
String label = required(in, "label");
|
||||
values.put("label", label);
|
||||
values.put("profile", requiredMap(in, "profile"));
|
||||
if(in.get("shares") instanceof List<?> shares && !shares.isEmpty())
|
||||
{
|
||||
values.put("shares", shares);
|
||||
}
|
||||
summary = "Save the bulk load profile \"" + label + "\" for " + tableName + (values.containsKey("shares") ? " and share it" : "");
|
||||
}
|
||||
else
|
||||
{
|
||||
values.put("uploadRef", required(in, "uploadRef"));
|
||||
values.put("profileId", requiredInteger(in, "profileId"));
|
||||
summary = "Load the attached file into " + tableName + " with saved profile " + values.get("profileId");
|
||||
}
|
||||
|
||||
if(context.pendingActions() == null)
|
||||
{
|
||||
throw (new QException("The " + toolName + " action could not be proposed."));
|
||||
}
|
||||
String operation = SAVE.equals(toolName) ? "POST /processes/{processName}/init" : "POST " + JobsLaunchBulkLoadRunner.LAUNCH_OPERATION_PATH;
|
||||
Integer actionId = context.pendingActions().propose(new UiApiToolbox.ProposedWrite(result.getToolCallId(), toolName, operation, tableName, null, values, summary));
|
||||
if(actionId == null)
|
||||
{
|
||||
throw (new QException("The " + toolName + " action could not be proposed."));
|
||||
}
|
||||
|
||||
Map<String, Object> render = new LinkedHashMap<>();
|
||||
render.put("type", "pending_action");
|
||||
render.put("actionId", actionId);
|
||||
render.put("toolName", toolName);
|
||||
render.put("tableName", tableName);
|
||||
render.put("summary", summary);
|
||||
result.withOutcome(AiToolOutcome.OK).withSummary("Waiting for your confirmation").withRender(render)
|
||||
.withModelContent("Proposed (pending action " + actionId + "): " + summary + ". Nothing happens until the user confirms it in the panel.");
|
||||
return (actionId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Confirmed save: storeSavedBulkLoadProfile, then insertSharedRecord per
|
||||
** share (core lets only the owner share).
|
||||
*******************************************************************************/
|
||||
private void save(Call call, AiToolResult result, String tableName, Map<String, Object> values) throws QException
|
||||
{
|
||||
Map<String, Object> processValues = new LinkedHashMap<>();
|
||||
processValues.put("tableName", tableName);
|
||||
processValues.put("label", values.get("label"));
|
||||
processValues.put("mappingJson", JsonUtils.toJson(values.get("profile")));
|
||||
processValues.put("isBulkEdit", false);
|
||||
UiApiResult response = call.invoke(new UiApiCall("POST", "/processes/{processName}/init").withPathParam("processName", "storeSavedBulkLoadProfile")
|
||||
.withFormParam("values", JsonUtils.toJson(processValues)));
|
||||
String error = processError(response);
|
||||
if(error != null)
|
||||
{
|
||||
result.withOutcome(response == null || response.isSuccess() ? AiToolOutcome.ERROR : outcomeFor(response.status())).withSummary(shortText(error)).withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent("The profile was not saved: " + error);
|
||||
return;
|
||||
}
|
||||
Integer id = ValueUtils.getValueAsInteger(map(map(list(map(response.json().get("values")).get("savedBulkLoadProfileList")).stream().findFirst().orElse(null)).get("values")).get("id"));
|
||||
|
||||
List<String> shared = new ArrayList<>();
|
||||
List<String> failed = new ArrayList<>();
|
||||
for(Object share : list(values.get("shares")))
|
||||
{
|
||||
Map<String, Object> s = map(share);
|
||||
Map<String, Object> shareValues = new LinkedHashMap<>();
|
||||
shareValues.put("tableName", PROFILE_TABLE);
|
||||
shareValues.put("recordId", id);
|
||||
shareValues.put("audienceType", s.get("audienceType"));
|
||||
shareValues.put("audienceId", s.get("audienceId"));
|
||||
shareValues.put("scopeId", s.get("scope") == null ? "READ_ONLY" : s.get("scope"));
|
||||
UiApiResult shareResponse = call.invoke(new UiApiCall("POST", "/processes/{processName}/init").withPathParam("processName", "insertSharedRecord")
|
||||
.withFormParam("values", JsonUtils.toJson(shareValues)));
|
||||
String shareError = processError(shareResponse);
|
||||
(shareError == null ? shared : failed).add(s.get("audienceType") + " " + s.get("audienceId") + (shareError == null ? "" : " (" + shortText(shareError) + ")"));
|
||||
}
|
||||
|
||||
String summary = "Saved profile \"" + values.get("label") + "\" (id " + id + ")" + (shared.isEmpty() ? "" : ", shared with " + String.join(", ", shared))
|
||||
+ (failed.isEmpty() ? "" : "; not shared with " + String.join(", ", failed));
|
||||
result.withOutcome(AiToolOutcome.OK).withRowCount(1).withSummary(shortText(summary)).withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(JsonUtils.toJson(Map.of("id", id == null ? "" : id, "label", values.get("label"), "shared", shared, "notShared", failed)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Confirmed run: through the runner seam.
|
||||
*******************************************************************************/
|
||||
private void run(Call call, AiToolResult result, UiApiToolbox.Context context, String tableName, Map<String, Object> values) throws QException
|
||||
{
|
||||
BulkLoadRunner.RunHandle handle = runner.start(new BulkLoadRunner.Request(context.sessionUUID(), context.userTimezone(), context.gateway(), tableName,
|
||||
String.valueOf(values.get("uploadRef")), ValueUtils.getValueAsInteger(values.get("profileId"))));
|
||||
call.operation = handle.operation();
|
||||
call.httpStatus = handle.httpStatus();
|
||||
boolean ok = !"FAILED".equals(handle.status()) && !"NOT_AVAILABLE".equals(handle.status());
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
out.put("status", handle.status());
|
||||
out.put("jobId", handle.jobId());
|
||||
out.put("processUUID", handle.processUUID());
|
||||
out.put("summary", handle.summary());
|
||||
result.withOutcome(ok ? AiToolOutcome.OK : AiToolOutcome.ERROR).withSummary(shortText(handle.summary())).withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(JsonUtils.toJson(out));
|
||||
}
|
||||
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// helpers //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A tool's input or result as kept in the tool-call log: profiles and
|
||||
** counts, never sample values, example bullets or value mappings.
|
||||
*******************************************************************************/
|
||||
public static String forLog(String json)
|
||||
{
|
||||
if(json == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
try
|
||||
{
|
||||
int start = json.indexOf('{');
|
||||
int end = json.lastIndexOf('}');
|
||||
if(start < 0 || end < start)
|
||||
{
|
||||
return (json);
|
||||
}
|
||||
JSONObject object = new JSONObject(json.substring(start, end + 1));
|
||||
strip(object);
|
||||
return (object.toString());
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
return ("{\"note\":\"not logged\"}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void strip(Object node)
|
||||
{
|
||||
if(node instanceof JSONObject object)
|
||||
{
|
||||
for(String key : new ArrayList<>(object.keySet()))
|
||||
{
|
||||
if(LOG_STRIPPED_KEYS.contains(key))
|
||||
{
|
||||
object.remove(key);
|
||||
}
|
||||
else
|
||||
{
|
||||
strip(object.get(key));
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(node instanceof JSONArray array)
|
||||
{
|
||||
for(int i = 0; i < array.length(); i++)
|
||||
{
|
||||
strip(array.get(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A compact render for the panel (counts and error groups, no values).
|
||||
*******************************************************************************/
|
||||
private static Map<String, Object> render(String type, String tableName, Map<String, Object> json)
|
||||
{
|
||||
Map<String, Object> render = new LinkedHashMap<>();
|
||||
render.put("type", type);
|
||||
render.put("tableName", tableName);
|
||||
render.put("mode", json.get("mode"));
|
||||
render.put("okCount", json.get("okCount"));
|
||||
render.put("errorCount", json.get("errorCount"));
|
||||
List<Map<String, Object>> errors = new ArrayList<>();
|
||||
for(Object error : list(json.get("errors")))
|
||||
{
|
||||
Map<String, Object> e = map(error);
|
||||
errors.add(new LinkedHashMap<>(Map.of("rows", String.valueOf(e.get("rowNumbers")), "field", String.valueOf(e.get("fieldName")), "message", String.valueOf(e.get("message")))));
|
||||
}
|
||||
render.put("errors", errors);
|
||||
return (render);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The error of a process init call (HTTP or the process's ERROR type), or
|
||||
** null when it completed.
|
||||
*******************************************************************************/
|
||||
private static String processError(UiApiResult response)
|
||||
{
|
||||
if(response == null)
|
||||
{
|
||||
return ("No response.");
|
||||
}
|
||||
if(!response.isSuccess())
|
||||
{
|
||||
return (response.errorMessage() == null ? "HTTP " + response.status() : response.errorMessage());
|
||||
}
|
||||
Map<String, Object> json = response.json();
|
||||
if("ERROR".equals(json.get("type")))
|
||||
{
|
||||
return (String.valueOf(json.getOrDefault("userFacingError", json.getOrDefault("error", "The process failed."))));
|
||||
}
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void fail(AiToolResult result, Call call, UiApiResult response)
|
||||
{
|
||||
if(response == null)
|
||||
{
|
||||
result.withOutcome(AiToolOutcome.ERROR).withSummary("Not available")
|
||||
.withModelContent("This server does not offer " + call.operation + " yet (it needs a newer QQQ). Use the table's Bulk Load screen instead.");
|
||||
return;
|
||||
}
|
||||
AiToolOutcome kind = outcomeFor(response.status());
|
||||
String message = response.errorMessage() == null ? "HTTP " + response.status() : response.errorMessage();
|
||||
result.withOutcome(kind).withSummary(kind == AiToolOutcome.DENIED ? "Not allowed" : shortText(message)).withPermissionCheck(call.permissionCheck())
|
||||
.withModelContent(call.operation + " answered HTTP " + response.status() + ": " + shortText(message));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static AiToolOutcome outcomeFor(int status)
|
||||
{
|
||||
return (switch(status)
|
||||
{
|
||||
case 401, 403 -> AiToolOutcome.DENIED;
|
||||
case 404 -> AiToolOutcome.NOT_FOUND;
|
||||
case 400, 409, 422 -> AiToolOutcome.INVALID_INPUT;
|
||||
default -> AiToolOutcome.ERROR;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void deny(AiToolResult result, String check, String message)
|
||||
{
|
||||
result.withOutcome(AiToolOutcome.DENIED).withSummary("Not allowed").withPermissionCheck(check).withModelContent(message);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String required(Map<String, Object> in, String name) throws AiToolException
|
||||
{
|
||||
if(in.get(name) instanceof String s && StringUtils.hasContent(s))
|
||||
{
|
||||
return (s.strip());
|
||||
}
|
||||
throw (AiToolException.invalidInput(name + " is required."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Integer requiredInteger(Map<String, Object> in, String name) throws AiToolException
|
||||
{
|
||||
try
|
||||
{
|
||||
Integer value = ValueUtils.getValueAsInteger(in.get(name));
|
||||
if(value != null)
|
||||
{
|
||||
return (value);
|
||||
}
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
// fall through
|
||||
}
|
||||
throw (AiToolException.invalidInput(name + " must be a number."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> requiredMap(Map<String, Object> in, String name) throws AiToolException
|
||||
{
|
||||
Object value = in.get(name);
|
||||
if(value instanceof String s && s.strip().startsWith("{"))
|
||||
{
|
||||
return (new JSONObject(s).toMap());
|
||||
}
|
||||
if(value instanceof Map<?, ?> m && !m.isEmpty())
|
||||
{
|
||||
return ((Map<String, Object>) m);
|
||||
}
|
||||
throw (AiToolException.invalidInput(name + " is required (a JSON object)."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<Object> fieldList(Object mappingJson)
|
||||
{
|
||||
if(mappingJson == null)
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
try
|
||||
{
|
||||
return (new JSONObject(String.valueOf(mappingJson)).optJSONArray("fieldList") == null ? List.of()
|
||||
: new JSONObject(String.valueOf(mappingJson)).getJSONArray("fieldList").toList());
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> map(Object value)
|
||||
{
|
||||
return (value instanceof Map<?, ?> m ? (Map<String, Object>) m : Map.of());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Object> list(Object value)
|
||||
{
|
||||
return (value instanceof List<?> l ? (List<Object>) l : List.of());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String shortText(String text)
|
||||
{
|
||||
if(text == null)
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
return (text.length() > 300 ? text.substring(0, 297) + "..." : text);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One tool call's UI API invocations: the user's session, the route header,
|
||||
** and the last operation and status (for the log).
|
||||
*******************************************************************************/
|
||||
private static class Call
|
||||
{
|
||||
private final UiApiToolbox.Context context;
|
||||
private final boolean hipaaRoute;
|
||||
private String operation;
|
||||
private Integer httpStatus;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
Call(UiApiToolbox.Context context, boolean hipaaRoute)
|
||||
{
|
||||
this.context = context;
|
||||
this.hipaaRoute = hipaaRoute;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invoke, or null when the server does not offer the operation (an
|
||||
** older QQQ).
|
||||
*******************************************************************************/
|
||||
UiApiResult invoke(UiApiCall uiApiCall) throws QException
|
||||
{
|
||||
operation = uiApiCall.operation();
|
||||
if(!context.gateway().isInvocable(uiApiCall.getMethod(), uiApiCall.getPathTemplate()))
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
uiApiCall.withSessionUUID(context.sessionUUID()).withUserTimezone(context.userTimezone()).withInvokedBy("assistant")
|
||||
.withHeader(ROUTE_HEADER, hipaaRoute ? "hipaa" : "standard");
|
||||
UiApiResult response = context.gateway().invoke(uiApiCall);
|
||||
httpStatus = response.status();
|
||||
return (response);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
String permissionCheck()
|
||||
{
|
||||
return ("uiApi:" + operation + ":" + httpStatus);
|
||||
}
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.assistant.tools.bulkload;
|
||||
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiCall;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiResult;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The default {@link BulkLoadRunner}: the durable jobs launch,
|
||||
** POST /qqq/v1/processes/{table}.bulkInsert/jobs with the upload ref and the
|
||||
** saved profile (design spec #1171 sections 2.4 and 3.4). From the assistant
|
||||
** channel core creates the job AWAITING_CONFIRMATION, so it starts only when
|
||||
** the user confirms it outside the assistant.
|
||||
**
|
||||
** STAND-IN(core=#1178, removeBy=RC.8): until the jobs kernel ships in a QQQ
|
||||
** RC, the launch is not a spec'd operation on the server; the runner then
|
||||
** says so instead of guessing at another path (a load needs the upload ref,
|
||||
** which the bulk-insert process endpoints cannot take).
|
||||
*******************************************************************************/
|
||||
public class JobsLaunchBulkLoadRunner implements BulkLoadRunner
|
||||
{
|
||||
public static final String LAUNCH_OPERATION_PATH = "/processes/{processName}/jobs";
|
||||
|
||||
public static final String NOT_AVAILABLE = "Running a bulk load from the assistant needs the durable jobs launch "
|
||||
+ "(POST /qqq/v1/processes/{process}/jobs), which this server does not have yet. Open the table's Bulk Load "
|
||||
+ "screen and choose the saved profile instead.";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RunHandle start(Request request) throws QException
|
||||
{
|
||||
String processName = request.tableName() + ".bulkInsert";
|
||||
if(!request.gateway().isInvocable("POST", LAUNCH_OPERATION_PATH))
|
||||
{
|
||||
return (new RunHandle("NOT_AVAILABLE", null, null, NOT_AVAILABLE, "POST " + LAUNCH_OPERATION_PATH, null));
|
||||
}
|
||||
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
body.put("uploadRefs", List.of(request.uploadRef()));
|
||||
body.put("savedBulkLoadProfileId", request.savedBulkLoadProfileId());
|
||||
body.put("mode", "EXECUTE");
|
||||
body.put("label", "Bulk load into " + request.tableName() + " (assistant)");
|
||||
|
||||
UiApiResult response = request.gateway().invoke(new UiApiCall("POST", LAUNCH_OPERATION_PATH)
|
||||
.withPathParam("processName", processName)
|
||||
.withJsonBody(body)
|
||||
.withSessionUUID(request.sessionUUID())
|
||||
.withUserTimezone(request.userTimezone())
|
||||
.withInvokedBy("assistant"));
|
||||
|
||||
if(!response.isSuccess())
|
||||
{
|
||||
return (new RunHandle("FAILED", null, null, response.errorMessage() == null ? "HTTP " + response.status() : response.errorMessage(),
|
||||
"POST " + LAUNCH_OPERATION_PATH, response.status()));
|
||||
}
|
||||
|
||||
Map<String, Object> json = response.json();
|
||||
Long jobId = ValueUtils.getValueAsLong(json.get("jobId"));
|
||||
String status = json.get("status") == null ? "QUEUED" : String.valueOf(json.get("status"));
|
||||
String summary = "AWAITING_CONFIRMATION".equals(status)
|
||||
? "Job " + jobId + " is waiting for your confirmation on its job record before it starts."
|
||||
: "Job " + jobId + " is " + status.toLowerCase() + ".";
|
||||
return (new RunHandle(status, jobId, null, summary, "POST " + LAUNCH_OPERATION_PATH, response.status()));
|
||||
}
|
||||
}
|
||||
+28
-1
@@ -30,6 +30,7 @@ import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
@@ -64,6 +65,9 @@ public class LoopbackUiApiGateway implements UiApiGateway
|
||||
public static final String INVOKED_BY_HEADER = "X-QQQ-Invoked-By";
|
||||
public static final String TIMEZONE_HEADER = "X-QQQ-UserTimezone";
|
||||
|
||||
/** Headers a call cannot set itself (lower case). */
|
||||
private static final Set<String> RESERVED_HEADERS = Set.of("cookie", "authorization", "x-api-key", "content-type", "accept", INVOKED_BY_HEADER.toLowerCase(), TIMEZONE_HEADER.toLowerCase());
|
||||
|
||||
private static final Set<String> SPEC_OPERATIONS = specOperations();
|
||||
|
||||
private final Supplier<String> baseUrlSupplier;
|
||||
@@ -204,8 +208,31 @@ public class LoopbackUiApiGateway implements UiApiGateway
|
||||
{
|
||||
builder.header(TIMEZONE_HEADER, call.getUserTimezone());
|
||||
}
|
||||
call.getHeaders().forEach((name, value) ->
|
||||
{
|
||||
if(!RESERVED_HEADERS.contains(name.toLowerCase()))
|
||||
{
|
||||
builder.header(name, value);
|
||||
}
|
||||
});
|
||||
|
||||
if(call.getJsonBody() != null)
|
||||
if(!call.getFormParams().isEmpty())
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// multipart/form-data text fields (the /processes endpoints) //
|
||||
////////////////////////////////////////////////////////////////
|
||||
String boundary = "qqq-assistant-" + UUID.randomUUID();
|
||||
StringBuilder body = new StringBuilder();
|
||||
String lineEnd = "\r\n";
|
||||
call.getFormParams().forEach((name, value) -> body.append("--").append(boundary).append(lineEnd)
|
||||
.append("Content-Disposition: form-data; name=\"").append(name.replace("\"", "")).append("\"").append(lineEnd)
|
||||
.append("Content-Type: text/plain; charset=UTF-8").append(lineEnd).append(lineEnd)
|
||||
.append(value).append(lineEnd));
|
||||
body.append("--").append(boundary).append("--").append(lineEnd);
|
||||
builder.header("Content-Type", "multipart/form-data; boundary=" + boundary);
|
||||
builder.method(call.getMethod(), HttpRequest.BodyPublishers.ofString(body.toString(), StandardCharsets.UTF_8));
|
||||
}
|
||||
else if(call.getJsonBody() != null)
|
||||
{
|
||||
builder.header("Content-Type", "application/json");
|
||||
builder.method(call.getMethod(), HttpRequest.BodyPublishers.ofString(call.getJsonBody() instanceof String s ? s : JsonUtils.toJson(call.getJsonBody()), StandardCharsets.UTF_8));
|
||||
|
||||
@@ -33,6 +33,8 @@ public class UiApiCall
|
||||
private final String pathTemplate;
|
||||
private final Map<String, String> pathParams = new LinkedHashMap<>();
|
||||
private final Map<String, String> queryParams = new LinkedHashMap<>();
|
||||
private final Map<String, String> formParams = new LinkedHashMap<>();
|
||||
private final Map<String, String> headers = new LinkedHashMap<>();
|
||||
private Object jsonBody;
|
||||
private String sessionUUID;
|
||||
private String userTimezone;
|
||||
@@ -128,6 +130,58 @@ public class UiApiCall
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a multipart form field (ignored when null). A call with form fields
|
||||
** is sent as multipart/form-data (the /processes endpoints), mirroring
|
||||
** core's UiApiRequest.withFormParam.
|
||||
*******************************************************************************/
|
||||
public UiApiCall withFormParam(String name, Object value)
|
||||
{
|
||||
if(value != null)
|
||||
{
|
||||
formParams.put(name, String.valueOf(value));
|
||||
}
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for formParams
|
||||
*******************************************************************************/
|
||||
public Map<String, String> getFormParams()
|
||||
{
|
||||
return (formParams);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a request header the invoker sets (e.g. X-QQQ-AI-Route); ignored
|
||||
** when null. The session cookie and the invoked-by marker cannot be
|
||||
** overridden this way.
|
||||
*******************************************************************************/
|
||||
public UiApiCall withHeader(String name, Object value)
|
||||
{
|
||||
if(name != null && value != null)
|
||||
{
|
||||
headers.put(name, String.valueOf(value));
|
||||
}
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for headers
|
||||
*******************************************************************************/
|
||||
public Map<String, String> getHeaders()
|
||||
{
|
||||
return (headers);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for jsonBody
|
||||
*******************************************************************************/
|
||||
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.assistant.tools.bulkload;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import com.kingsrook.qbits.assistant.AssistantQBitConfig;
|
||||
import com.kingsrook.qbits.assistant.access.DataAccessMode;
|
||||
import com.kingsrook.qbits.assistant.api.AssistantApi;
|
||||
import com.kingsrook.qbits.assistant.engine.PolicyService;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantDataAccess;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantPendingAction;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantToolCall;
|
||||
import com.kingsrook.qbits.assistant.provider.ModelPart;
|
||||
import com.kingsrook.qbits.assistant.provider.ModelResponse;
|
||||
import com.kingsrook.qbits.assistant.testing.AssistantTestFixture;
|
||||
import com.kingsrook.qbits.assistant.testing.BaseAssistantTest;
|
||||
import com.kingsrook.qbits.assistant.testing.TurnRunner;
|
||||
import com.kingsrook.qbits.assistant.tools.UiApiToolbox;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiCall;
|
||||
import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.assistant.testing.EvalSupport.firstResult;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The bulk-load tools: definitions, the UI API calls they make (as the
|
||||
** user, with the route header from the provider, never the model), pending
|
||||
** actions for writes, the confirmed save and run, and log hygiene. Plus the
|
||||
** golden eval: a messy CSV -> profile -> dry run with an error -> fixed ->
|
||||
** saved -> loaded.
|
||||
*******************************************************************************/
|
||||
class BulkLoadToolsTest extends BaseAssistantTest
|
||||
{
|
||||
private final FakeBulkLoadServer server = new FakeBulkLoadServer(true);
|
||||
private final List<BulkLoadRunner.Request> runs = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The fake server as the gateway, and a recording runner.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected void customizeConfig(AssistantQBitConfig config)
|
||||
{
|
||||
config.withUiApiGateway(server).withBulkLoadRunner(request ->
|
||||
{
|
||||
runs.add(request);
|
||||
return (new BulkLoadRunner.RunHandle("AWAITING_CONFIRMATION", 42L, null, "Job 42 is waiting for your confirmation on its job record.", "POST /processes/{processName}/jobs", 201));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The saved-profile tables (so the tools are offered), and write access.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws QException
|
||||
{
|
||||
new SavedBulkLoadProfileMetaDataProvider().defineAll(qInstance, AssistantTestFixture.BACKEND, null);
|
||||
setAccess(BulkLoadTools.PROFILE_TABLE, DataAccessMode.READ_WRITE);
|
||||
setAccess("blog_post", DataAccessMode.READ_WRITE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void clear()
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void setAccess(String tableName, DataAccessMode mode) throws QException
|
||||
{
|
||||
QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN));
|
||||
try
|
||||
{
|
||||
config.getRuntime().getStore().insert(new AssistantDataAccess().withObjectType(AssistantDataAccess.TYPE_TABLE).withObjectName(tableName).withMode(mode.name()));
|
||||
config.getRuntime().getDataAccessService().invalidate();
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One tool call as the turn engine makes it.
|
||||
*******************************************************************************/
|
||||
private UiApiToolbox.Outcome tool(String toolName, Map<String, Object> in, boolean hipaa)
|
||||
{
|
||||
QContext.init(qInstance, AssistantTestFixture.ownerSession(AssistantTestFixture.ADMIN, config));
|
||||
try
|
||||
{
|
||||
return (config.getRuntime().getBulkLoadTools().execute(new UiApiToolbox.Context("session-1", null, server, PolicyService.ToolPolicy.defaults(),
|
||||
write -> config.getRuntime().getStore().insert(new AssistantPendingAction().withUserId(AssistantTestFixture.ADMIN).withToolName(write.toolName())
|
||||
.withTableName(write.tableName()).withUiApiOperation(write.operation()).withParamsJson(JsonUtils.toJson(write.values()))
|
||||
.withSummary(write.summary()).withStatus(AssistantPendingAction.STATUS_PENDING).withExpiresAt(java.time.Instant.now().plusSeconds(600))).getId()),
|
||||
"call-1", toolName, in, hipaa));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDefinitionsAreOffered()
|
||||
{
|
||||
List<String> names = config.getRuntime().getContextBuilder().tools().stream().map(d -> d.getName()).toList();
|
||||
assertThat(names).contains(BulkLoadTools.LIST, BulkLoadTools.DESCRIBE, BulkLoadTools.ATTACH, BulkLoadTools.SUGGEST, BulkLoadTools.DRY_RUN);
|
||||
assertThat(config.getRuntime().getBulkLoadTools().definitions(PolicyService.ToolPolicy.defaults(), true))
|
||||
.extracting(d -> d.getName()).containsExactlyInAnyOrderElementsOf(BulkLoadTools.NAMES);
|
||||
assertThat(config.getRuntime().getBulkLoadTools().definitions(PolicyService.ToolPolicy.defaults(), false))
|
||||
.extracting(d -> d.getName()).doesNotContain(BulkLoadTools.SAVE, BulkLoadTools.RUN);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReadToolsCallTheUiApiWithTheProvidersRoute()
|
||||
{
|
||||
UiApiToolbox.Outcome list = tool(BulkLoadTools.LIST, Map.of("tableName", "blog_post"), false);
|
||||
assertThat(list.getResult().getOutcome()).isEqualTo(AiToolOutcome.OK);
|
||||
assertThat(list.getResult().getModelContent()).contains("Posts from the CMS export");
|
||||
assertThat(list.getOperation()).isEqualTo("POST /table/{tableName}/query");
|
||||
|
||||
UiApiToolbox.Outcome describe = tool(BulkLoadTools.DESCRIBE, Map.of("id", 3), false);
|
||||
assertThat(describe.getResult().getModelContent()).contains("fieldList");
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// the route header comes from the provider flag, whatever the input //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
tool(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "up_1", "X-QQQ-AI-Route", "hipaa", "route", "hipaa"), false);
|
||||
tool(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "up_1"), true);
|
||||
List<UiApiCall> attaches = server.calls("GET /bulkLoad/uploads/{uploadRef}/profile");
|
||||
assertThat(attaches.get(0).getHeaders()).containsEntry(BulkLoadTools.ROUTE_HEADER, "standard");
|
||||
assertThat(attaches.get(1).getHeaders()).containsEntry(BulkLoadTools.ROUTE_HEADER, "hipaa");
|
||||
assertThat(attaches.get(0).getSessionUUID()).isEqualTo("session-1");
|
||||
|
||||
UiApiToolbox.Outcome missing = tool(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "nope"), false);
|
||||
assertThat(missing.getResult().getOutcome()).isEqualTo(AiToolOutcome.NOT_FOUND);
|
||||
|
||||
UiApiToolbox.Outcome invalid = tool(BulkLoadTools.DRY_RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1"), false);
|
||||
assertThat(invalid.getResult().getOutcome()).isEqualTo(AiToolOutcome.INVALID_INPUT);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDataAccessAndAnOlderServer() throws QException
|
||||
{
|
||||
setAccess("page", DataAccessMode.NONE);
|
||||
UiApiToolbox.Outcome denied = tool(BulkLoadTools.SUGGEST, Map.of("tableName", "page", "uploadRef", "up_1"), false);
|
||||
assertThat(denied.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED);
|
||||
assertThat(server.getCalls()).isEmpty();
|
||||
|
||||
setAccess("media", DataAccessMode.READ);
|
||||
UiApiToolbox.Outcome readOnly = tool(BulkLoadTools.RUN, Map.of("tableName", "media", "uploadRef", "up_1", "profileId", 7), false);
|
||||
assertThat(readOnly.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED);
|
||||
|
||||
BulkLoadTools older = new BulkLoadTools(qInstance, config.getRuntime().getDataAccessService(), null);
|
||||
FakeBulkLoadServer oldApi = new FakeBulkLoadServer(false);
|
||||
QContext.init(qInstance, AssistantTestFixture.ownerSession(AssistantTestFixture.ADMIN, config));
|
||||
UiApiToolbox.Outcome outcome = older.execute(new UiApiToolbox.Context("s", null, oldApi, null, null), "c", BulkLoadTools.SUGGEST,
|
||||
Map.of("tableName", "blog_post", "uploadRef", "up_1"), false);
|
||||
assertThat(outcome.getResult().getOutcome()).isEqualTo(AiToolOutcome.ERROR);
|
||||
assertThat(outcome.getResult().getModelContent()).contains("does not offer");
|
||||
assertThat(oldApi.getCalls()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testJobsLaunchRunnerSaysWhenTheLaunchIsMissing() throws QException
|
||||
{
|
||||
BulkLoadRunner.RunHandle handle = new JobsLaunchBulkLoadRunner().start(new BulkLoadRunner.Request("s", null, server, "blog_post", "up_1", 7));
|
||||
assertThat(handle.status()).isEqualTo("NOT_AVAILABLE");
|
||||
assertThat(handle.summary()).contains("durable jobs launch");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testForLogDropsSamplesAndValues()
|
||||
{
|
||||
String logged = BulkLoadTools.forLog("<qqq_data source=\"x\">{\"columns\":[{\"header\":\"A\",\"sampleValues\":[\"" + FakeBulkLoadServer.SAMPLE_SENTINEL
|
||||
+ "\"]}],\"summaryLines\":[{\"bulletsOfText\":[\"Row 3 [x]\"]}],\"profile\":{\"fieldList\":[{\"valueMappings\":{\"secret\":1}}]}}</qqq_data>");
|
||||
assertThat(logged).doesNotContain(FakeBulkLoadServer.SAMPLE_SENTINEL).doesNotContain("Row 3 [x]").doesNotContain("secret").contains("header");
|
||||
assertThat(BulkLoadTools.forLog(null)).isNull();
|
||||
assertThat(BulkLoadTools.forLog("plain")).isEqualTo("plain");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Golden eval (owner request 2026-10-13): upload a CSV with messy headers;
|
||||
** the assistant proposes a profile, validates it, fixes one error, saves
|
||||
** it (shared with a group, after the user confirms) and runs the load
|
||||
** (after the user confirms). The scripted model stands in for the LLM;
|
||||
** its fix reads the dry run's error from the tool result.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void testGoldenMessyCsvToSavedProfileAndLoad() throws Exception
|
||||
{
|
||||
Map<String, Object> attachments = Map.of("attachments", List.of(Map.of("uploadRef", "up_1", "fileName", "posts.csv")));
|
||||
AtomicReference<String> suggested = new AtomicReference<>();
|
||||
|
||||
provider.thenToolUse(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "up_1"))
|
||||
.thenToolUse(BulkLoadTools.SUGGEST, Map.of("tableName", "blog_post", "uploadRef", "up_1"))
|
||||
.thenRespond(request ->
|
||||
{
|
||||
Map<String, Object> suggestion = firstResult(request);
|
||||
suggested.set(JsonUtils.toJson(suggestion.get("profile")));
|
||||
return (toolUse("dry1", BulkLoadTools.DRY_RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1", "profile", suggestion.get("profile"))));
|
||||
})
|
||||
.thenRespond(request ->
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// read the error, fix it: default the missing required field //
|
||||
////////////////////////////////////////////////////////////////
|
||||
Map<String, Object> dryRun = firstResult(request);
|
||||
assertThat(dryRun.get("errorCount")).isEqualTo(2);
|
||||
String missingField = String.valueOf(((Map<?, ?>) ((List<?>) dryRun.get("errors")).get(0)).get("fieldName"));
|
||||
Map<String, Object> profile = new JSONObject(suggested.get()).toMap();
|
||||
List<Map<String, Object>> fields = new ArrayList<>((List<Map<String, Object>>) profile.get("fieldList"));
|
||||
fields.add(new LinkedHashMap<>(Map.of("fieldName", missingField, "defaultValue", "DRAFT")));
|
||||
profile.put("fieldList", fields);
|
||||
return (toolUse("dry2", BulkLoadTools.DRY_RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1", "profile", profile, "mode", "FULL")));
|
||||
})
|
||||
.thenRespond(request ->
|
||||
{
|
||||
Map<String, Object> profile = new JSONObject(suggested.get()).toMap();
|
||||
((List<Map<String, Object>>) profile.get("fieldList")).add(new LinkedHashMap<>(Map.of("fieldName", "status", "defaultValue", "DRAFT")));
|
||||
return (toolUse("save", BulkLoadTools.SAVE, Map.of("tableName", "blog_post", "label", "CMS post export", "profile", profile,
|
||||
"shares", List.of(Map.of("audienceType", "group", "audienceId", "kof22-admins")))));
|
||||
})
|
||||
.thenText("The file maps cleanly once Status defaults to DRAFT (2 rows, no errors). Confirm to save the profile \"CMS post export\".");
|
||||
|
||||
TurnRunner.Outcome first = runner.run(AssistantTestFixture.ADMIN, null, "Load the attached posts.csv into blog posts and save the mapping for next time.", attachments);
|
||||
List<String> tools = first.toolCalls().stream().map(AssistantToolCall::getToolName).toList();
|
||||
assertThat(tools).containsExactly(BulkLoadTools.ATTACH, BulkLoadTools.SUGGEST, BulkLoadTools.DRY_RUN, BulkLoadTools.DRY_RUN, BulkLoadTools.SAVE);
|
||||
|
||||
List<UiApiCall> validates = server.calls("POST /bulkLoad/{tableName}/validate");
|
||||
assertThat(validates).hasSize(2);
|
||||
assertThat(validates).allSatisfy(c -> assertThat(c.getHeaders()).containsEntry(BulkLoadTools.ROUTE_HEADER, "standard"));
|
||||
assertThat(first.toolCalls().get(2).getResultSummary()).contains("2 with errors");
|
||||
assertThat(first.toolCalls().get(3).getResultSummary()).contains("0 with errors");
|
||||
assertThat(first.toolCalls()).allSatisfy(c -> assertThat(String.valueOf(c.getResultJson()) + c.getInputJson()).doesNotContain(FakeBulkLoadServer.SAMPLE_SENTINEL));
|
||||
assertThat(server.calls("POST /processes/{processName}/init")).as("nothing is saved before the user confirms").isEmpty();
|
||||
|
||||
Integer saveActionId = first.toolCalls().get(4).getPendingActionId();
|
||||
assertThat(saveActionId).isNotNull();
|
||||
QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN));
|
||||
Map<String, Object> saved = new AssistantApi(config.getRuntime(), QContext.getQSession()).confirmAction(saveActionId);
|
||||
QContext.clear();
|
||||
assertThat(saved.get("status")).isEqualTo(AssistantPendingAction.STATUS_EXECUTED);
|
||||
List<UiApiCall> processCalls = server.calls("POST /processes/{processName}/init");
|
||||
assertThat(processCalls).extracting(c -> c.getPathParams().get("processName")).containsExactly("storeSavedBulkLoadProfile", "insertSharedRecord");
|
||||
Map<String, Object> stored = FakeBulkLoadServer.formValues(processCalls.get(0));
|
||||
assertThat(stored).containsEntry("label", "CMS post export").containsEntry("tableName", "blog_post");
|
||||
assertThat(String.valueOf(stored.get("mappingJson"))).contains("DRAFT");
|
||||
assertThat(FakeBulkLoadServer.formValues(processCalls.get(1))).containsEntry("audienceType", "group").containsEntry("audienceId", "kof22-admins")
|
||||
.containsEntry("recordId", 7).containsEntry("scopeId", "READ_ONLY");
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// second turn: run the load with the saved profile; confirm //
|
||||
////////////////////////////////////////////////////////////////
|
||||
provider.thenToolUse(BulkLoadTools.RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1", "profileId", 7))
|
||||
.thenText("The load is queued as job 42; start it from its job record when you are ready.");
|
||||
TurnRunner.Outcome second = runner.run(AssistantTestFixture.ADMIN, first.conversationId(), "Now load it.", attachments);
|
||||
AssistantToolCall runCall = second.toolCalls().get(0);
|
||||
assertThat(runCall.getToolName()).isEqualTo(BulkLoadTools.RUN);
|
||||
assertThat(runs).as("nothing runs before the user confirms").isEmpty();
|
||||
|
||||
QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN));
|
||||
Map<String, Object> ran = new AssistantApi(config.getRuntime(), QContext.getQSession()).confirmAction(runCall.getPendingActionId());
|
||||
QContext.clear();
|
||||
assertThat(ran.get("outcome")).isEqualTo("OK");
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).savedBulkLoadProfileId()).isEqualTo(7);
|
||||
assertThat(runs.get(0).uploadRef()).isEqualTo("up_1");
|
||||
assertThat(runs.get(0).tableName()).isEqualTo("blog_post");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static ModelResponse toolUse(String id, String name, Map<String, Object> input)
|
||||
{
|
||||
return (new ModelResponse().withParts(List.of(ModelPart.toolUse("toolu_" + id, name, input))).withStopReason(ModelResponse.STOP_TOOL_USE));
|
||||
}
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.assistant.tools.bulkload;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiCall;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiGateway;
|
||||
import com.kingsrook.qbits.assistant.uiapi.UiApiResult;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A stand-in for the server's bulk-load UI API operations (core's
|
||||
** /bulkLoad/... and the profile processes), recording every call. Its
|
||||
** validate answers like core's dry run: the blog_post status field is
|
||||
** required, so a profile that neither maps nor defaults it has errors.
|
||||
*******************************************************************************/
|
||||
public class FakeBulkLoadServer implements UiApiGateway
|
||||
{
|
||||
public static final String SAMPLE_SENTINEL = "SAMPLE_SENTINEL_5b1e";
|
||||
|
||||
private static final Set<String> OPERATIONS = Set.of(
|
||||
"GET /bulkLoad/uploads/{uploadRef}/profile", "POST /bulkLoad/{tableName}/suggestMapping", "POST /bulkLoad/{tableName}/validate",
|
||||
"POST /table/{tableName}/query", "GET /table/{tableName}/{primaryKey}", "POST /processes/{processName}/init");
|
||||
|
||||
private final List<UiApiCall> calls = new ArrayList<>();
|
||||
private final boolean offersBulkLoad;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public FakeBulkLoadServer(boolean offersBulkLoad)
|
||||
{
|
||||
this.offersBulkLoad = offersBulkLoad;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isInvocable(String method, String pathTemplate)
|
||||
{
|
||||
String operation = method + " " + pathTemplate;
|
||||
return (OPERATIONS.contains(operation) && (offersBulkLoad || !pathTemplate.startsWith("/bulkLoad")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized UiApiResult invoke(UiApiCall call)
|
||||
{
|
||||
calls.add(call);
|
||||
String operation = call.operation();
|
||||
return (switch(operation)
|
||||
{
|
||||
case "GET /bulkLoad/uploads/{uploadRef}/profile" -> "up_1".equals(call.getPathParams().get("uploadRef"))
|
||||
? ok(new JSONObject().put("uploadRef", "up_1").put("fileName", "posts.csv").put("hasHeaderRow", true).put("rowsProfiled", 2)
|
||||
.put("columns", new JSONArray()
|
||||
.put(new JSONObject().put("columnIndex", 0).put("header", "Post Title").put("mappedFieldName", "title").put("samplesAllowed", true)
|
||||
.put("sampleValues", new JSONArray().put(SAMPLE_SENTINEL)))
|
||||
.put(new JSONObject().put("columnIndex", 1).put("header", "Teaser").put("samplesAllowed", false))))
|
||||
: new UiApiResult(404, "{\"error\":\"Upload not found.\"}");
|
||||
case "POST /bulkLoad/{tableName}/suggestMapping" -> ok(new JSONObject()
|
||||
.put("profile", new JSONObject().put("version", "v1").put("layout", "FLAT").put("hasHeaderRow", true)
|
||||
.put("fieldList", new JSONArray().put(new JSONObject().put("fieldName", "title").put("columnIndex", 0).put("headerName", "Post Title"))))
|
||||
.put("unmappedColumns", new JSONArray().put(new JSONObject().put("columnIndex", 1).put("header", "Teaser")))
|
||||
.put("unmappedRequiredFields", new JSONArray().put(new JSONObject().put("fieldName", "status").put("label", "Status"))));
|
||||
case "POST /bulkLoad/{tableName}/validate" -> validate(call);
|
||||
case "POST /table/{tableName}/query" -> ok(new JSONObject().put("records", new JSONArray()
|
||||
.put(new JSONObject().put("values", new JSONObject().put("id", 3).put("label", "Posts from the CMS export").put("userId", "admin-1")
|
||||
.put("mappingJson", "{\"version\":\"v1\",\"fieldList\":[{\"fieldName\":\"title\",\"columnIndex\":0}]}")))));
|
||||
case "GET /table/{tableName}/{primaryKey}" -> ok(new JSONObject().put("record", new JSONObject().put("values", new JSONObject().put("id", 3)
|
||||
.put("label", "Posts from the CMS export").put("tableName", "blog_post").put("userId", "admin-1")
|
||||
.put("mappingJson", "{\"version\":\"v1\",\"fieldList\":[{\"fieldName\":\"title\",\"columnIndex\":0}]}"))));
|
||||
case "POST /processes/{processName}/init" -> "storeSavedBulkLoadProfile".equals(call.getPathParams().get("processName"))
|
||||
? ok(new JSONObject().put("type", "COMPLETE").put("values", new JSONObject().put("savedBulkLoadProfileList",
|
||||
new JSONArray().put(new JSONObject().put("values", new JSONObject().put("id", 7).put("label", "x"))))))
|
||||
: ok(new JSONObject().put("type", "COMPLETE").put("values", new JSONObject()));
|
||||
default -> new UiApiResult(404, "{\"error\":\"no such operation\"}");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Errors unless status is mapped or defaulted.
|
||||
*******************************************************************************/
|
||||
private UiApiResult validate(UiApiCall call)
|
||||
{
|
||||
JSONObject body = new JSONObject(JsonUtils.toJson(call.getJsonBody()));
|
||||
JSONArray fieldList = body.getJSONObject("profile").getJSONArray("fieldList");
|
||||
boolean hasStatus = false;
|
||||
for(int i = 0; i < fieldList.length(); i++)
|
||||
{
|
||||
JSONObject field = fieldList.getJSONObject(i);
|
||||
if("status".equals(field.optString("fieldName")) && (field.has("columnIndex") || !field.isNull("defaultValue") && field.has("defaultValue")))
|
||||
{
|
||||
hasStatus = true;
|
||||
}
|
||||
}
|
||||
JSONObject result = new JSONObject().put("mode", body.optString("mode", "SAMPLE")).put("processName", "blog_post.bulkInsert").put("recordCount", 2);
|
||||
if(hasStatus)
|
||||
{
|
||||
return (ok(result.put("okCount", 2).put("errorCount", 0).put("errors", new JSONArray())));
|
||||
}
|
||||
return (ok(result.put("okCount", 0).put("errorCount", 2).put("errors", new JSONArray()
|
||||
.put(new JSONObject().put("rowNumbers", "Rows 2-3").put("fieldName", "status").put("errorCode", "REQUIRED").put("message", "Missing value in required field: Status")))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static UiApiResult ok(JSONObject json)
|
||||
{
|
||||
return (new UiApiResult(200, json.toString()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The calls, in order.
|
||||
*******************************************************************************/
|
||||
public synchronized List<UiApiCall> getCalls()
|
||||
{
|
||||
return (new ArrayList<>(calls));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Calls of one operation.
|
||||
*******************************************************************************/
|
||||
public synchronized List<UiApiCall> calls(String operation)
|
||||
{
|
||||
return (calls.stream().filter(c -> c.operation().equals(operation)).toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A form value of a process call, as JSON.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> formValues(UiApiCall call)
|
||||
{
|
||||
return (new JSONObject(call.getFormParams().get("values")).toMap());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user