diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitConfig.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitConfig.java index a2af573..75cce19 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitConfig.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitConfig.java @@ -133,6 +133,7 @@ public class AssistantQBitConfig implements QBitConfig ////////////////////////////////////////////////////////////////////////// private String uiApiBaseUrl; private UiApiGateway uiApiGateway; + private com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner bulkLoadRunner; private Integer pendingActionTtlSeconds = 900; private List customRedactionPatterns = new ArrayList<>(); private Integer killSwitchPollMillis = 2000; @@ -1202,6 +1203,38 @@ public class AssistantQBitConfig implements QBitConfig + /******************************************************************************* + ** Getter for bulkLoadRunner (null: the durable jobs launch, + ** JobsLaunchBulkLoadRunner) - how bulk_load_run starts a load. + *******************************************************************************/ + public com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner getBulkLoadRunner() + { + return (this.bulkLoadRunner); + } + + + + /******************************************************************************* + ** Setter for bulkLoadRunner + *******************************************************************************/ + public void setBulkLoadRunner(com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner bulkLoadRunner) + { + this.bulkLoadRunner = bulkLoadRunner; + } + + + + /******************************************************************************* + ** Fluent setter for bulkLoadRunner + *******************************************************************************/ + public AssistantQBitConfig withBulkLoadRunner(com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadRunner bulkLoadRunner) + { + this.bulkLoadRunner = bulkLoadRunner; + return (this); + } + + + /******************************************************************************* ** Getter for uiApiGateway (null: the loopback stand-in) *******************************************************************************/ diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantRuntime.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantRuntime.java index c6c6fc2..9800a8a 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantRuntime.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantRuntime.java @@ -37,6 +37,7 @@ import com.kingsrook.qbits.assistant.rag.InMemoryVectorStore; import com.kingsrook.qbits.assistant.rag.RagService; import com.kingsrook.qbits.assistant.rag.VectorStore; import com.kingsrook.qbits.assistant.tools.KnowledgeTool; +import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools; import com.kingsrook.qbits.assistant.tools.UiApiToolbox; import com.kingsrook.qbits.assistant.uiapi.LoopbackUiApiGateway; import com.kingsrook.qbits.assistant.uiapi.UiApiGateway; @@ -58,6 +59,7 @@ public class AssistantRuntime { private final RagService ragService; private final KnowledgeTool knowledgeTool; + private final BulkLoadTools bulkLoadTools; private final QInstance qInstance; private final AssistantQBitConfig config; @@ -116,7 +118,8 @@ public class AssistantRuntime VectorStore vectors = config.getVectorStore() == null ? new InMemoryVectorStore() : config.getVectorStore(); this.ragService = new RagService(this, vectors); this.knowledgeTool = new KnowledgeTool(this, ragService, vectors); - this.contextBuilder = new ContextBuilder(config, kernel, toolbox).withKnowledgeTool(knowledgeTool); + this.bulkLoadTools = new BulkLoadTools(qInstance, dataAccessService, config.getBulkLoadRunner()); + this.contextBuilder = new ContextBuilder(config, kernel, toolbox).withKnowledgeTool(knowledgeTool).withBulkLoadTools(bulkLoadTools); } @@ -447,4 +450,14 @@ public class AssistantRuntime return (knowledgeTool); } + + + /******************************************************************************* + ** The bulk-load tools (saved profiles, attached files, dry runs, loads). + *******************************************************************************/ + public BulkLoadTools getBulkLoadTools() + { + return (bulkLoadTools); + } + } diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/api/AssistantApi.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/api/AssistantApi.java index 86f76fe..29cf24b 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/api/AssistantApi.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/api/AssistantApi.java @@ -46,6 +46,7 @@ import com.kingsrook.qbits.assistant.model.AssistantTurn; import com.kingsrook.qbits.assistant.model.AssistantTurnEvent; import com.kingsrook.qbits.assistant.persistence.AssistantStore; import com.kingsrook.qbits.assistant.tools.UiApiToolbox; +import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools; import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome; import com.kingsrook.qqq.backend.core.actions.ai.AiToolResult; import com.kingsrook.qqq.backend.core.actions.ai.output.AiRedactor; @@ -598,7 +599,9 @@ public class AssistantApi } PolicyService.EffectivePolicy policy = runtime.getPolicyService().current(true); UiApiToolbox.Context context = new UiApiToolbox.Context(session.getUuid(), session.getValue(QSession.VALUE_KEY_USER_TIMEZONE), runtime.getUiApiGateway(), policy.tools(), null); - UiApiToolbox.Outcome outcome = runtime.getToolbox().executeConfirmedWrite(context, action.getToolCallId(), action.getToolName(), action.getTableName(), action.getRecordId(), values); + UiApiToolbox.Outcome outcome = BulkLoadTools.WRITE_TOOLS.contains(action.getToolName()) + ? runtime.getBulkLoadTools().executeConfirmed(context, action.getToolCallId(), action.getToolName(), action.getTableName(), values) + : runtime.getToolbox().executeConfirmedWrite(context, action.getToolCallId(), action.getToolName(), action.getTableName(), action.getRecordId(), values); AiToolResult result = outcome.getResult(); boolean ok = result.getOutcome() == AiToolOutcome.OK; diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ContextBuilder.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ContextBuilder.java index d785f48..446ec56 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ContextBuilder.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ContextBuilder.java @@ -36,6 +36,7 @@ import com.kingsrook.qbits.assistant.provider.ModelToolDefinition; import com.kingsrook.qbits.assistant.provider.SystemBlock; import com.kingsrook.qbits.assistant.tools.KnowledgeTool; import com.kingsrook.qbits.assistant.tools.PresentResultsTool; +import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools; import com.kingsrook.qbits.assistant.tools.UiApiToolbox; import com.kingsrook.qqq.backend.core.actions.ai.AiToolDefinition; import com.kingsrook.qqq.backend.core.actions.ai.AiToolKernel; @@ -66,12 +67,14 @@ public class ContextBuilder public static final int MAX_CONTEXT_CHARS = 512; public static final int MAX_SELECTED_IDS = 100; + public static final int MAX_ATTACHMENTS = 5; private final AssistantQBitConfig config; private final AiToolKernel kernel; private final UiApiToolbox toolbox; private volatile String appDigest; private KnowledgeTool knowledgeTool; + private BulkLoadTools bulkLoadTools; @@ -125,6 +128,13 @@ public class ContextBuilder byName.put(KnowledgeTool.NAME, knowledgeTool.definition()); } byName.put(PresentResultsTool.NAME, PresentResultsTool.definition(isStrict(PresentResultsTool.NAME, true))); + if(bulkLoadTools != null && bulkLoadTools.offered()) + { + for(ModelToolDefinition definition : bulkLoadTools.definitions(policy, toolbox.anyWritableTable())) + { + byName.put(definition.getName(), definition); + } + } return (new ArrayList<>(byName.values())); } @@ -352,6 +362,32 @@ public class ContextBuilder } } + //////////////////////////////////////////////////////////////////// + // files the user attached in the panel (already uploaded through // + // the UI API as the user): only their opaque refs and names // + //////////////////////////////////////////////////////////////////// + if(raw.get("attachments") instanceof List attachments) + { + List> kept = new ArrayList<>(); + for(Object attachment : attachments) + { + if(attachment instanceof Map a && scalar(a.get("uploadRef")) != null && kept.size() < MAX_ATTACHMENTS) + { + Map one = new LinkedHashMap<>(); + one.put("uploadRef", scalar(a.get("uploadRef"))); + if(scalar(a.get("fileName")) != null) + { + one.put("fileName", scalar(a.get("fileName"))); + } + kept.add(one); + } + } + if(!kept.isEmpty()) + { + clean.put("attachments", kept); + } + } + if(JsonUtils.toJson(clean).length() > 4000) { clean.remove("filter"); @@ -390,4 +426,15 @@ public class ContextBuilder return (this); } + + + /******************************************************************************* + ** Fluent setter for the bulk-load tools. + *******************************************************************************/ + public ContextBuilder withBulkLoadTools(BulkLoadTools bulkLoadTools) + { + this.bulkLoadTools = bulkLoadTools; + return (this); + } + } diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java index 8acdbf5..eea1f75 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java @@ -37,7 +37,14 @@ public class ToolLabels Map.entry("get_record_history", new String[] { "Reading the change history", "Read the change history" }), Map.entry("render_widget", new String[] { "Rendering a widget", "Rendered a widget" }), Map.entry("search_help", new String[] { "Searching help", "Searched help" }), - Map.entry("navigate", new String[] { "Opening a screen", "Opened a screen" }) + Map.entry("navigate", new String[] { "Opening a screen", "Opened a screen" }), + Map.entry("bulk_profiles_list", new String[] { "Listing saved bulk load profiles", "Listed saved bulk load profiles" }), + Map.entry("bulk_profile_describe", new String[] { "Reading a bulk load profile", "Read a bulk load profile" }), + Map.entry("bulk_file_attach", new String[] { "Reading the attached file", "Read the attached file" }), + Map.entry("bulk_mapping_suggest", new String[] { "Suggesting a column mapping", "Suggested a column mapping" }), + Map.entry("bulk_load_dry_run", new String[] { "Checking the mapping (dry run)", "Checked the mapping (dry run)" }), + Map.entry("bulk_profile_save", new String[] { "Preparing to save the profile", "Prepared to save the profile" }), + Map.entry("bulk_load_run", new String[] { "Preparing the load", "Prepared the load" }) ); diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java index 7729f7c..6cf6f8b 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java @@ -53,6 +53,7 @@ import com.kingsrook.qbits.assistant.provider.ModelUsage; import com.kingsrook.qbits.assistant.provider.StreamHandle; import com.kingsrook.qbits.assistant.tools.KnowledgeTool; import com.kingsrook.qbits.assistant.tools.PresentResultsTool; +import com.kingsrook.qbits.assistant.tools.bulkload.BulkLoadTools; import com.kingsrook.qbits.assistant.tools.UiApiToolbox; import com.kingsrook.qqq.backend.core.actions.ai.AiRecordRef; import com.kingsrook.qqq.backend.core.actions.ai.AiToolCall; @@ -716,6 +717,11 @@ public class TurnEngine implements Runnable { results[index] = runtime.getKnowledgeTool().execute(toolContext(), call.getId(), call.getInput(), classification.getClassification(), conversation.getId(), turn.getId()); } + else if(BulkLoadTools.NAMES.contains(call.getName())) + { + outcomes[index] = runtime.getBulkLoadTools().execute(toolContext(), call.getId(), call.getName(), call.getInput(), hipaaRoute()); + results[index] = outcomes[index].getResult(); + } else { outcomes[index] = runtime.getToolbox().execute(toolContext(), call.getId(), call.getName(), call.getInput()); @@ -856,7 +862,7 @@ public class TurnEngine implements Runnable .withRowCount(result.getRowCount()) .withDurationMs(result.getDurationMs()) .withResultSummary(summary) - .withResultJson(truncate(result.getModelContent(), MAX_RESULT_JSON_CHARS)) + .withResultJson(truncate(BulkLoadTools.NAMES.contains(use.getToolName()) ? BulkLoadTools.forLog(result.getModelContent()) : result.getModelContent(), MAX_RESULT_JSON_CHARS)) .withRedactionsJson(result.getRedactions() == null || result.getRedactions().isEmpty() ? null : JsonUtils.toJson(result.getRedactions())) .withUiApiOperation(extras == null ? ("search_knowledge".equals(use.getToolName()) ? "GET /table/{tableName}/{primaryKey}" : null) : extras.getOperation()) .withHttpStatus(extras == null ? null : extras.getHttpStatus()) @@ -876,6 +882,21 @@ public class TurnEngine implements Runnable ** API), the gateway, the policy's tool settings, and the pending-action ** store for writes that need confirmation. *******************************************************************************/ + private boolean hipaaRoute() + { + /////////////////////////////////////////////////////////////////////// + // the X-QQQ-AI-Route the bulk-load tools send: the turn's provider's // + // HIPAA eligibility, never anything the model says // + /////////////////////////////////////////////////////////////////////// + return (active != null && active.row() != null && Boolean.TRUE.equals(active.row().getHipaaEligible())); + } + + + + /******************************************************************************* + ** The UI API tool context: the user's session, the gateway, the policy's + ** tools and the pending-action hook. + *******************************************************************************/ private UiApiToolbox.Context toolContext() { return (new UiApiToolbox.Context(session.getUuid(), userTimezone(), runtime.getUiApiGateway(), policy.tools(), this::proposeWrite)); diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java index 6e1e6aa..905f9a2 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java @@ -203,6 +203,21 @@ public class UiApiToolbox + /******************************************************************************* + ** An outcome built by another tool family (e.g. the bulk-load tools), so + ** TurnEngine and the confirm route log it the same way. + *******************************************************************************/ + public static Outcome of(AiToolResult result, String operation, Integer httpStatus, Integer pendingActionId) + { + Outcome outcome = new Outcome(result); + outcome.operation = operation; + outcome.httpStatus = httpStatus; + outcome.pendingActionId = pendingActionId; + return (outcome); + } + + + /******************************************************************************* ** Getter for result *******************************************************************************/ diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadRunner.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadRunner.java new file mode 100644 index 0000000..04868d6 --- /dev/null +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadRunner.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools.bulkload; + + +import com.kingsrook.qbits.assistant.uiapi.UiApiGateway; +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** How bulk_load_run starts a load (design spec #1171 section 3.4): one seam, + ** so the run can move from today's path to durable jobs without the tool + ** changing. Every implementation goes through the /qqq/v1 UI API as the user. + *******************************************************************************/ +public interface BulkLoadRunner +{ + /******************************************************************************* + ** A load to start: the target table, the upload, and the saved profile. + *******************************************************************************/ + record Request(String sessionUUID, String userTimezone, UiApiGateway gateway, String tableName, String uploadRef, Integer savedBulkLoadProfileId) + { + } + + + + /******************************************************************************* + ** What started: status uses the durable job status names (jobs_job.status: + ** AWAITING_CONFIRMATION, QUEUED, RUNNING, COMPLETED, + ** COMPLETED_WITH_ERRORS, FAILED, ...); jobId when it runs as a job, + ** processUUID when it ran synchronously; a short summary; the HTTP status. + *******************************************************************************/ + record RunHandle(String status, Long jobId, String processUUID, String summary, String operation, Integer httpStatus) + { + } + + + + /******************************************************************************* + ** Start the load. Throws when it cannot start at all (the handle's status + ** reports a load that started and failed). + *******************************************************************************/ + RunHandle start(Request request) throws QException; +} diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadTools.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadTools.java new file mode 100644 index 0000000..ee703d1 --- /dev/null +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadTools.java @@ -0,0 +1,883 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools.bulkload; + + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.assistant.access.DataAccessService; +import com.kingsrook.qbits.assistant.engine.PolicyService; +import com.kingsrook.qbits.assistant.provider.ModelToolDefinition; +import com.kingsrook.qbits.assistant.tools.UiApiToolbox; +import com.kingsrook.qbits.assistant.uiapi.UiApiCall; +import com.kingsrook.qbits.assistant.uiapi.UiApiResult; +import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome; +import com.kingsrook.qqq.backend.core.actions.ai.AiToolResult; +import com.kingsrook.qqq.backend.core.actions.ai.output.AiDataEnvelope; +import com.kingsrook.qqq.backend.core.actions.ai.tools.AiJsonSchema; +import com.kingsrook.qqq.backend.core.actions.ai.tools.AiToolException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** The assistant's bulk-load tools (design spec #1171 section 3.4): use and + ** create saved bulk load profiles from a file the user attached in the chat. + ** Every call goes through the /qqq/v1 UI API with the user's own session, so + ** table and process permissions, record locks and the PHI rule are enforced + ** by the server, never by the model: + ** - bulk_profiles_list / bulk_profile_describe: the user's (and shared) + ** saved profiles, through the savedBulkLoadProfile table (record locks); + ** - bulk_file_attach: an upload the panel already made (by its ref), with + ** its column profiles (samples only where the server allows); + ** - bulk_mapping_suggest: core's suggester, then the model refines it; + ** - bulk_load_dry_run: the table's real bulk-insert validation, no store; + ** - bulk_profile_save (write): storeSavedBulkLoadProfile, and shares through + ** insertSharedRecord, after the user confirms; + ** - bulk_load_run (write): through {@link BulkLoadRunner}, after the user + ** confirms (and, as a durable job, again on its job record). + ** + ** The AI route header (X-QQQ-AI-Route: hipaa|standard) is set here from the + ** turn's provider, never from the model's input. Tool-call logs keep + ** profiles and counts, never sample values (see {@link #forLog}). + ** + ** STAND-IN(core=#1179, removeBy=RC.8): the profile, suggestMapping and + ** validate operations ship in a QQQ RC after 4.1.0-RC.5; until then the + ** tools report that the server does not offer them. + *******************************************************************************/ +public class BulkLoadTools +{ + public static final String LIST = "bulk_profiles_list"; + public static final String DESCRIBE = "bulk_profile_describe"; + public static final String ATTACH = "bulk_file_attach"; + public static final String SUGGEST = "bulk_mapping_suggest"; + public static final String DRY_RUN = "bulk_load_dry_run"; + public static final String SAVE = "bulk_profile_save"; + public static final String RUN = "bulk_load_run"; + + public static final Set NAMES = Set.of(LIST, DESCRIBE, ATTACH, SUGGEST, DRY_RUN, SAVE, RUN); + public static final Set WRITE_TOOLS = Set.of(SAVE, RUN); + + public static final String PROFILE_TABLE = "savedBulkLoadProfile"; + public static final String ROUTE_HEADER = "X-QQQ-AI-Route"; + + private static final Set LOG_STRIPPED_KEYS = Set.of("sampleValues", "bulletsOfText", "valueMappings"); + + private final QInstance qInstance; + private final DataAccessService dataAccess; + private final BulkLoadRunner runner; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BulkLoadTools(QInstance qInstance, DataAccessService dataAccess, BulkLoadRunner runner) + { + this.qInstance = qInstance; + this.dataAccess = dataAccess; + this.runner = runner == null ? new JobsLaunchBulkLoadRunner() : runner; + } + + + + /******************************************************************************* + ** Offered when the instance has saved bulk load profiles. + *******************************************************************************/ + public boolean offered() + { + return (qInstance != null && qInstance.getTable(PROFILE_TABLE) != null); + } + + + + /******************************************************************************* + ** The definitions the policy allows (writes only when anyWritable). + *******************************************************************************/ + public List definitions(PolicyService.ToolPolicy policy, boolean anyWritable) + { + List definitions = new ArrayList<>(); + if(!offered()) + { + return (definitions); + } + + Map table = AiJsonSchema.string("The table to load into (from list_tables)."); + Map upload = AiJsonSchema.string("The upload ref of the file the user attached (from the conversation's attachments)."); + Map profile = new LinkedHashMap<>(Map.of("type", "object", "description", """ + A v1 bulk load profile: {"version":"v1","layout":"FLAT","hasHeaderRow":true,"fieldList":[{"fieldName":"...","columnIndex":0, \ + "headerName":"...","defaultValue":null,"doValueMapping":false,"valueMappings":{"file value":"field value"}}]}.""")); + + add(definitions, policy, LIST, "List the saved bulk load profiles (the user's own and those shared with them) for a table.", + AiJsonSchema.object(AiJsonSchema.properties("tableName", table), List.of("tableName"))); + add(definitions, policy, DESCRIBE, "Read one saved bulk load profile: its table, owner and column mapping.", + AiJsonSchema.object(AiJsonSchema.properties("id", AiJsonSchema.integer("The profile id (from bulk_profiles_list).")), List.of("id"))); + add(definitions, policy, ATTACH, """ + Look at a file the user attached for a bulk load: whether row 1 is a header, and each column's type, blanks, distinct count, \ + pattern and (only where the server allows) a few sample values.""", + AiJsonSchema.object(AiJsonSchema.properties("uploadRef", upload, "tableName", table), List.of("uploadRef", "tableName"))); + add(definitions, policy, SUGGEST, """ + Suggest a bulk load profile for an attached file: the server's header matching, with the columns it left unmapped, required \ + fields it did not fill, and fields whose values need mapping. Refine the profile yourself from the headers and column \ + profiles, then check it with bulk_load_dry_run.""", + AiJsonSchema.object(AiJsonSchema.properties("uploadRef", upload, "tableName", table), List.of("uploadRef", "tableName"))); + add(definitions, policy, DRY_RUN, """ + Dry-run a bulk load profile over an attached file through the table's real bulk insert validation. Nothing is stored. \ + Returns counts, summary lines and error groups with row numbers. SAMPLE checks the first 100 rows; FULL the whole file. \ + Fix the profile and run it again until there are no errors (or explain the rest to the user).""", + AiJsonSchema.object(AiJsonSchema.properties("uploadRef", upload, "tableName", table, "profile", profile, + "mode", AiJsonSchema.enumOf(List.of("SAMPLE", "FULL"), "SAMPLE (default) or FULL.")), List.of("uploadRef", "tableName", "profile"))); + + if(anyWritable) + { + Map share = AiJsonSchema.object(AiJsonSchema.properties( + "audienceType", AiJsonSchema.enumOf(List.of("user", "group"), "Share with a user (email) or a group."), + "audienceId", AiJsonSchema.string("The user's email or the group's name."), + "scope", AiJsonSchema.enumOf(List.of("READ_ONLY", "READ_WRITE"), "The share's scope (default READ_ONLY).")), List.of("audienceType", "audienceId")); + add(definitions, policy, SAVE, """ + Save a bulk load profile (after a clean dry run) under a name, optionally sharing it. The user confirms before it is saved; \ + it is saved as the user, and shares are limited to what the user may share.""", + AiJsonSchema.object(AiJsonSchema.properties("tableName", table, "label", AiJsonSchema.string("The profile's name."), "profile", profile, + "shares", AiJsonSchema.array(share, "Optional shares.")), List.of("tableName", "label", "profile"))); + add(definitions, policy, RUN, """ + Load an attached file into a table with a saved profile. The user confirms first; a large load runs as a background job that \ + the user starts from its job record.""", + AiJsonSchema.object(AiJsonSchema.properties("tableName", table, "uploadRef", upload, "profileId", AiJsonSchema.integer("The saved profile id.")), + List.of("tableName", "uploadRef", "profileId"))); + } + return (definitions); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void add(List definitions, PolicyService.ToolPolicy policy, String name, String description, Map schema) + { + if(policy == null || policy.allows(name)) + { + definitions.add(new ModelToolDefinition().withName(name).withDescription(description).withInputSchema(schema).withStrict(false)); + } + } + + + + /******************************************************************************* + ** Run a tool. hipaaRoute is the turn's provider's HIPAA eligibility (sent + ** as X-QQQ-AI-Route; the model cannot set it). + *******************************************************************************/ + public UiApiToolbox.Outcome execute(UiApiToolbox.Context context, String toolCallId, String toolName, Map input, boolean hipaaRoute) + { + AiToolResult result = new AiToolResult().withToolCallId(toolCallId).withToolName(toolName); + Map in = input == null ? Map.of() : input; + Call call = new Call(context, hipaaRoute); + Integer actionId = null; + long started = System.nanoTime(); + try + { + if(context.policy() != null && !context.policy().allows(toolName)) + { + deny(result, "policy:tool", "The administrator has not allowed the " + toolName + " tool."); + return (UiApiToolbox.Outcome.of(result, null, null, null)); + } + + String tableName = in.get("tableName") instanceof String t ? t.strip() : null; + if(tableName != null) + { + result.setTableName(tableName); + if(!dataAccess.forTable(tableName).mode().allowsRead()) + { + deny(result, "dataAccess:" + dataAccess.forTable(tableName).mode(), "The assistant may not use the " + tableName + " table."); + return (UiApiToolbox.Outcome.of(result, null, null, null)); + } + } + + switch(toolName) + { + case LIST -> list(call, result, required(in, "tableName")); + case DESCRIBE -> describe(call, result, requiredInteger(in, "id")); + case ATTACH -> attach(call, result, required(in, "tableName"), required(in, "uploadRef")); + case SUGGEST -> suggest(call, result, required(in, "tableName"), required(in, "uploadRef")); + case DRY_RUN -> dryRun(call, result, required(in, "tableName"), required(in, "uploadRef"), requiredMap(in, "profile"), in.get("mode") instanceof String m ? m : "SAMPLE"); + case SAVE, RUN -> actionId = propose(context, result, toolName, required(in, "tableName"), in); + default -> throw (AiToolException.invalidInput("Unknown tool \"" + toolName + "\".")); + } + } + catch(AiToolException e) + { + result.withOutcome(e.getOutcome() == null ? AiToolOutcome.INVALID_INPUT : e.getOutcome()).withSummary(shortText(e.getMessage())).withModelContent(e.getMessage()); + } + catch(QException | RuntimeException e) + { + result.withOutcome(AiToolOutcome.ERROR).withSummary("Failed").withModelContent("The tool failed: " + shortText(e.getMessage())); + } + finally + { + result.setDurationMs((System.nanoTime() - started) / 1_000_000); + } + return (UiApiToolbox.Outcome.of(result, call.operation, call.httpStatus, actionId)); + } + + + + /******************************************************************************* + ** Run a confirmed write (from a pending action) with the confirming user's + ** session. + *******************************************************************************/ + public UiApiToolbox.Outcome executeConfirmed(UiApiToolbox.Context context, String toolCallId, String toolName, String tableName, Map values) + { + AiToolResult result = new AiToolResult().withToolCallId(toolCallId).withToolName(toolName).withTableName(tableName); + Call call = new Call(context, false); + long started = System.nanoTime(); + try + { + if(!WRITE_TOOLS.contains(toolName)) + { + throw (AiToolException.invalidInput("Not a write tool: " + toolName)); + } + DataAccessService.Decision decision = dataAccess.forTable(SAVE.equals(toolName) ? PROFILE_TABLE : tableName); + if(!decision.mode().allowsWrite()) + { + deny(result, "dataAccess:" + decision.mode(), "The assistant may not write here (data access " + decision.mode() + ", " + decision.source() + ")."); + return (UiApiToolbox.Outcome.of(result, null, null, null)); + } + if(SAVE.equals(toolName)) + { + save(call, result, tableName, values); + } + else + { + run(call, result, context, tableName, values); + } + } + catch(AiToolException e) + { + result.withOutcome(e.getOutcome() == null ? AiToolOutcome.INVALID_INPUT : e.getOutcome()).withSummary(shortText(e.getMessage())).withModelContent(e.getMessage()); + } + catch(QException | RuntimeException e) + { + result.withOutcome(AiToolOutcome.ERROR).withSummary("Failed").withModelContent("The write failed: " + shortText(e.getMessage())); + } + finally + { + result.setDurationMs((System.nanoTime() - started) / 1_000_000); + } + return (UiApiToolbox.Outcome.of(result, call.operation, call.httpStatus, null)); + } + + + + //////////////////////////////////////////////////////////////////////////// + // read tools // + //////////////////////////////////////////////////////////////////////////// + + + + /******************************************************************************* + ** POST /table/savedBulkLoadProfile/query: the profiles the record locks let + ** the user see, for one table. + *******************************************************************************/ + private void list(Call call, AiToolResult result, String tableName) throws QException + { + Map filter = Map.of( + "criteria", List.of(Map.of("fieldName", "tableName", "operator", "EQUALS", "values", List.of(tableName))), + "orderBys", List.of(Map.of("fieldName", "label", "isAscending", true))); + UiApiResult response = call.invoke(new UiApiCall("POST", "/table/{tableName}/query").withPathParam("tableName", PROFILE_TABLE) + .withJsonBody(Map.of("filter", filter))); + if(response == null || !response.isSuccess()) + { + fail(result, call, response); + return; + } + + List> rows = new ArrayList<>(); + for(Object record : list(response.json().get("records"))) + { + Map values = map(map(record).get("values")); + Map row = new LinkedHashMap<>(); + row.put("id", values.get("id")); + row.put("label", values.get("label")); + row.put("owner", values.get("userId")); + row.put("fieldCount", fieldList(values.get("mappingJson")).size()); + rows.add(row); + } + result.withOutcome(AiToolOutcome.OK).withRowCount(rows.size()).withSummary("Found " + rows.size() + " saved profile" + (rows.size() == 1 ? "" : "s")) + .withPermissionCheck(call.permissionCheck()) + .withModelContent(AiDataEnvelope.wrap(LIST, PROFILE_TABLE, JsonUtils.toJson(Map.of("tableName", tableName, "profiles", rows)))); + } + + + + /******************************************************************************* + ** GET /table/savedBulkLoadProfile/{id}: one profile (when the locks allow). + *******************************************************************************/ + private void describe(Call call, AiToolResult result, Integer id) throws QException + { + UiApiResult response = call.invoke(new UiApiCall("GET", "/table/{tableName}/{primaryKey}").withPathParam("tableName", PROFILE_TABLE).withPathParam("primaryKey", id)); + if(response == null || !response.isSuccess()) + { + fail(result, call, response); + return; + } + Map values = map(map(response.json().get("record")).get("values")); + Map out = new LinkedHashMap<>(); + out.put("id", values.get("id")); + out.put("label", values.get("label")); + out.put("tableName", values.get("tableName")); + out.put("owner", values.get("userId")); + out.put("profile", values.get("mappingJson") == null ? null : new JSONObject(String.valueOf(values.get("mappingJson"))).toMap()); + result.setTableName(String.valueOf(values.get("tableName"))); + result.withOutcome(AiToolOutcome.OK).withRowCount(1).withSummary("Read profile " + values.get("label")) + .withPermissionCheck(call.permissionCheck()) + .withModelContent(AiDataEnvelope.wrap(DESCRIBE, PROFILE_TABLE, JsonUtils.toJson(out))); + } + + + + /******************************************************************************* + ** GET /bulkLoad/uploads/{uploadRef}/profile. + *******************************************************************************/ + private void attach(Call call, AiToolResult result, String tableName, String uploadRef) throws QException + { + UiApiResult response = call.invoke(new UiApiCall("GET", "/bulkLoad/uploads/{uploadRef}/profile").withPathParam("uploadRef", uploadRef) + .withQueryParam("tableName", tableName).withQueryParam("sampleCount", 5)); + if(response == null || !response.isSuccess()) + { + fail(result, call, response); + return; + } + Map json = response.json(); + int columns = list(json.get("columns")).size(); + result.withOutcome(AiToolOutcome.OK).withRowCount(columns).withSummary("Read " + json.get("fileName") + ": " + columns + " columns") + .withPermissionCheck(call.permissionCheck()) + .withModelContent(AiDataEnvelope.wrap(ATTACH, tableName, response.body())); + } + + + + /******************************************************************************* + ** POST /bulkLoad/{tableName}/suggestMapping. + *******************************************************************************/ + private void suggest(Call call, AiToolResult result, String tableName, String uploadRef) throws QException + { + UiApiResult response = call.invoke(new UiApiCall("POST", "/bulkLoad/{tableName}/suggestMapping").withPathParam("tableName", tableName) + .withJsonBody(Map.of("uploadRef", uploadRef))); + if(response == null || !response.isSuccess()) + { + fail(result, call, response); + return; + } + Map json = response.json(); + int mapped = list(map(json.get("profile")).get("fieldList")).size(); + result.withOutcome(AiToolOutcome.OK).withRowCount(mapped).withSummary("Suggested a mapping of " + mapped + " field" + (mapped == 1 ? "" : "s")) + .withPermissionCheck(call.permissionCheck()) + .withModelContent(AiDataEnvelope.wrap(SUGGEST, tableName, response.body())); + } + + + + /******************************************************************************* + ** POST /bulkLoad/{tableName}/validate (nothing is stored). + *******************************************************************************/ + private void dryRun(Call call, AiToolResult result, String tableName, String uploadRef, Map profile, String mode) throws QException + { + UiApiResult response = call.invoke(new UiApiCall("POST", "/bulkLoad/{tableName}/validate").withPathParam("tableName", tableName) + .withJsonBody(Map.of("uploadRef", uploadRef, "profile", profile, "mode", "FULL".equalsIgnoreCase(mode) ? "FULL" : "SAMPLE")) + .withTimeout(Duration.ofSeconds(120))); + if(response == null || !response.isSuccess()) + { + fail(result, call, response); + return; + } + Map json = response.json(); + Integer ok = ValueUtils.getValueAsInteger(json.get("okCount")); + Integer errors = ValueUtils.getValueAsInteger(json.get("errorCount")); + result.withOutcome(AiToolOutcome.OK).withRowCount(ok) + .withSummary("Dry run (" + json.get("mode") + "): " + ok + " ok, " + errors + " with errors") + .withPermissionCheck(call.permissionCheck()) + .withRender(render("bulk_load_dry_run", tableName, json)) + .withModelContent(AiDataEnvelope.wrap(DRY_RUN, tableName, response.body())); + } + + + + //////////////////////////////////////////////////////////////////////////// + // write tools // + //////////////////////////////////////////////////////////////////////////// + + + + /******************************************************************************* + ** Propose a save or a run as a pending action (the user confirms in the + ** panel; nothing happens until then). + *******************************************************************************/ + private Integer propose(UiApiToolbox.Context context, AiToolResult result, String toolName, String tableName, Map in) throws QException + { + DataAccessService.Decision decision = dataAccess.forTable(SAVE.equals(toolName) ? PROFILE_TABLE : tableName); + if(!decision.mode().allowsWrite()) + { + deny(result, "dataAccess:" + decision.mode(), "The assistant may only read here (data access " + decision.mode() + ", " + decision.source() + "). Writes need READ_WRITE."); + return (null); + } + + Map values = new LinkedHashMap<>(); + String summary; + if(SAVE.equals(toolName)) + { + String label = required(in, "label"); + values.put("label", label); + values.put("profile", requiredMap(in, "profile")); + if(in.get("shares") instanceof List shares && !shares.isEmpty()) + { + values.put("shares", shares); + } + summary = "Save the bulk load profile \"" + label + "\" for " + tableName + (values.containsKey("shares") ? " and share it" : ""); + } + else + { + values.put("uploadRef", required(in, "uploadRef")); + values.put("profileId", requiredInteger(in, "profileId")); + summary = "Load the attached file into " + tableName + " with saved profile " + values.get("profileId"); + } + + if(context.pendingActions() == null) + { + throw (new QException("The " + toolName + " action could not be proposed.")); + } + String operation = SAVE.equals(toolName) ? "POST /processes/{processName}/init" : "POST " + JobsLaunchBulkLoadRunner.LAUNCH_OPERATION_PATH; + Integer actionId = context.pendingActions().propose(new UiApiToolbox.ProposedWrite(result.getToolCallId(), toolName, operation, tableName, null, values, summary)); + if(actionId == null) + { + throw (new QException("The " + toolName + " action could not be proposed.")); + } + + Map render = new LinkedHashMap<>(); + render.put("type", "pending_action"); + render.put("actionId", actionId); + render.put("toolName", toolName); + render.put("tableName", tableName); + render.put("summary", summary); + result.withOutcome(AiToolOutcome.OK).withSummary("Waiting for your confirmation").withRender(render) + .withModelContent("Proposed (pending action " + actionId + "): " + summary + ". Nothing happens until the user confirms it in the panel."); + return (actionId); + } + + + + /******************************************************************************* + ** Confirmed save: storeSavedBulkLoadProfile, then insertSharedRecord per + ** share (core lets only the owner share). + *******************************************************************************/ + private void save(Call call, AiToolResult result, String tableName, Map values) throws QException + { + Map processValues = new LinkedHashMap<>(); + processValues.put("tableName", tableName); + processValues.put("label", values.get("label")); + processValues.put("mappingJson", JsonUtils.toJson(values.get("profile"))); + processValues.put("isBulkEdit", false); + UiApiResult response = call.invoke(new UiApiCall("POST", "/processes/{processName}/init").withPathParam("processName", "storeSavedBulkLoadProfile") + .withFormParam("values", JsonUtils.toJson(processValues))); + String error = processError(response); + if(error != null) + { + result.withOutcome(response == null || response.isSuccess() ? AiToolOutcome.ERROR : outcomeFor(response.status())).withSummary(shortText(error)).withPermissionCheck(call.permissionCheck()) + .withModelContent("The profile was not saved: " + error); + return; + } + Integer id = ValueUtils.getValueAsInteger(map(map(list(map(response.json().get("values")).get("savedBulkLoadProfileList")).stream().findFirst().orElse(null)).get("values")).get("id")); + + List shared = new ArrayList<>(); + List failed = new ArrayList<>(); + for(Object share : list(values.get("shares"))) + { + Map s = map(share); + Map shareValues = new LinkedHashMap<>(); + shareValues.put("tableName", PROFILE_TABLE); + shareValues.put("recordId", id); + shareValues.put("audienceType", s.get("audienceType")); + shareValues.put("audienceId", s.get("audienceId")); + shareValues.put("scopeId", s.get("scope") == null ? "READ_ONLY" : s.get("scope")); + UiApiResult shareResponse = call.invoke(new UiApiCall("POST", "/processes/{processName}/init").withPathParam("processName", "insertSharedRecord") + .withFormParam("values", JsonUtils.toJson(shareValues))); + String shareError = processError(shareResponse); + (shareError == null ? shared : failed).add(s.get("audienceType") + " " + s.get("audienceId") + (shareError == null ? "" : " (" + shortText(shareError) + ")")); + } + + String summary = "Saved profile \"" + values.get("label") + "\" (id " + id + ")" + (shared.isEmpty() ? "" : ", shared with " + String.join(", ", shared)) + + (failed.isEmpty() ? "" : "; not shared with " + String.join(", ", failed)); + result.withOutcome(AiToolOutcome.OK).withRowCount(1).withSummary(shortText(summary)).withPermissionCheck(call.permissionCheck()) + .withModelContent(JsonUtils.toJson(Map.of("id", id == null ? "" : id, "label", values.get("label"), "shared", shared, "notShared", failed))); + } + + + + /******************************************************************************* + ** Confirmed run: through the runner seam. + *******************************************************************************/ + private void run(Call call, AiToolResult result, UiApiToolbox.Context context, String tableName, Map values) throws QException + { + BulkLoadRunner.RunHandle handle = runner.start(new BulkLoadRunner.Request(context.sessionUUID(), context.userTimezone(), context.gateway(), tableName, + String.valueOf(values.get("uploadRef")), ValueUtils.getValueAsInteger(values.get("profileId")))); + call.operation = handle.operation(); + call.httpStatus = handle.httpStatus(); + boolean ok = !"FAILED".equals(handle.status()) && !"NOT_AVAILABLE".equals(handle.status()); + Map out = new LinkedHashMap<>(); + out.put("status", handle.status()); + out.put("jobId", handle.jobId()); + out.put("processUUID", handle.processUUID()); + out.put("summary", handle.summary()); + result.withOutcome(ok ? AiToolOutcome.OK : AiToolOutcome.ERROR).withSummary(shortText(handle.summary())).withPermissionCheck(call.permissionCheck()) + .withModelContent(JsonUtils.toJson(out)); + } + + + + //////////////////////////////////////////////////////////////////////////// + // helpers // + //////////////////////////////////////////////////////////////////////////// + + + + /******************************************************************************* + ** A tool's input or result as kept in the tool-call log: profiles and + ** counts, never sample values, example bullets or value mappings. + *******************************************************************************/ + public static String forLog(String json) + { + if(json == null) + { + return (null); + } + try + { + int start = json.indexOf('{'); + int end = json.lastIndexOf('}'); + if(start < 0 || end < start) + { + return (json); + } + JSONObject object = new JSONObject(json.substring(start, end + 1)); + strip(object); + return (object.toString()); + } + catch(RuntimeException e) + { + return ("{\"note\":\"not logged\"}"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void strip(Object node) + { + if(node instanceof JSONObject object) + { + for(String key : new ArrayList<>(object.keySet())) + { + if(LOG_STRIPPED_KEYS.contains(key)) + { + object.remove(key); + } + else + { + strip(object.get(key)); + } + } + } + else if(node instanceof JSONArray array) + { + for(int i = 0; i < array.length(); i++) + { + strip(array.get(i)); + } + } + } + + + + /******************************************************************************* + ** A compact render for the panel (counts and error groups, no values). + *******************************************************************************/ + private static Map render(String type, String tableName, Map json) + { + Map render = new LinkedHashMap<>(); + render.put("type", type); + render.put("tableName", tableName); + render.put("mode", json.get("mode")); + render.put("okCount", json.get("okCount")); + render.put("errorCount", json.get("errorCount")); + List> errors = new ArrayList<>(); + for(Object error : list(json.get("errors"))) + { + Map e = map(error); + errors.add(new LinkedHashMap<>(Map.of("rows", String.valueOf(e.get("rowNumbers")), "field", String.valueOf(e.get("fieldName")), "message", String.valueOf(e.get("message"))))); + } + render.put("errors", errors); + return (render); + } + + + + /******************************************************************************* + ** The error of a process init call (HTTP or the process's ERROR type), or + ** null when it completed. + *******************************************************************************/ + private static String processError(UiApiResult response) + { + if(response == null) + { + return ("No response."); + } + if(!response.isSuccess()) + { + return (response.errorMessage() == null ? "HTTP " + response.status() : response.errorMessage()); + } + Map json = response.json(); + if("ERROR".equals(json.get("type"))) + { + return (String.valueOf(json.getOrDefault("userFacingError", json.getOrDefault("error", "The process failed.")))); + } + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void fail(AiToolResult result, Call call, UiApiResult response) + { + if(response == null) + { + result.withOutcome(AiToolOutcome.ERROR).withSummary("Not available") + .withModelContent("This server does not offer " + call.operation + " yet (it needs a newer QQQ). Use the table's Bulk Load screen instead."); + return; + } + AiToolOutcome kind = outcomeFor(response.status()); + String message = response.errorMessage() == null ? "HTTP " + response.status() : response.errorMessage(); + result.withOutcome(kind).withSummary(kind == AiToolOutcome.DENIED ? "Not allowed" : shortText(message)).withPermissionCheck(call.permissionCheck()) + .withModelContent(call.operation + " answered HTTP " + response.status() + ": " + shortText(message)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static AiToolOutcome outcomeFor(int status) + { + return (switch(status) + { + case 401, 403 -> AiToolOutcome.DENIED; + case 404 -> AiToolOutcome.NOT_FOUND; + case 400, 409, 422 -> AiToolOutcome.INVALID_INPUT; + default -> AiToolOutcome.ERROR; + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void deny(AiToolResult result, String check, String message) + { + result.withOutcome(AiToolOutcome.DENIED).withSummary("Not allowed").withPermissionCheck(check).withModelContent(message); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String required(Map in, String name) throws AiToolException + { + if(in.get(name) instanceof String s && StringUtils.hasContent(s)) + { + return (s.strip()); + } + throw (AiToolException.invalidInput(name + " is required.")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Integer requiredInteger(Map in, String name) throws AiToolException + { + try + { + Integer value = ValueUtils.getValueAsInteger(in.get(name)); + if(value != null) + { + return (value); + } + } + catch(RuntimeException e) + { + // fall through + } + throw (AiToolException.invalidInput(name + " must be a number.")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static Map requiredMap(Map in, String name) throws AiToolException + { + Object value = in.get(name); + if(value instanceof String s && s.strip().startsWith("{")) + { + return (new JSONObject(s).toMap()); + } + if(value instanceof Map m && !m.isEmpty()) + { + return ((Map) m); + } + throw (AiToolException.invalidInput(name + " is required (a JSON object).")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List fieldList(Object mappingJson) + { + if(mappingJson == null) + { + return (List.of()); + } + try + { + return (new JSONObject(String.valueOf(mappingJson)).optJSONArray("fieldList") == null ? List.of() + : new JSONObject(String.valueOf(mappingJson)).getJSONArray("fieldList").toList()); + } + catch(RuntimeException e) + { + return (List.of()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static Map map(Object value) + { + return (value instanceof Map m ? (Map) m : Map.of()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static List list(Object value) + { + return (value instanceof List l ? (List) l : List.of()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String shortText(String text) + { + if(text == null) + { + return (""); + } + return (text.length() > 300 ? text.substring(0, 297) + "..." : text); + } + + + + /******************************************************************************* + ** One tool call's UI API invocations: the user's session, the route header, + ** and the last operation and status (for the log). + *******************************************************************************/ + private static class Call + { + private final UiApiToolbox.Context context; + private final boolean hipaaRoute; + private String operation; + private Integer httpStatus; + + + + /******************************************************************************* + ** + *******************************************************************************/ + Call(UiApiToolbox.Context context, boolean hipaaRoute) + { + this.context = context; + this.hipaaRoute = hipaaRoute; + } + + + + /******************************************************************************* + ** Invoke, or null when the server does not offer the operation (an + ** older QQQ). + *******************************************************************************/ + UiApiResult invoke(UiApiCall uiApiCall) throws QException + { + operation = uiApiCall.operation(); + if(!context.gateway().isInvocable(uiApiCall.getMethod(), uiApiCall.getPathTemplate())) + { + return (null); + } + uiApiCall.withSessionUUID(context.sessionUUID()).withUserTimezone(context.userTimezone()).withInvokedBy("assistant") + .withHeader(ROUTE_HEADER, hipaaRoute ? "hipaa" : "standard"); + UiApiResult response = context.gateway().invoke(uiApiCall); + httpStatus = response.status(); + return (response); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + String permissionCheck() + { + return ("uiApi:" + operation + ":" + httpStatus); + } + } +} diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/JobsLaunchBulkLoadRunner.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/JobsLaunchBulkLoadRunner.java new file mode 100644 index 0000000..17a4f2d --- /dev/null +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/bulkload/JobsLaunchBulkLoadRunner.java @@ -0,0 +1,90 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools.bulkload; + + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.assistant.uiapi.UiApiCall; +import com.kingsrook.qbits.assistant.uiapi.UiApiResult; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** The default {@link BulkLoadRunner}: the durable jobs launch, + ** POST /qqq/v1/processes/{table}.bulkInsert/jobs with the upload ref and the + ** saved profile (design spec #1171 sections 2.4 and 3.4). From the assistant + ** channel core creates the job AWAITING_CONFIRMATION, so it starts only when + ** the user confirms it outside the assistant. + ** + ** STAND-IN(core=#1178, removeBy=RC.8): until the jobs kernel ships in a QQQ + ** RC, the launch is not a spec'd operation on the server; the runner then + ** says so instead of guessing at another path (a load needs the upload ref, + ** which the bulk-insert process endpoints cannot take). + *******************************************************************************/ +public class JobsLaunchBulkLoadRunner implements BulkLoadRunner +{ + public static final String LAUNCH_OPERATION_PATH = "/processes/{processName}/jobs"; + + public static final String NOT_AVAILABLE = "Running a bulk load from the assistant needs the durable jobs launch " + + "(POST /qqq/v1/processes/{process}/jobs), which this server does not have yet. Open the table's Bulk Load " + + "screen and choose the saved profile instead."; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RunHandle start(Request request) throws QException + { + String processName = request.tableName() + ".bulkInsert"; + if(!request.gateway().isInvocable("POST", LAUNCH_OPERATION_PATH)) + { + return (new RunHandle("NOT_AVAILABLE", null, null, NOT_AVAILABLE, "POST " + LAUNCH_OPERATION_PATH, null)); + } + + Map body = new LinkedHashMap<>(); + body.put("uploadRefs", List.of(request.uploadRef())); + body.put("savedBulkLoadProfileId", request.savedBulkLoadProfileId()); + body.put("mode", "EXECUTE"); + body.put("label", "Bulk load into " + request.tableName() + " (assistant)"); + + UiApiResult response = request.gateway().invoke(new UiApiCall("POST", LAUNCH_OPERATION_PATH) + .withPathParam("processName", processName) + .withJsonBody(body) + .withSessionUUID(request.sessionUUID()) + .withUserTimezone(request.userTimezone()) + .withInvokedBy("assistant")); + + if(!response.isSuccess()) + { + return (new RunHandle("FAILED", null, null, response.errorMessage() == null ? "HTTP " + response.status() : response.errorMessage(), + "POST " + LAUNCH_OPERATION_PATH, response.status())); + } + + Map json = response.json(); + Long jobId = ValueUtils.getValueAsLong(json.get("jobId")); + String status = json.get("status") == null ? "QUEUED" : String.valueOf(json.get("status")); + String summary = "AWAITING_CONFIRMATION".equals(status) + ? "Job " + jobId + " is waiting for your confirmation on its job record before it starts." + : "Job " + jobId + " is " + status.toLowerCase() + "."; + return (new RunHandle(status, jobId, null, summary, "POST " + LAUNCH_OPERATION_PATH, response.status())); + } +} diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/LoopbackUiApiGateway.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/LoopbackUiApiGateway.java index a91993b..435b353 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/LoopbackUiApiGateway.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/LoopbackUiApiGateway.java @@ -30,6 +30,7 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.StringJoiner; +import java.util.UUID; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; @@ -64,6 +65,9 @@ public class LoopbackUiApiGateway implements UiApiGateway public static final String INVOKED_BY_HEADER = "X-QQQ-Invoked-By"; public static final String TIMEZONE_HEADER = "X-QQQ-UserTimezone"; + /** Headers a call cannot set itself (lower case). */ + private static final Set RESERVED_HEADERS = Set.of("cookie", "authorization", "x-api-key", "content-type", "accept", INVOKED_BY_HEADER.toLowerCase(), TIMEZONE_HEADER.toLowerCase()); + private static final Set SPEC_OPERATIONS = specOperations(); private final Supplier baseUrlSupplier; @@ -204,8 +208,31 @@ public class LoopbackUiApiGateway implements UiApiGateway { builder.header(TIMEZONE_HEADER, call.getUserTimezone()); } + call.getHeaders().forEach((name, value) -> + { + if(!RESERVED_HEADERS.contains(name.toLowerCase())) + { + builder.header(name, value); + } + }); - if(call.getJsonBody() != null) + if(!call.getFormParams().isEmpty()) + { + //////////////////////////////////////////////////////////////// + // multipart/form-data text fields (the /processes endpoints) // + //////////////////////////////////////////////////////////////// + String boundary = "qqq-assistant-" + UUID.randomUUID(); + StringBuilder body = new StringBuilder(); + String lineEnd = "\r\n"; + call.getFormParams().forEach((name, value) -> body.append("--").append(boundary).append(lineEnd) + .append("Content-Disposition: form-data; name=\"").append(name.replace("\"", "")).append("\"").append(lineEnd) + .append("Content-Type: text/plain; charset=UTF-8").append(lineEnd).append(lineEnd) + .append(value).append(lineEnd)); + body.append("--").append(boundary).append("--").append(lineEnd); + builder.header("Content-Type", "multipart/form-data; boundary=" + boundary); + builder.method(call.getMethod(), HttpRequest.BodyPublishers.ofString(body.toString(), StandardCharsets.UTF_8)); + } + else if(call.getJsonBody() != null) { builder.header("Content-Type", "application/json"); builder.method(call.getMethod(), HttpRequest.BodyPublishers.ofString(call.getJsonBody() instanceof String s ? s : JsonUtils.toJson(call.getJsonBody()), StandardCharsets.UTF_8)); diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/UiApiCall.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/UiApiCall.java index 5ba1e9d..6b48370 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/UiApiCall.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/uiapi/UiApiCall.java @@ -33,6 +33,8 @@ public class UiApiCall private final String pathTemplate; private final Map pathParams = new LinkedHashMap<>(); private final Map queryParams = new LinkedHashMap<>(); + private final Map formParams = new LinkedHashMap<>(); + private final Map headers = new LinkedHashMap<>(); private Object jsonBody; private String sessionUUID; private String userTimezone; @@ -128,6 +130,58 @@ public class UiApiCall + /******************************************************************************* + ** Add a multipart form field (ignored when null). A call with form fields + ** is sent as multipart/form-data (the /processes endpoints), mirroring + ** core's UiApiRequest.withFormParam. + *******************************************************************************/ + public UiApiCall withFormParam(String name, Object value) + { + if(value != null) + { + formParams.put(name, String.valueOf(value)); + } + return (this); + } + + + + /******************************************************************************* + ** Getter for formParams + *******************************************************************************/ + public Map getFormParams() + { + return (formParams); + } + + + + /******************************************************************************* + ** Add a request header the invoker sets (e.g. X-QQQ-AI-Route); ignored + ** when null. The session cookie and the invoked-by marker cannot be + ** overridden this way. + *******************************************************************************/ + public UiApiCall withHeader(String name, Object value) + { + if(name != null && value != null) + { + headers.put(name, String.valueOf(value)); + } + return (this); + } + + + + /******************************************************************************* + ** Getter for headers + *******************************************************************************/ + public Map getHeaders() + { + return (headers); + } + + + /******************************************************************************* ** Fluent setter for jsonBody *******************************************************************************/ diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadToolsTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadToolsTest.java new file mode 100644 index 0000000..d0a99ce --- /dev/null +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/bulkload/BulkLoadToolsTest.java @@ -0,0 +1,350 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools.bulkload; + + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qbits.assistant.AssistantQBitConfig; +import com.kingsrook.qbits.assistant.access.DataAccessMode; +import com.kingsrook.qbits.assistant.api.AssistantApi; +import com.kingsrook.qbits.assistant.engine.PolicyService; +import com.kingsrook.qbits.assistant.model.AssistantDataAccess; +import com.kingsrook.qbits.assistant.model.AssistantPendingAction; +import com.kingsrook.qbits.assistant.model.AssistantToolCall; +import com.kingsrook.qbits.assistant.provider.ModelPart; +import com.kingsrook.qbits.assistant.provider.ModelResponse; +import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; +import com.kingsrook.qbits.assistant.testing.BaseAssistantTest; +import com.kingsrook.qbits.assistant.testing.TurnRunner; +import com.kingsrook.qbits.assistant.tools.UiApiToolbox; +import com.kingsrook.qbits.assistant.uiapi.UiApiCall; +import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.assistant.testing.EvalSupport.firstResult; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The bulk-load tools: definitions, the UI API calls they make (as the + ** user, with the route header from the provider, never the model), pending + ** actions for writes, the confirmed save and run, and log hygiene. Plus the + ** golden eval: a messy CSV -> profile -> dry run with an error -> fixed -> + ** saved -> loaded. + *******************************************************************************/ +class BulkLoadToolsTest extends BaseAssistantTest +{ + private final FakeBulkLoadServer server = new FakeBulkLoadServer(true); + private final List runs = new ArrayList<>(); + + + + /******************************************************************************* + ** The fake server as the gateway, and a recording runner. + *******************************************************************************/ + @Override + protected void customizeConfig(AssistantQBitConfig config) + { + config.withUiApiGateway(server).withBulkLoadRunner(request -> + { + runs.add(request); + return (new BulkLoadRunner.RunHandle("AWAITING_CONFIRMATION", 42L, null, "Job 42 is waiting for your confirmation on its job record.", "POST /processes/{processName}/jobs", 201)); + }); + } + + + + /******************************************************************************* + ** The saved-profile tables (so the tools are offered), and write access. + *******************************************************************************/ + @BeforeEach + void setUp() throws QException + { + new SavedBulkLoadProfileMetaDataProvider().defineAll(qInstance, AssistantTestFixture.BACKEND, null); + setAccess(BulkLoadTools.PROFILE_TABLE, DataAccessMode.READ_WRITE); + setAccess("blog_post", DataAccessMode.READ_WRITE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void clear() + { + QContext.clear(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setAccess(String tableName, DataAccessMode mode) throws QException + { + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + try + { + config.getRuntime().getStore().insert(new AssistantDataAccess().withObjectType(AssistantDataAccess.TYPE_TABLE).withObjectName(tableName).withMode(mode.name())); + config.getRuntime().getDataAccessService().invalidate(); + } + finally + { + QContext.clear(); + } + } + + + + /******************************************************************************* + ** One tool call as the turn engine makes it. + *******************************************************************************/ + private UiApiToolbox.Outcome tool(String toolName, Map in, boolean hipaa) + { + QContext.init(qInstance, AssistantTestFixture.ownerSession(AssistantTestFixture.ADMIN, config)); + try + { + return (config.getRuntime().getBulkLoadTools().execute(new UiApiToolbox.Context("session-1", null, server, PolicyService.ToolPolicy.defaults(), + write -> config.getRuntime().getStore().insert(new AssistantPendingAction().withUserId(AssistantTestFixture.ADMIN).withToolName(write.toolName()) + .withTableName(write.tableName()).withUiApiOperation(write.operation()).withParamsJson(JsonUtils.toJson(write.values())) + .withSummary(write.summary()).withStatus(AssistantPendingAction.STATUS_PENDING).withExpiresAt(java.time.Instant.now().plusSeconds(600))).getId()), + "call-1", toolName, in, hipaa)); + } + finally + { + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDefinitionsAreOffered() + { + List names = config.getRuntime().getContextBuilder().tools().stream().map(d -> d.getName()).toList(); + assertThat(names).contains(BulkLoadTools.LIST, BulkLoadTools.DESCRIBE, BulkLoadTools.ATTACH, BulkLoadTools.SUGGEST, BulkLoadTools.DRY_RUN); + assertThat(config.getRuntime().getBulkLoadTools().definitions(PolicyService.ToolPolicy.defaults(), true)) + .extracting(d -> d.getName()).containsExactlyInAnyOrderElementsOf(BulkLoadTools.NAMES); + assertThat(config.getRuntime().getBulkLoadTools().definitions(PolicyService.ToolPolicy.defaults(), false)) + .extracting(d -> d.getName()).doesNotContain(BulkLoadTools.SAVE, BulkLoadTools.RUN); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReadToolsCallTheUiApiWithTheProvidersRoute() + { + UiApiToolbox.Outcome list = tool(BulkLoadTools.LIST, Map.of("tableName", "blog_post"), false); + assertThat(list.getResult().getOutcome()).isEqualTo(AiToolOutcome.OK); + assertThat(list.getResult().getModelContent()).contains("Posts from the CMS export"); + assertThat(list.getOperation()).isEqualTo("POST /table/{tableName}/query"); + + UiApiToolbox.Outcome describe = tool(BulkLoadTools.DESCRIBE, Map.of("id", 3), false); + assertThat(describe.getResult().getModelContent()).contains("fieldList"); + + //////////////////////////////////////////////////////////////////////// + // the route header comes from the provider flag, whatever the input // + //////////////////////////////////////////////////////////////////////// + tool(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "up_1", "X-QQQ-AI-Route", "hipaa", "route", "hipaa"), false); + tool(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "up_1"), true); + List attaches = server.calls("GET /bulkLoad/uploads/{uploadRef}/profile"); + assertThat(attaches.get(0).getHeaders()).containsEntry(BulkLoadTools.ROUTE_HEADER, "standard"); + assertThat(attaches.get(1).getHeaders()).containsEntry(BulkLoadTools.ROUTE_HEADER, "hipaa"); + assertThat(attaches.get(0).getSessionUUID()).isEqualTo("session-1"); + + UiApiToolbox.Outcome missing = tool(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "nope"), false); + assertThat(missing.getResult().getOutcome()).isEqualTo(AiToolOutcome.NOT_FOUND); + + UiApiToolbox.Outcome invalid = tool(BulkLoadTools.DRY_RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1"), false); + assertThat(invalid.getResult().getOutcome()).isEqualTo(AiToolOutcome.INVALID_INPUT); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDataAccessAndAnOlderServer() throws QException + { + setAccess("page", DataAccessMode.NONE); + UiApiToolbox.Outcome denied = tool(BulkLoadTools.SUGGEST, Map.of("tableName", "page", "uploadRef", "up_1"), false); + assertThat(denied.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED); + assertThat(server.getCalls()).isEmpty(); + + setAccess("media", DataAccessMode.READ); + UiApiToolbox.Outcome readOnly = tool(BulkLoadTools.RUN, Map.of("tableName", "media", "uploadRef", "up_1", "profileId", 7), false); + assertThat(readOnly.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED); + + BulkLoadTools older = new BulkLoadTools(qInstance, config.getRuntime().getDataAccessService(), null); + FakeBulkLoadServer oldApi = new FakeBulkLoadServer(false); + QContext.init(qInstance, AssistantTestFixture.ownerSession(AssistantTestFixture.ADMIN, config)); + UiApiToolbox.Outcome outcome = older.execute(new UiApiToolbox.Context("s", null, oldApi, null, null), "c", BulkLoadTools.SUGGEST, + Map.of("tableName", "blog_post", "uploadRef", "up_1"), false); + assertThat(outcome.getResult().getOutcome()).isEqualTo(AiToolOutcome.ERROR); + assertThat(outcome.getResult().getModelContent()).contains("does not offer"); + assertThat(oldApi.getCalls()).isEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testJobsLaunchRunnerSaysWhenTheLaunchIsMissing() throws QException + { + BulkLoadRunner.RunHandle handle = new JobsLaunchBulkLoadRunner().start(new BulkLoadRunner.Request("s", null, server, "blog_post", "up_1", 7)); + assertThat(handle.status()).isEqualTo("NOT_AVAILABLE"); + assertThat(handle.summary()).contains("durable jobs launch"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testForLogDropsSamplesAndValues() + { + String logged = BulkLoadTools.forLog("{\"columns\":[{\"header\":\"A\",\"sampleValues\":[\"" + FakeBulkLoadServer.SAMPLE_SENTINEL + + "\"]}],\"summaryLines\":[{\"bulletsOfText\":[\"Row 3 [x]\"]}],\"profile\":{\"fieldList\":[{\"valueMappings\":{\"secret\":1}}]}}"); + assertThat(logged).doesNotContain(FakeBulkLoadServer.SAMPLE_SENTINEL).doesNotContain("Row 3 [x]").doesNotContain("secret").contains("header"); + assertThat(BulkLoadTools.forLog(null)).isNull(); + assertThat(BulkLoadTools.forLog("plain")).isEqualTo("plain"); + } + + + + /******************************************************************************* + ** Golden eval (owner request 2026-10-13): upload a CSV with messy headers; + ** the assistant proposes a profile, validates it, fixes one error, saves + ** it (shared with a group, after the user confirms) and runs the load + ** (after the user confirms). The scripted model stands in for the LLM; + ** its fix reads the dry run's error from the tool result. + *******************************************************************************/ + @Test + @SuppressWarnings("unchecked") + void testGoldenMessyCsvToSavedProfileAndLoad() throws Exception + { + Map attachments = Map.of("attachments", List.of(Map.of("uploadRef", "up_1", "fileName", "posts.csv"))); + AtomicReference suggested = new AtomicReference<>(); + + provider.thenToolUse(BulkLoadTools.ATTACH, Map.of("tableName", "blog_post", "uploadRef", "up_1")) + .thenToolUse(BulkLoadTools.SUGGEST, Map.of("tableName", "blog_post", "uploadRef", "up_1")) + .thenRespond(request -> + { + Map suggestion = firstResult(request); + suggested.set(JsonUtils.toJson(suggestion.get("profile"))); + return (toolUse("dry1", BulkLoadTools.DRY_RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1", "profile", suggestion.get("profile")))); + }) + .thenRespond(request -> + { + //////////////////////////////////////////////////////////////// + // read the error, fix it: default the missing required field // + //////////////////////////////////////////////////////////////// + Map dryRun = firstResult(request); + assertThat(dryRun.get("errorCount")).isEqualTo(2); + String missingField = String.valueOf(((Map) ((List) dryRun.get("errors")).get(0)).get("fieldName")); + Map profile = new JSONObject(suggested.get()).toMap(); + List> fields = new ArrayList<>((List>) profile.get("fieldList")); + fields.add(new LinkedHashMap<>(Map.of("fieldName", missingField, "defaultValue", "DRAFT"))); + profile.put("fieldList", fields); + return (toolUse("dry2", BulkLoadTools.DRY_RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1", "profile", profile, "mode", "FULL"))); + }) + .thenRespond(request -> + { + Map profile = new JSONObject(suggested.get()).toMap(); + ((List>) profile.get("fieldList")).add(new LinkedHashMap<>(Map.of("fieldName", "status", "defaultValue", "DRAFT"))); + return (toolUse("save", BulkLoadTools.SAVE, Map.of("tableName", "blog_post", "label", "CMS post export", "profile", profile, + "shares", List.of(Map.of("audienceType", "group", "audienceId", "kof22-admins"))))); + }) + .thenText("The file maps cleanly once Status defaults to DRAFT (2 rows, no errors). Confirm to save the profile \"CMS post export\"."); + + TurnRunner.Outcome first = runner.run(AssistantTestFixture.ADMIN, null, "Load the attached posts.csv into blog posts and save the mapping for next time.", attachments); + List tools = first.toolCalls().stream().map(AssistantToolCall::getToolName).toList(); + assertThat(tools).containsExactly(BulkLoadTools.ATTACH, BulkLoadTools.SUGGEST, BulkLoadTools.DRY_RUN, BulkLoadTools.DRY_RUN, BulkLoadTools.SAVE); + + List validates = server.calls("POST /bulkLoad/{tableName}/validate"); + assertThat(validates).hasSize(2); + assertThat(validates).allSatisfy(c -> assertThat(c.getHeaders()).containsEntry(BulkLoadTools.ROUTE_HEADER, "standard")); + assertThat(first.toolCalls().get(2).getResultSummary()).contains("2 with errors"); + assertThat(first.toolCalls().get(3).getResultSummary()).contains("0 with errors"); + assertThat(first.toolCalls()).allSatisfy(c -> assertThat(String.valueOf(c.getResultJson()) + c.getInputJson()).doesNotContain(FakeBulkLoadServer.SAMPLE_SENTINEL)); + assertThat(server.calls("POST /processes/{processName}/init")).as("nothing is saved before the user confirms").isEmpty(); + + Integer saveActionId = first.toolCalls().get(4).getPendingActionId(); + assertThat(saveActionId).isNotNull(); + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + Map saved = new AssistantApi(config.getRuntime(), QContext.getQSession()).confirmAction(saveActionId); + QContext.clear(); + assertThat(saved.get("status")).isEqualTo(AssistantPendingAction.STATUS_EXECUTED); + List processCalls = server.calls("POST /processes/{processName}/init"); + assertThat(processCalls).extracting(c -> c.getPathParams().get("processName")).containsExactly("storeSavedBulkLoadProfile", "insertSharedRecord"); + Map stored = FakeBulkLoadServer.formValues(processCalls.get(0)); + assertThat(stored).containsEntry("label", "CMS post export").containsEntry("tableName", "blog_post"); + assertThat(String.valueOf(stored.get("mappingJson"))).contains("DRAFT"); + assertThat(FakeBulkLoadServer.formValues(processCalls.get(1))).containsEntry("audienceType", "group").containsEntry("audienceId", "kof22-admins") + .containsEntry("recordId", 7).containsEntry("scopeId", "READ_ONLY"); + + //////////////////////////////////////////////////////////////// + // second turn: run the load with the saved profile; confirm // + //////////////////////////////////////////////////////////////// + provider.thenToolUse(BulkLoadTools.RUN, Map.of("tableName", "blog_post", "uploadRef", "up_1", "profileId", 7)) + .thenText("The load is queued as job 42; start it from its job record when you are ready."); + TurnRunner.Outcome second = runner.run(AssistantTestFixture.ADMIN, first.conversationId(), "Now load it.", attachments); + AssistantToolCall runCall = second.toolCalls().get(0); + assertThat(runCall.getToolName()).isEqualTo(BulkLoadTools.RUN); + assertThat(runs).as("nothing runs before the user confirms").isEmpty(); + + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + Map ran = new AssistantApi(config.getRuntime(), QContext.getQSession()).confirmAction(runCall.getPendingActionId()); + QContext.clear(); + assertThat(ran.get("outcome")).isEqualTo("OK"); + assertThat(runs).hasSize(1); + assertThat(runs.get(0).savedBulkLoadProfileId()).isEqualTo(7); + assertThat(runs.get(0).uploadRef()).isEqualTo("up_1"); + assertThat(runs.get(0).tableName()).isEqualTo("blog_post"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static ModelResponse toolUse(String id, String name, Map input) + { + return (new ModelResponse().withParts(List.of(ModelPart.toolUse("toolu_" + id, name, input))).withStopReason(ModelResponse.STOP_TOOL_USE)); + } +} diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/bulkload/FakeBulkLoadServer.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/bulkload/FakeBulkLoadServer.java new file mode 100644 index 0000000..99544b3 --- /dev/null +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/bulkload/FakeBulkLoadServer.java @@ -0,0 +1,176 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools.bulkload; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.assistant.uiapi.UiApiCall; +import com.kingsrook.qbits.assistant.uiapi.UiApiGateway; +import com.kingsrook.qbits.assistant.uiapi.UiApiResult; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** A stand-in for the server's bulk-load UI API operations (core's + ** /bulkLoad/... and the profile processes), recording every call. Its + ** validate answers like core's dry run: the blog_post status field is + ** required, so a profile that neither maps nor defaults it has errors. + *******************************************************************************/ +public class FakeBulkLoadServer implements UiApiGateway +{ + public static final String SAMPLE_SENTINEL = "SAMPLE_SENTINEL_5b1e"; + + private static final Set OPERATIONS = Set.of( + "GET /bulkLoad/uploads/{uploadRef}/profile", "POST /bulkLoad/{tableName}/suggestMapping", "POST /bulkLoad/{tableName}/validate", + "POST /table/{tableName}/query", "GET /table/{tableName}/{primaryKey}", "POST /processes/{processName}/init"); + + private final List calls = new ArrayList<>(); + private final boolean offersBulkLoad; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public FakeBulkLoadServer(boolean offersBulkLoad) + { + this.offersBulkLoad = offersBulkLoad; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isInvocable(String method, String pathTemplate) + { + String operation = method + " " + pathTemplate; + return (OPERATIONS.contains(operation) && (offersBulkLoad || !pathTemplate.startsWith("/bulkLoad"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public synchronized UiApiResult invoke(UiApiCall call) + { + calls.add(call); + String operation = call.operation(); + return (switch(operation) + { + case "GET /bulkLoad/uploads/{uploadRef}/profile" -> "up_1".equals(call.getPathParams().get("uploadRef")) + ? ok(new JSONObject().put("uploadRef", "up_1").put("fileName", "posts.csv").put("hasHeaderRow", true).put("rowsProfiled", 2) + .put("columns", new JSONArray() + .put(new JSONObject().put("columnIndex", 0).put("header", "Post Title").put("mappedFieldName", "title").put("samplesAllowed", true) + .put("sampleValues", new JSONArray().put(SAMPLE_SENTINEL))) + .put(new JSONObject().put("columnIndex", 1).put("header", "Teaser").put("samplesAllowed", false)))) + : new UiApiResult(404, "{\"error\":\"Upload not found.\"}"); + case "POST /bulkLoad/{tableName}/suggestMapping" -> ok(new JSONObject() + .put("profile", new JSONObject().put("version", "v1").put("layout", "FLAT").put("hasHeaderRow", true) + .put("fieldList", new JSONArray().put(new JSONObject().put("fieldName", "title").put("columnIndex", 0).put("headerName", "Post Title")))) + .put("unmappedColumns", new JSONArray().put(new JSONObject().put("columnIndex", 1).put("header", "Teaser"))) + .put("unmappedRequiredFields", new JSONArray().put(new JSONObject().put("fieldName", "status").put("label", "Status")))); + case "POST /bulkLoad/{tableName}/validate" -> validate(call); + case "POST /table/{tableName}/query" -> ok(new JSONObject().put("records", new JSONArray() + .put(new JSONObject().put("values", new JSONObject().put("id", 3).put("label", "Posts from the CMS export").put("userId", "admin-1") + .put("mappingJson", "{\"version\":\"v1\",\"fieldList\":[{\"fieldName\":\"title\",\"columnIndex\":0}]}"))))); + case "GET /table/{tableName}/{primaryKey}" -> ok(new JSONObject().put("record", new JSONObject().put("values", new JSONObject().put("id", 3) + .put("label", "Posts from the CMS export").put("tableName", "blog_post").put("userId", "admin-1") + .put("mappingJson", "{\"version\":\"v1\",\"fieldList\":[{\"fieldName\":\"title\",\"columnIndex\":0}]}")))); + case "POST /processes/{processName}/init" -> "storeSavedBulkLoadProfile".equals(call.getPathParams().get("processName")) + ? ok(new JSONObject().put("type", "COMPLETE").put("values", new JSONObject().put("savedBulkLoadProfileList", + new JSONArray().put(new JSONObject().put("values", new JSONObject().put("id", 7).put("label", "x")))))) + : ok(new JSONObject().put("type", "COMPLETE").put("values", new JSONObject())); + default -> new UiApiResult(404, "{\"error\":\"no such operation\"}"); + }); + } + + + + /******************************************************************************* + ** Errors unless status is mapped or defaulted. + *******************************************************************************/ + private UiApiResult validate(UiApiCall call) + { + JSONObject body = new JSONObject(JsonUtils.toJson(call.getJsonBody())); + JSONArray fieldList = body.getJSONObject("profile").getJSONArray("fieldList"); + boolean hasStatus = false; + for(int i = 0; i < fieldList.length(); i++) + { + JSONObject field = fieldList.getJSONObject(i); + if("status".equals(field.optString("fieldName")) && (field.has("columnIndex") || !field.isNull("defaultValue") && field.has("defaultValue"))) + { + hasStatus = true; + } + } + JSONObject result = new JSONObject().put("mode", body.optString("mode", "SAMPLE")).put("processName", "blog_post.bulkInsert").put("recordCount", 2); + if(hasStatus) + { + return (ok(result.put("okCount", 2).put("errorCount", 0).put("errors", new JSONArray()))); + } + return (ok(result.put("okCount", 0).put("errorCount", 2).put("errors", new JSONArray() + .put(new JSONObject().put("rowNumbers", "Rows 2-3").put("fieldName", "status").put("errorCode", "REQUIRED").put("message", "Missing value in required field: Status"))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static UiApiResult ok(JSONObject json) + { + return (new UiApiResult(200, json.toString())); + } + + + + /******************************************************************************* + ** The calls, in order. + *******************************************************************************/ + public synchronized List getCalls() + { + return (new ArrayList<>(calls)); + } + + + + /******************************************************************************* + ** Calls of one operation. + *******************************************************************************/ + public synchronized List calls(String operation) + { + return (calls.stream().filter(c -> c.operation().equals(operation)).toList()); + } + + + + /******************************************************************************* + ** A form value of a process call, as JSON. + *******************************************************************************/ + public static Map formValues(UiApiCall call) + { + return (new JSONObject(call.getFormParams().get("values")).toMap()); + } +}