From a712e6fbc87808fec07a2b092d081f617dc7d303 Mon Sep 17 00:00:00 2001 From: CI Date: Sat, 7 Nov 2026 17:28:26 +0000 Subject: [PATCH 1/8] Bump version to 0.2.0-SNAPSHOT [skip ci] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 647f607..c800fdf 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ - 0.1.0-SNAPSHOT + 0.2.0-SNAPSHOT 21 21 UTF-8 -- 2.54.0 From a588aafe2e3894c3e1657bc7f97fed3ff7e19b81 Mon Sep 17 00:00:00 2001 From: James Maes Date: Sun, 8 Nov 2026 19:46:55 -0600 Subject: [PATCH 2/8] test(assistant): pin the live thread to real title, question, answer and tool steps at message.end The panel reloads the conversation at message.end. A tool-using turn must already have stored the real title, the question and answer parts and the tool-call rows then, not the transcript redactor's placeholders (changed:v1, [], {}). The redactor is always attached to a turn, so this is independent of Secured Data. Fails without the flush-before-message.end ordering (RC.24), passes on develop and RC.25. Co-Authored-By: Claude Sonnet 5.5 --- .../assistant/engine/TurnEngineTest.java | 28 ++++++++++++ .../qbits/assistant/testing/TurnRunner.java | 43 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/TurnEngineTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/TurnEngineTest.java index 7660df2..d2c7ee2 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/TurnEngineTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/TurnEngineTest.java @@ -162,6 +162,34 @@ class TurnEngineTest extends BaseAssistantTest + /******************************************************************************* + ** The live thread after a tool-using turn: when message.end arrives the panel + ** reloads, and must find the real title, the question, the answer and the tool + ** steps - not the transcript redactor's placeholders (changed:v1, [], {}). + *******************************************************************************/ + @Test + void testLiveThreadIsNotPlaceholdersAtMessageEnd() throws Exception + { + provider.thenToolUse("count_records", input("table", "blog_post", "filter", filter("AND", "status", "EQUALS", List.of("PUBLISHED")))) + .thenText("Four posts are published."); + + TurnRunner.Outcome outcome = runner.run(EDITOR, "how many published posts?"); + assertThat(outcome.end()).containsEntry("status", "COMPLETE"); + + assertThat(runner.conversationAtEnd().getTitle()).isEqualTo("how many published posts?"); + List stored = runner.messagesAtEnd(); + assertThat(stored).extracting(AssistantMessage::getPartsJson).noneMatch(parts -> parts.equals("[]") || parts.contains("changed:v1")); + assertThat(stored.get(0).getPartsJson()).contains("how many published posts?"); + assertThat(stored.get(stored.size() - 1).getPartsJson()).contains("Four posts are published."); + assertThat(runner.toolCallsAtEnd()).isNotEmpty().allSatisfy(call -> + { + assertThat(call.getInputJson()).isNotEqualTo("{}"); + assertThat(call.getResultSummary()).doesNotContain("changed:v1"); + }); + } + + + /******************************************************************************* ** Parallel tool calls: all results in ONE user message, in order. *******************************************************************************/ diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/TurnRunner.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/TurnRunner.java index 00f17ac..fe5d3e9 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/TurnRunner.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/TurnRunner.java @@ -27,6 +27,7 @@ import com.kingsrook.qbits.assistant.AssistantRuntime; import com.kingsrook.qbits.assistant.api.AssistantApi; import com.kingsrook.qbits.assistant.engine.RunningTurn; import com.kingsrook.qbits.assistant.engine.TurnEvent; +import com.kingsrook.qbits.assistant.model.AssistantConversation; import com.kingsrook.qbits.assistant.model.AssistantMessage; import com.kingsrook.qbits.assistant.model.AssistantToolCall; import com.kingsrook.qbits.assistant.persistence.AssistantStore; @@ -49,6 +50,8 @@ public class TurnRunner private final AssistantQBitConfig config; private List messagesAtEnd = List.of(); + private AssistantConversation conversationAtEnd; + private List toolCallsAtEnd = List.of(); @@ -175,6 +178,8 @@ public class TurnRunner if(batch.stream().anyMatch(e -> TurnEvent.MESSAGE_END.equals(e.getType()))) { messagesAtEnd = messagesOf(userId, convId); + conversationAtEnd = conversationOf(userId, convId); + toolCallsAtEnd = toolCalls(userId, running.getTurnId()); } if(!batch.isEmpty()) { @@ -213,6 +218,44 @@ public class TurnRunner + /******************************************************************************* + ** The conversation row as stored the moment message.end was seen. + *******************************************************************************/ + public AssistantConversation conversationAtEnd() + { + return (conversationAtEnd); + } + + + + /******************************************************************************* + ** The audited tool calls as stored the moment message.end was seen. + *******************************************************************************/ + public List toolCallsAtEnd() + { + return (toolCallsAtEnd); + } + + + + /******************************************************************************* + ** One conversation of the user, read as the user. + *******************************************************************************/ + private AssistantConversation conversationOf(String userId, Integer conversationId) throws Exception + { + QContext.init(qInstance, AssistantTestFixture.ownerSession(userId, config)); + try + { + return (new AssistantStore().getConversation(userId, conversationId)); + } + finally + { + QContext.clear(); + } + } + + + /******************************************************************************* ** The stored messages of a conversation (read as the user). *******************************************************************************/ -- 2.54.0 From dfe63dcc90e6a970e6a228857edea58c4cb768e4 Mon Sep 17 00:00:00 2001 From: CI Date: Mon, 9 Nov 2026 02:44:36 +0000 Subject: [PATCH 3/8] Bump version to 0.3.0-SNAPSHOT [skip ci] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c800fdf..afa6391 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ - 0.2.0-SNAPSHOT + 0.3.0-SNAPSHOT 21 21 UTF-8 -- 2.54.0 From 0c32c43f8c3e40ce508f58985c1b9ccdf26b2112 Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 9 Nov 2026 09:48:47 -0600 Subject: [PATCH 4/8] fix(engine): a stalled database can no longer hang a turn on 'Thinking' The watchdog ran the deadline logic after two database reads and heartbeat() was synchronized on the monitor that finish() needs, so one blocked database call stopped the deadline, the force-finalize and the final message.end. The deadline now runs first with no database access, housekeeping runs on a bounded abandonable worker, finish() has a guard that ends the stream with an error event, and every phase logs at INFO. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 + .../qbits/assistant/engine/TurnEngine.java | 260 ++++++++++-- .../engine/StalledDatabaseTurnsTest.java | 395 ++++++++++++++++++ 3 files changed, 629 insertions(+), 32 deletions(-) create mode 100644 qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/StalledDatabaseTurnsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c83d7f3..aac4947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased - A turn always ends + +- Fixed an answer that stayed on "Thinking..." forever (seen after the tool steps had finished) when the host database stopped answering, for example a connection pool with every connection checked out. The watchdog's heartbeat did its database read and write while holding the engine's monitor and before it checked the deadline, so one blocked heartbeat stopped the deadline from ever firing and stopped `finish` from running. The deadline is now checked first on every tick without touching the database; the heartbeat, cancel poll and kill-switch poll run on a separate bounded worker that is reported (WARN), interrupted and abandoned if it does not return within two heartbeat intervals; and the engine's monitor is no longer held across database I/O in the heartbeat or while waiting to finalize. +- A finalize guard ends the live stream with an `error` event and `message.end` if the terminal save has not completed within `turnForceFinalizeGraceSeconds`, so the panel never waits on the database to be told the answer failed. The terminal row is left for lease recovery. +- New INFO log lines for every phase: `Assistant turn started`, `Assistant model call started` / `finished`, `Assistant tool call started` / `finished`, `Assistant turn finalizing`, `Assistant turn finished` (no message content). WARN lines when the deadline passes, when the watchdog finalizes a turn, when a heartbeat does not return, and when the guard ends a stream. + ## Unreleased - Answer visible at the end of the stream - The turn's redacted transcript (the question and answer parts, pending actions, stored events) is now written before `message.end` is sent. The panel reloads the conversation the moment the stream ends, and used to read the empty placeholders (`[]`) the transcript redactor stores until its flush, so the newest question and answer showed as empty bubbles until the panel was closed and reopened. diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java index 4e2f034..27fa450 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java @@ -31,12 +31,15 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReference; import com.kingsrook.qbits.assistant.AssistantQBitConfig; import com.kingsrook.qbits.assistant.AssistantRuntime; import com.kingsrook.qbits.assistant.attachments.AttachmentService; @@ -156,6 +159,22 @@ public class TurnEngine implements Runnable private ProviderService.Active budgetOrigin; private volatile boolean finished = false; + + /////////////////////////////////////////////////////////////////////////// + // Terminal-event bookkeeping, shared with the finalize guard. The guard // + // ends the live stream when the database will not let finish() reach // + // message.end, so these must be usable without the engine's monitor. // + /////////////////////////////////////////////////////////////////////////// + private final CountDownLatch finishedLatch = new CountDownLatch(1); + private final AtomicBoolean streamEnded = new AtomicBoolean(); + private final AtomicBoolean errorEmitted = new AtomicBoolean(); + + /////////////////////////////////////////////////////////////////////////// + // The watchdog's database duties (heartbeat, cancel poll, kill switch) // + // run on one bounded worker at a time, never on the watchdog thread // + /////////////////////////////////////////////////////////////////////////// + private final AtomicReference housekeeper = new AtomicReference<>(); + private volatile long housekeeperStartedNanos; private UiApiCredentials credentials; private volatile TranscriptRedactor transcript; private boolean engineRunning; @@ -259,6 +278,8 @@ public class TurnEngine implements Runnable engineRunning = true; attachTranscript(); } + LOG.info("Assistant turn started", logPair("turnId", turn.getId()), logPair("conversationId", conversation.getId()), logPair("userId", userId), + logPair("maxTurnSeconds", config.getMaxTurnSeconds())); startWatchdog(); scheduledTask = runtime.getScheduledTurn().task(conversation.getId()); if(conversation.getScheduledTaskId() != null && scheduledTask == null) @@ -520,8 +541,15 @@ public class TurnEngine implements Runnable /******************************************************************************* - ** Renew the lease, poll cancellation and the kill switch with the user's - ** own session, and enforce a hard deadline even if a provider is wedged. + ** Enforce a hard deadline even if a provider or the database is wedged, and + ** renew the lease, poll cancellation and the kill switch with the user's own + ** session. + ** + ** The deadline is checked first on every tick and touches no database: a + ** connection pool with every connection checked out blocks callers forever, + ** so anything that reads or writes the database runs on a separate, bounded + ** worker (see tickHousekeeping) and can never stop this thread from + ** cancelling and finalizing the turn. *******************************************************************************/ private void startWatchdog() { @@ -540,36 +568,15 @@ public class TurnEngine implements Runnable { break; } - try + enforceDeadline(); + if(running.isFinalized()) { - if(System.nanoTime() - lastHeartbeat >= heartbeatMillis * 1_000_000L) - { - lastHeartbeat = System.nanoTime(); - heartbeat(); - } - if(runtime.getPolicyService().isKillSwitchOn()) - { - runtime.cancelAll("KILL_SWITCH"); - } + break; } - catch(Exception e) + boolean heartbeatDue = System.nanoTime() - lastHeartbeat >= heartbeatMillis * 1_000_000L; + if(tickHousekeeping(heartbeatDue, heartbeatMillis) && heartbeatDue) { - LOG.warn("Assistant watchdog tick failed", hipaaSafe(e), logPair("turnId", turn.getId())); - } - finally - { - // Deadlines do not depend on successful database or policy reads. - Instant now = runtime.getClock().instant(); - if(!now.isBefore(deadline)) - { - safely(() -> running.cancel("TIMEOUT")); - } - if(!now.isBefore(deadline.plusSeconds(config.getTurnForceFinalizeGraceSeconds())) - || running.cancelGraceElapsed(config.getTurnForceFinalizeGraceSeconds())) - { - Stop stop = running.isCancelled() ? cancelledStop() : new Stop("ERROR", "TIMEOUT", TIMEOUT_TEXT, "timeout"); - finish(stop.status(), stop.stopReason(), stop.errorCode(), stop.message()); - } + lastHeartbeat = System.nanoTime(); } } } @@ -577,6 +584,10 @@ public class TurnEngine implements Runnable { Thread.currentThread().interrupt(); } + catch(Exception | Error e) + { + LOG.warn("Assistant watchdog stopped unexpectedly", hipaaSafe(e), logPair("turnId", turn.getId())); + } finally { QContext.clear(); @@ -587,9 +598,103 @@ public class TurnEngine implements Runnable /******************************************************************************* - ** Renew only an active row, then observe persisted cancellation requests. + ** The deadline and the force-finalize grace period. No database access. *******************************************************************************/ - private synchronized void heartbeat() throws QException + private void enforceDeadline() + { + Instant now = runtime.getClock().instant(); + if(!now.isBefore(deadline) && !running.isCancelled()) + { + LOG.warn("Assistant turn passed its deadline; cancelling", logPair("turnId", turn.getId()), logPair("phase", running.getPhase()), + logPair("tool", running.getToolName()), logPair("maxTurnSeconds", config.getMaxTurnSeconds())); + cancelWithoutBlocking("TIMEOUT"); + } + if(!now.isBefore(deadline.plusSeconds(config.getTurnForceFinalizeGraceSeconds())) + || running.cancelGraceElapsed(config.getTurnForceFinalizeGraceSeconds())) + { + LOG.warn("Assistant watchdog is finalizing the turn", logPair("turnId", turn.getId()), logPair("phase", running.getPhase()), + logPair("tool", running.getToolName()), logPair("cancelReason", running.getCancelReason()), logPair("modelCalls", modelCallCount), + logPair("toolCalls", toolCallCount), logPair("graceSeconds", config.getTurnForceFinalizeGraceSeconds())); + Stop stop = running.isCancelled() ? cancelledStop() : new Stop("ERROR", "TIMEOUT", TIMEOUT_TEXT, "timeout"); + finish(stop.status(), stop.stopReason(), stop.errorCode(), stop.message()); + } + } + + + + /******************************************************************************* + ** Cancel on a throwaway thread: cancelling aborts the provider's stream, + ** which is a call into code that may itself be wedged. + *******************************************************************************/ + private void cancelWithoutBlocking(String reason) + { + Thread.ofVirtual().name("assistant-cancel-" + turn.getId()).start(() -> safely(() -> running.cancel(reason))); + } + + + + /******************************************************************************* + ** Start the database duties of one watchdog tick (the heartbeat when due, + ** then the kill-switch poll) on a worker thread, unless the previous one is + ** still running. A worker that has not returned after two heartbeat + ** intervals is reported, interrupted and abandoned, so a database that + ** stopped answering is visible and later ticks can try again. + ** Returns whether a worker was started. + *******************************************************************************/ + private boolean tickHousekeeping(boolean heartbeatDue, long heartbeatMillis) + { + Thread current = housekeeper.get(); + if(current != null) + { + long runningMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - housekeeperStartedNanos); + if(runningMillis < 2 * heartbeatMillis) + { + return (false); + } + LOG.warn("Assistant heartbeat did not finish in time; the database may not be answering", logPair("turnId", turn.getId()), + logPair("phase", running.getPhase()), logPair("blockedMillis", runningMillis)); + current.interrupt(); + housekeeper.compareAndSet(current, null); + } + + housekeeperStartedNanos = System.nanoTime(); + Thread worker = Thread.ofVirtual().name("assistant-housekeeping-" + turn.getId()).unstarted(() -> + { + try + { + QContext.init(runtime.getQInstance(), session); + if(heartbeatDue) + { + heartbeat(); + } + if(runtime.getPolicyService().isKillSwitchOn()) + { + runtime.cancelAll("KILL_SWITCH"); + } + } + catch(Exception | Error e) + { + LOG.warn("Assistant watchdog tick failed", hipaaSafe(e), logPair("turnId", turn.getId())); + } + finally + { + QContext.clear(); + housekeeper.compareAndSet(Thread.currentThread(), null); + } + }); + housekeeper.set(worker); + worker.start(); + return (true); + } + + + + /******************************************************************************* + ** Renew only an active row, then observe persisted cancellation requests. + ** Not synchronized on the engine: it does database I/O, and the engine's + ** monitor must never be held across it. + *******************************************************************************/ + private void heartbeat() throws QException { if(running.isFinalized()) { @@ -838,12 +943,17 @@ public class TurnEngine implements Runnable } modelCallCount++; phase("thinking", null); + LOG.info("Assistant model call started", logPair("turnId", turn.getId()), logPair("call", modelCallCount), logPair("model", model), + logPair("messages", request.getMessages().size())); StreamHandle handle = provider.stream(request.copy(), fallback.listener(listener)); running.setCurrentHandle(handle); try { Duration remaining = Duration.between(runtime.getClock().instant(), deadline); response = handle.await(remaining.isNegative() ? Duration.ofMillis(1) : remaining); + LOG.info("Assistant model call finished", logPair("turnId", turn.getId()), logPair("call", modelCallCount), logPair("model", model), + logPair("stopReason", response.getStopReason()), logPair("toolUses", response.getToolUses().size()), + logPair("durationMs", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started))); } finally { @@ -936,6 +1046,7 @@ public class TurnEngine implements Runnable QContext.init(runtime.getQInstance(), session); permits.acquire(); startedNanos.set(index, System.nanoTime()); + LOG.info("Assistant tool call started", logPair("turnId", turn.getId()), logPair("tool", call.getName())); try { if(!running.isCancelled()) @@ -984,6 +1095,9 @@ public class TurnEngine implements Runnable finally { permits.release(); + LOG.info("Assistant tool call finished", logPair("turnId", turn.getId()), logPair("tool", call.getName()), + logPair("outcome", results[index] == null || results[index].getOutcome() == null ? null : results[index].getOutcome().name()), + logPair("durationMs", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos.get(index)))); } } catch(InterruptedException e) @@ -1287,12 +1401,74 @@ public class TurnEngine implements Runnable /******************************************************************************* ** Persist the message, turn, conversation and usage; emit the end events. *******************************************************************************/ - private synchronized void finish(String status, String stopReason, String errorCode, String errorText) + private void finish(String status, String stopReason, String errorCode, String errorText) { if(!running.tryFinalize()) { return; } + LOG.info("Assistant turn finalizing", logPair("turnId", turn.getId()), logPair("status", status), logPair("errorCode", errorCode), + logPair("phase", running.getPhase()), logPair("cancelReason", running.getCancelReason())); + startFinalizeGuard(status, stopReason, errorCode, errorText); + synchronized(this) + { + finishLocked(status, stopReason, errorCode, errorText); + } + } + + + + /******************************************************************************* + ** The user must never wait on the database to be told the turn is over. + ** finish() persists the transcript, the message and the terminal row before + ** it sends message.end (the panel reloads the conversation at once), and a + ** database that stopped answering would hold it there forever with the + ** panel on "Thinking". If finish() has not completed within the + ** force-finalize grace period, end the live stream from here, without + ** touching the database or the engine's monitor: an error event with the + ** reason, then message.end. The terminal row is left for lease recovery. + *******************************************************************************/ + private void startFinalizeGuard(String status, String stopReason, String errorCode, String errorText) + { + long graceMillis = config.getTurnForceFinalizeGraceSeconds() * 1000L; + Thread.ofVirtual().name("assistant-finalize-guard-" + turn.getId()).start(() -> + { + try + { + if(finishedLatch.await(graceMillis, TimeUnit.MILLISECONDS) || !streamEnded.compareAndSet(false, true)) + { + return; + } + LOG.warn("Assistant turn could not be saved in time; ending the stream without it (database not answering?)", logPair("turnId", turn.getId()), + logPair("status", status), logPair("errorCode", errorCode), logPair("graceSeconds", config.getTurnForceFinalizeGraceSeconds())); + TurnEventLog log = running.getLog(); + if(errorCode != null && !errorEmitted.get()) + { + String message = StringUtils.hasContent(errorText) ? errorText : "Something went wrong while answering. Try again."; + log.append(TurnEvent.ERROR, map("code", errorCode, "message", message)); + } + log.append(TurnEvent.MESSAGE_END, map("messageId", assistantMessage.getId(), "status", status, "stopReason", stopReason)); + log.close(); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + } + catch(Exception | Error e) + { + LOG.warn("Assistant finalize guard failed", hipaaSafe(e), logPair("turnId", turn.getId())); + running.getLog().close(); + } + }); + } + + + + /******************************************************************************* + ** The body of finish(), under the engine's monitor. + *******************************************************************************/ + private void finishLocked(String status, String stopReason, String errorCode, String errorText) + { TranscriptRedactor previousTranscript = TranscriptRedactor.current(); try { @@ -1421,6 +1597,7 @@ public class TurnEngine implements Runnable finally { finished = true; + finishedLatch.countDown(); running.getLog().close(); safely(() -> { @@ -1559,6 +1736,25 @@ public class TurnEngine implements Runnable { return; } + /////////////////////////////////////////////////////////////////////// + // the finalize guard may already have ended the stream; nothing may // + // follow its message.end, and message.end is sent exactly once // + /////////////////////////////////////////////////////////////////////// + if(TurnEvent.MESSAGE_END.equals(type)) + { + if(!streamEnded.compareAndSet(false, true)) + { + return; + } + } + else if(streamEnded.get()) + { + return; + } + if(TurnEvent.ERROR.equals(type)) + { + errorEmitted.set(true); + } toPersist.add(running.getLog().append(type, data)); } diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/StalledDatabaseTurnsTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/StalledDatabaseTurnsTest.java new file mode 100644 index 0000000..fb1cafe --- /dev/null +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/StalledDatabaseTurnsTest.java @@ -0,0 +1,395 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.engine; + + +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import com.kingsrook.qbits.assistant.AssistantQBitConfig; +import com.kingsrook.qbits.assistant.AssistantRuntime; +import com.kingsrook.qbits.assistant.api.AssistantApi; +import com.kingsrook.qbits.assistant.model.AssistantTurn; +import com.kingsrook.qbits.assistant.persistence.AssistantStore; +import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider; +import com.kingsrook.qbits.assistant.spi.ModelEventListener; +import com.kingsrook.qbits.assistant.spi.ModelPart; +import com.kingsrook.qbits.assistant.spi.ModelRequest; +import com.kingsrook.qbits.assistant.spi.ModelResponse; +import com.kingsrook.qbits.assistant.spi.StreamHandle; +import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; +import com.kingsrook.qbits.assistant.testing.BaseAssistantTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.assistant.testing.AssistantTestFixture.EDITOR; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** A turn must always end for the user, even when the database stops answering + ** (a connection pool with every connection checked out blocks forever). These + ** tests model that with a store whose calls park until the test releases + ** them, and prove the user still gets an error and message.end. + *******************************************************************************/ +class StalledDatabaseTurnsTest extends BaseAssistantTest +{ + private final CountDownLatch dbGate = new CountDownLatch(1); + private final AtomicBoolean stallHeartbeat = new AtomicBoolean(); + private final AtomicBoolean stallAllCalls = new AtomicBoolean(); + + private QSession session; + private AssistantApi api; + private AssistantStore store; + + + + /******************************************************************************* + ** Short lease, deadline and grace settings keep the tests bounded. + *******************************************************************************/ + @Override + protected void customizeConfig(AssistantQBitConfig config) + { + config.withTurnHeartbeatSeconds(1).withTurnLeaseSeconds(3).withTurnForceFinalizeGraceSeconds(1) + .withMaxTurnSeconds(2).withKillSwitchPollMillis(50); + } + + + + /******************************************************************************* + ** Install the store whose calls can stall. + *******************************************************************************/ + @BeforeEach + void stallingBackend() + { + config.getRuntime().close(); + config.setRuntime(new AssistantRuntime(qInstance, config, Clock.systemUTC(), new StallingStore(config))); + session = AssistantTestFixture.ownerSession(EDITOR, config); + QContext.init(qInstance, session); + api = new AssistantApi(config.getRuntime(), session); + store = config.getRuntime().getStore(); + } + + + + /******************************************************************************* + ** Never leave a parked thread behind. + *******************************************************************************/ + @AfterEach + void releaseDatabase() + { + dbGate.countDown(); + } + + + + /******************************************************************************* + ** The incident's shape, part one: the heartbeat's database write never + ** returns while a model call is wedged. The deadline must still fire and + ** the turn must end as a TIMEOUT error, with the row written. + *******************************************************************************/ + @Test + void blockedHeartbeatDoesNotStopTheDeadline() throws Exception + { + stallHeartbeat.set(true); + config.withModelProvider(new WedgedProvider()); + var logger = QLogger.activateCollectingLoggerForClass(TurnEngine.class); + try + { + RunningTurn run = api.startTurn(conversation(), Map.of("text", "how many scanners are connected?")); + await(() -> run.getLog().isClosed(), 8); + + List events = run.getLog().after(0, 1); + assertThat(events).filteredOn(TurnEvent::isTerminal).hasSize(1); + assertThat(events).filteredOn(event -> TurnEvent.ERROR.equals(event.getType())).singleElement() + .satisfies(event -> assertThat(event.getData()).containsEntry("code", "TIMEOUT")); + await(() -> config.getRuntime().getRunningTurn(run.getTurnId()) == null, 8); + AssistantTurn row = store.getTurn(EDITOR, run.getTurnId()); + assertThat(row.getStatus()).isEqualTo("ERROR"); + assertThat(row.getErrorCode()).isEqualTo("TIMEOUT"); + assertThat(messages(logger)).anySatisfy(message -> assertThat(message).contains("Assistant watchdog is finalizing the turn")); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(TurnEngine.class); + } + } + + + + /******************************************************************************* + ** The incident's shape, part two: tools ran ("Described Mobile Devices"), + ** then the database stopped answering for everything, so the second model + ** call, the usage write and the terminal writes all hang. The stream must + ** still end with an error and message.end instead of "Thinking" forever. + *******************************************************************************/ + @Test + void databaseStallAfterToolsStillEndsTheStream() throws Exception + { + stallHeartbeat.set(true); + provider.thenToolUse("count_records", input("table", "blog_post", "filter", filter("AND"))) + .thenRespond(request -> + { + stallAllCalls.set(true); + return (new ModelResponse().withParts(List.of(ModelPart.text("There are 3 scanners."))).withStopReason(ModelResponse.STOP_END_TURN)); + }); + var logger = QLogger.activateCollectingLoggerForClass(TurnEngine.class); + try + { + RunningTurn run = api.startTurn(conversation(), Map.of("text", "how many scanners are connected?")); + await(() -> run.getLog().isClosed(), 10); + + List events = run.getLog().after(0, 1); + assertThat(events).extracting(TurnEvent::getType).contains(TurnEvent.TOOL_START, TurnEvent.TOOL_RESULT); + assertThat(events).filteredOn(TurnEvent::isTerminal).hasSize(1); + assertThat(events.getLast().isTerminal()).isTrue(); + assertThat(events).filteredOn(event -> TurnEvent.ERROR.equals(event.getType())).isNotEmpty(); + assertThat(messages(logger)).anySatisfy(message -> assertThat(message).contains("Assistant tool call finished")); + assertThat(messages(logger)).anySatisfy(message -> assertThat(message).contains("Assistant model call started")); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(TurnEngine.class); + } + } + + + + /******************************************************************************* + ** A healthy turn logs every phase transition at INFO and is unchanged. + *******************************************************************************/ + @Test + void healthyTurnLogsEveryPhase() throws Exception + { + provider.thenToolUse("count_records", input("table", "blog_post", "filter", filter("AND"))).thenText("Three."); + var logger = QLogger.activateCollectingLoggerForClass(TurnEngine.class); + try + { + RunningTurn run = api.startTurn(conversation(), Map.of("text", "how many posts?")); + await(() -> run.getLog().isClosed(), 8); + await(() -> config.getRuntime().getRunningTurn(run.getTurnId()) == null, 8); + + assertThat(store.getTurn(EDITOR, run.getTurnId()).getStatus()).isEqualTo("COMPLETE"); + List messages = messages(logger); + assertThat(messages).anySatisfy(message -> assertThat(message).contains("Assistant turn started")); + assertThat(messages.stream().filter(message -> message.contains("Assistant model call started")).count()).isEqualTo(2); + assertThat(messages.stream().filter(message -> message.contains("Assistant model call finished")).count()).isEqualTo(2); + assertThat(messages).anySatisfy(message -> assertThat(message).contains("Assistant tool call started")); + assertThat(messages).anySatisfy(message -> assertThat(message).contains("Assistant tool call finished")); + assertThat(messages).anySatisfy(message -> assertThat(message).contains("Assistant turn finished")); + assertThat(messages).noneSatisfy(message -> assertThat(message).contains("watchdog is finalizing")); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(TurnEngine.class); + } + } + + + + /******************************************************************************* + ** The messages the collecting logger saw. + *******************************************************************************/ + private static List messages(QCollectingLogger logger) + { + return (logger.getCollectedMessages().stream().map(entry -> entry.getMessage()).toList()); + } + + + + /******************************************************************************* + ** Create a user-owned conversation. + *******************************************************************************/ + private Integer conversation() throws Exception + { + return ((Integer) api.createConversation(Map.of()).get("id")); + } + + + + /******************************************************************************* + ** Poll with a hard bound. + *******************************************************************************/ + private static void await(BooleanSupplier condition, int seconds) throws InterruptedException + { + long until = System.nanoTime() + Duration.ofSeconds(seconds).toNanos(); + while(!condition.getAsBoolean() && System.nanoTime() < until) + { + Thread.sleep(10); + } + assertThat(condition.getAsBoolean()).as("condition reached within " + seconds + " seconds").isTrue(); + } + + + + /******************************************************************************* + ** Park like a pool checkout that never returns: interruption does not help. + *******************************************************************************/ + private static void awaitUninterruptibly(CountDownLatch latch) + { + boolean interrupted = false; + while(latch.getCount() != 0) + { + try + { + latch.await(50, TimeUnit.MILLISECONDS); + } + catch(InterruptedException e) + { + interrupted = true; + } + } + if(interrupted) + { + Thread.currentThread().interrupt(); + } + } + + + + /******************************************************************************* + ** A provider that ignores both its timeout and cancellation until the test ends. + *******************************************************************************/ + private class WedgedProvider extends ScriptedModelProvider + { + /******************************************************************************* + ** Start a stream that never completes. + *******************************************************************************/ + @Override + public StreamHandle stream(ModelRequest request, ModelEventListener listener) + { + return (new StreamHandle() + { + private volatile boolean cancelled; + + + + @Override + public void cancel() + { + cancelled = true; + } + + + + @Override + public boolean isCancelled() + { + return (cancelled); + } + + + + @Override + public ModelResponse await(Duration timeout) + { + awaitUninterruptibly(dbGate); + return (new ModelResponse().withParts(List.of(ModelPart.text("late"))).withStopReason(ModelResponse.STOP_CANCELLED)); + } + }); + } + } + + + + /******************************************************************************* + ** The real store, except that chosen calls park until the test ends. + *******************************************************************************/ + private class StallingStore extends AssistantStore + { + /******************************************************************************* + ** Keep the configured persistence behavior. + *******************************************************************************/ + StallingStore(AssistantQBitConfig config) + { + super(config); + } + + + + /******************************************************************************* + ** Heartbeat renewals (and everything, once the whole database is stalled). + *******************************************************************************/ + @Override + public void updateFields(String table, QRecord record) throws QException + { + stallIf(stallAllCalls.get() || (stallHeartbeat.get() && AssistantTurn.TABLE_NAME.equals(table) && record.getValue("heartbeatAt") != null)); + super.updateFields(table, record); + } + + + + /******************************************************************************* + ** Reads stall too once the whole database is stalled. + *******************************************************************************/ + @Override + public AssistantTurn getTurn(String userId, Integer id) throws QException + { + stallIf(stallAllCalls.get()); + return (super.getTurn(userId, id)); + } + + + + /******************************************************************************* + ** The terminal row write. + *******************************************************************************/ + @Override + public boolean updateRunningTurn(String userId, QRecord changes) throws QException + { + stallIf(stallAllCalls.get()); + return (super.updateRunningTurn(userId, changes)); + } + + + + /******************************************************************************* + ** Inserts (usage rows, tool-call audit rows, events) stall too. + *******************************************************************************/ + @Override + public T insert(T entity) throws QException + { + stallIf(stallAllCalls.get()); + return (super.insert(entity)); + } + + + + /******************************************************************************* + ** Park when asked to. + *******************************************************************************/ + private void stallIf(boolean stall) + { + if(stall) + { + awaitUninterruptibly(dbGate); + } + } + } +} -- 2.54.0 From 485b89173ff19a969018aee76baf1f98b7ce3824 Mon Sep 17 00:00:00 2001 From: CI Date: Mon, 9 Nov 2026 16:12:47 +0000 Subject: [PATCH 5/8] Bump version to 0.4.0-SNAPSHOT [skip ci] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index afa6391..a94732b 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ - 0.3.0-SNAPSHOT + 0.4.0-SNAPSHOT 21 21 UTF-8 -- 2.54.0 From e634d6a192f6f16ce068ecfc54c6a2e75bf5e1e9 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 11 Nov 2026 07:06:27 -0600 Subject: [PATCH 6/8] feat(tools): analyze_records answers grouped questions through the data-analysis pivot A new read tool: group a table by one or two fields (a date field optionally by a calendar unit) and compute a count, sum, average, minimum, maximum or distinct count per group, with the usual filter. It is one call of the data-analysis qbit's pivot operation (POST /qbit/data-analysis/tables/{table}/pivot) through the same gateway and with the signed-in user's session, so table read, the table's analyze permission, field access and record locks decide as on the Analyses screens. The assistant only narrows it: data access READ on the table, fields whose values it may see, the policy's row cap. Offered only where the data-analysis extension is registered; no production dependency on that qbit. Tests run against the released data-analysis qbit served by the real UI API. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 6 + docs/DESIGN.md | 3 + qbit-assistant/pom.xml | 8 + .../qbits/assistant/engine/SystemPrompt.java | 1 + .../qbits/assistant/engine/ToolLabels.java | 1 + .../qbits/assistant/tools/AnalysisTools.java | 447 ++++++++++++++++++ .../qbits/assistant/tools/UiApiToolbox.java | 35 +- .../main/resources/help/qbit-assistant.yaml | 2 +- .../testing/AnalysisTestFixture.java | 133 ++++++ .../testing/AssistantTestFixture.java | 54 ++- .../assistant/testing/BaseAssistantTest.java | 13 +- .../assistant/tools/AnalysisToolsTest.java | 338 +++++++++++++ 12 files changed, 1023 insertions(+), 18 deletions(-) create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/AnalysisTools.java create mode 100644 qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AnalysisTestFixture.java create mode 100644 qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index aac4947..e379f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased - Grouped questions (analyze_records) + +- New read tool `analyze_records`: group a table by one or two fields (optionally a date field by year, quarter, month, week, day or hour) and compute one number per group: a count, or a sum, average, minimum, maximum or distinct count of a field, with the same filter the other tools take (relative dates included). "How many documents were printed in the last day, by format?" is one tool call, and the groups carry links that open their records. +- It is one call of the data-analysis qbit's pivot operation (`POST /qbit/data-analysis/tables/{table}/pivot`, UI API extension `data-analysis`), made through the same gateway with the user's own session, so table read, the table's analyze permission, field access and record locks decide exactly as on the Analyses screens; a refusal reaches the model with the same status and message. The assistant only narrows it: the table's data access must allow reads, group-by, measure and filter fields must be fields whose values the assistant may see, and the policy's row cap bounds the groups returned. +- The tool is offered only where the `data-analysis` extension is registered. The assistant gets no dependency on that qbit (test scope only). Relative dates in an analysis filter need qbit-data-analysis 0.3.0-RC.2 or later (earlier versions answer HTTP 500 for them). + ## Unreleased - A turn always ends - Fixed an answer that stayed on "Thinking..." forever (seen after the tool steps had finished) when the host database stopped answering, for example a connection pool with every connection checked out. The watchdog's heartbeat did its database read and write while holding the engine's monitor and before it checked the deadline, so one blocked heartbeat stopped the deadline from ever firing and stopped `finish` from running. The deadline is now checked first on every tick without touching the database; the heartbeat, cancel poll and kill-switch poll run on a separate bounded worker that is reported (WARN), interrupted and abandoned if it does not return within two heartbeat intervals; and the engine's monitor is no longer held across database I/O in the heartbeat or while waiting to finalize. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 0d9109c..8f70800 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -166,11 +166,14 @@ This matches core's design (`docs/UI-API-EXTENSIONS.md` §5, `UiApiInvoker`, QRu | `render_widget` | `POST /widget/{widgetName}` | | `get_record_history` | `GET /table/{tableName}/{primaryKey}`, then `POST /table/{tableName}/query` on `audit_table`, `audit` and `audit_detail` | | `create_record` / `update_record` / `delete_record` | `POST /table/{tableName}` / `PATCH /table/{tableName}/{primaryKey}` / `DELETE /table/{tableName}/{primaryKey}` | +| `analyze_records` | `POST /qbit/data-analysis/tables/{table}/pivot` (the data-analysis qbit's UI API extension; offered only where it is registered) | `navigate` and `search_help` stay on the core kernel because they read no records: - `navigate` only builds an `/app` link, and the page enforces permissions when it opens. - `search_help` reads help text from metadata. +`analyze_records` answers grouped questions (a count, sum, average, minimum, maximum or distinct count per group, for one or two group-by fields). The user needs what the Analyses screens need: read on the table and the table's analyze permission. The assistant adds its own limits first: data access READ on the table, and group-by, measure and filter fields whose values it may see (a hidden field is refused as an unknown one, before any request). It returns at most the policy's rows per call. + Running processes and reports through the assistant (`/processes/...`, `/reports/...`) is not offered yet. Process data access rows are stored so that the screens can already show and set them. ### Layers (each one can only narrow the one before) diff --git a/qbit-assistant/pom.xml b/qbit-assistant/pom.xml index bd5d84c..d1e6660 100644 --- a/qbit-assistant/pom.xml +++ b/qbit-assistant/pom.xml @@ -80,6 +80,14 @@ archunit-junit5 test + + + + com.kingsrook.qbits + qbit-data-analysis + 0.1.0-RC.7 + test + com.h2database diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/SystemPrompt.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/SystemPrompt.java index 6134792..f736495 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/SystemPrompt.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/SystemPrompt.java @@ -52,6 +52,7 @@ public class SystemPrompt - Data lives in tables (with fields, labels and possible values). Users see apps that group tables, processes, reports and dashboards (widgets). - Start with list_tables / describe_table to learn table and field names; never guess them. - query_records filters with criteria (field, operator, values) joined by AND / OR; count_records counts with the same filter; get_record reads one record; search_records finds records by text across tables. + - analyze_records (when it is offered) groups a table by one or two fields and returns one number per group (a count, or a sum, average, minimum, maximum or distinct count of a field) with the same filter. Use it for "how many per ..." questions instead of one count_records per group, then show the groups with present_results (rowKeys are the keys it returned). The user needs the table's analyze permission; a DENIED result means they do not have it. - get_record_history shows the audit trail (who changed which field, when). Audit values are shortened to 250 characters. - render_widget shows an existing dashboard widget; search_help searches the application's help text. - navigate opens a screen in the user's browser (a list, possibly filtered, or a record). diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java index eea1f75..e7b21a4 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ToolLabels.java @@ -32,6 +32,7 @@ public class ToolLabels Map.entry("describe_table", new String[] { "Reading table details", "Read table details" }), Map.entry("query_records", new String[] { "Querying records", "Queried records" }), Map.entry("count_records", new String[] { "Counting records", "Counted records" }), + Map.entry("analyze_records", new String[] { "Grouping and counting records", "Analyzed records" }), Map.entry("get_record", new String[] { "Opening a record", "Read a record" }), Map.entry("search_records", new String[] { "Searching", "Searched" }), Map.entry("get_record_history", new String[] { "Reading the change history", "Read the change history" }), diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/AnalysisTools.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/AnalysisTools.java new file mode 100644 index 0000000..b9a28fb --- /dev/null +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/AnalysisTools.java @@ -0,0 +1,447 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools; + + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.assistant.engine.PolicyService; +import com.kingsrook.qbits.assistant.persistence.TranscriptRedactor; +import com.kingsrook.qbits.assistant.spi.ModelToolDefinition; +import com.kingsrook.qbits.assistant.uiapi.UiApiCall; +import com.kingsrook.qbits.assistant.uiapi.UiApiResult; +import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome; +import com.kingsrook.qqq.backend.core.actions.ai.filter.AiFilterBuilder; +import com.kingsrook.qqq.backend.core.actions.ai.output.AiDataEnvelope; +import com.kingsrook.qqq.backend.core.actions.ai.output.AiLinkBuilder; +import com.kingsrook.qqq.backend.core.actions.ai.tools.AiJsonSchema; +import com.kingsrook.qqq.backend.core.actions.ai.tools.AiToolException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData; + + +/******************************************************************************* + ** The assistant's grouped questions ("how many, by what"): one call of the + ** data-analysis qbit's pivot operation, a spec'd `/qqq/v1` extension, made + ** through the same gateway and with the same session as every other data + ** tool. So the answer is held to what the Analyses screens enforce for that + ** user: table read, the table's analyze permission, field access and record + ** locks. The assistant only narrows it further: + ** - the table's data access must allow reads; + ** - group-by, measure and filter fields must be fields whose values the + ** assistant may see (AI exposure VALUES); + ** - the policy's row cap bounds the groups returned. + ** + ** The tool is offered only where the data-analysis extension is registered; + ** the assistant has no dependency on that qbit, only on its UI API contract + ** (extension `data-analysis`, `POST /qbit/data-analysis/tables/{table}/pivot`). + *******************************************************************************/ +public class AnalysisTools +{ + public static final String ANALYZE_RECORDS = "analyze_records"; + + public static final String EXTENSION_NAME = "data-analysis"; + public static final String PIVOT_PATH = "/qbit/data-analysis/tables/{table}/pivot"; + + public static final int MAX_GROUP_BY = 2; + public static final int MAX_GROUPS = 200; + public static final int DEFAULT_GROUPS = 50; + + public static final List OPERATORS = List.of("COUNT", "SUM", "AVG", "MIN", "MAX", "COUNT_DISTINCT"); + public static final List DATE_BUCKETS = List.of("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "DAY_OF_WEEK", "HOUR_OF_DAY"); + + private final UiApiToolbox toolbox; + private final QInstance qInstance; + + + + /******************************************************************************* + ** + *******************************************************************************/ + AnalysisTools(UiApiToolbox toolbox, QInstance qInstance) + { + this.toolbox = toolbox; + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** One grouped question, checked: the table, the pivot request body, and the + ** labels to show it with. + *******************************************************************************/ + record Plan(QTableMetaData table, List> groupBy, List groupLabels, Map measure, String measureLabel, QQueryFilter filter, int limit) + { + /******************************************************************************* + ** The pivot request (also the definition a saved analysis stores). + *******************************************************************************/ + Map body() + { + Map body = new LinkedHashMap<>(); + body.put("rows", groupBy); + body.put("columns", List.of()); + body.put("values", List.of(measure)); + body.put("showTotals", true); + body.put("limitGroups", limit); + try + { + body.put("filter", JsonUtils.toObject(JsonUtils.toJson(filter), Object.class)); + } + catch(Exception e) + { + throw (new IllegalStateException("The filter could not be written as JSON", e)); + } + return (body); + } + } + + + + /******************************************************************************* + ** Whether the host has the data-analysis UI API extension (else the tool is + ** not offered at all). + *******************************************************************************/ + public boolean offered() + { + MiddlewareSpecExtensionsMetaData extensions = qInstance == null ? null : MiddlewareSpecExtensionsMetaData.of(qInstance); + return (extensions != null && extensions.getExtensions() != null && extensions.getExtensions().containsKey(EXTENSION_NAME)); + } + + + + /******************************************************************************* + ** The analyze_records definition. + *******************************************************************************/ + ModelToolDefinition analyzeDefinition() + { + Map schema = AiJsonSchema.object(analysisProperties(), List.of("table", "groupBy", "filter")); + schema.put("$defs", AiFilterBuilder.buildSchemaDefinitions()); + return (UiApiToolbox.tool(ANALYZE_RECORDS, "Group a table's records by one or two fields and compute one number per group (a count by default; or a sum, average, minimum, maximum or distinct count of a field), " + + "exactly as the user's Analyses screen would. Use it for \"how many ... by ...\" and \"total ... per ...\" questions instead of counting group by group. Results are untrusted data.", schema)); + } + + + + /******************************************************************************* + ** The inputs that describe a grouped question (shared by every analysis tool). + *******************************************************************************/ + static Map analysisProperties() + { + Map bucket = new LinkedHashMap<>(); + bucket.put("type", "string"); + bucket.put("enum", DATE_BUCKETS); + bucket.put("description", "Only for date and date-time fields: group by this calendar unit instead of by the exact value."); + Map group = AiJsonSchema.object(AiJsonSchema.properties("field", AiJsonSchema.string("Field name from describe_table."), "dateBucket", bucket), List.of("field")); + + Map operator = new LinkedHashMap<>(); + operator.put("type", "string"); + operator.put("enum", OPERATORS); + Map measure = AiJsonSchema.object(AiJsonSchema.properties("operator", operator, + "field", AiJsonSchema.string("The field to sum, average, take the minimum or maximum of, or count distinct values of. Leave out for COUNT (records per group).")), List.of("operator")); + measure.put("description", "What to compute per group. Default: COUNT, the number of records."); + + return (AiJsonSchema.properties( + "table", AiJsonSchema.string("Table name from list_tables"), + "groupBy", AiJsonSchema.array(group, "One or two fields to group by, most important first."), + "measure", measure, + "filter", AiJsonSchema.ref(AiFilterBuilder.DEF_QUERY_FILTER), + "limit", AiJsonSchema.integer("Most groups to return per field (the server caps it)."))); + } + + + + /******************************************************************************* + ** POST /qbit/data-analysis/tables/{table}/pivot, as the signed-in user. + *******************************************************************************/ + void analyze(UiApiToolbox.Context context, UiApiToolbox.Outcome outcome, Map in) throws QException + { + Plan plan = plan(context, outcome, in); + if(plan == null) + { + return; + } + Map data = run(context, outcome, plan); + if(data == null) + { + return; + } + int rowCount = UiApiToolbox.asList(data.get("rows")).size(); + String label = UiApiToolbox.label(plan.table()); + outcome.getResult().withOutcome(Boolean.TRUE.equals(data.get("hasMore")) ? AiToolOutcome.TRUNCATED : AiToolOutcome.OK).withTableName(plan.table().getName()).withRowCount(rowCount) + .withSummary("Analyzed " + label + " · " + plan.measureLabel() + " by " + String.join(", ", plan.groupLabels()) + " · " + rowCount + " group" + (rowCount == 1 ? "" : "s")) + .withPermissionCheck("uiApi:POST " + PIVOT_PATH + ":" + outcome.getHttpStatus()) + .withModelContent(AiDataEnvelope.wrap(ANALYZE_RECORDS, plan.table().getName(), JsonUtils.toJson(data))); + } + + + + /******************************************************************************* + ** Check the question against the assistant's own limits before anything is + ** asked of the server: null (with a DENIED outcome) when the table may not + ** be read; INVALID_INPUT for a field the assistant may not use. + *******************************************************************************/ + Plan plan(UiApiToolbox.Context context, UiApiToolbox.Outcome outcome, Map in) throws QException + { + String tableName = UiApiToolbox.requiredString(in, "table"); + QTableMetaData table = toolbox.requireReadable(outcome, tableName); + if(table == null) + { + return (null); + } + Map usable = toolbox.filterable(table); + + List requested = UiApiToolbox.asList(in.get("groupBy")); + if(requested.isEmpty() || requested.size() > MAX_GROUP_BY) + { + throw (AiToolException.invalidInput("groupBy must name one or two fields.")); + } + List> groupBy = new ArrayList<>(); + List labels = new ArrayList<>(); + Set seen = new LinkedHashSet<>(); + for(Object item : requested) + { + Map group = UiApiToolbox.asMap(item); + String name = group.get("field") instanceof String s ? s.strip() : null; + QFieldMetaData field = usableField(table, usable, name); + if(!seen.add(name)) + { + throw (AiToolException.invalidInput("groupBy names " + name + " twice.")); + } + Map dimension = new LinkedHashMap<>(); + dimension.put("field", name); + String label = fieldLabel(field); + if(group.get("dateBucket") != null && StringUtils.hasContent(String.valueOf(group.get("dateBucket")))) + { + String bucket = String.valueOf(group.get("dateBucket")).strip().toUpperCase(Locale.ROOT); + if(!DATE_BUCKETS.contains(bucket)) + { + throw (AiToolException.invalidInput("dateBucket must be one of " + String.join(", ", DATE_BUCKETS) + ".")); + } + if(field.getType() != QFieldType.DATE && field.getType() != QFieldType.DATE_TIME) + { + throw (AiToolException.invalidInput(name + " is not a date field, so it cannot be grouped by " + bucket + ".")); + } + dimension.put("dateBucket", bucket); + label += " (" + bucket.toLowerCase(Locale.ROOT).replace('_', ' ') + ")"; + } + groupBy.add(dimension); + labels.add(label); + } + + Map given = UiApiToolbox.asMap(in.get("measure")); + String operator = given.get("operator") == null ? "COUNT" : String.valueOf(given.get("operator")).strip().toUpperCase(Locale.ROOT); + if(!OPERATORS.contains(operator)) + { + throw (AiToolException.invalidInput("measure.operator must be one of " + String.join(", ", OPERATORS) + ".")); + } + Map measure = new LinkedHashMap<>(); + String measureLabel = "Count"; + measure.put("operator", operator); + String measureField = given.get("field") instanceof String s && StringUtils.hasContent(s) ? s.strip() : null; + if(!"COUNT".equals(operator) || measureField != null) + { + QFieldMetaData field = usableField(table, usable, measureField); + measure.put("field", measureField); + measureLabel = switch(operator) + { + case "SUM" -> "Sum of "; + case "AVG" -> "Average "; + case "MIN" -> "Minimum "; + case "MAX" -> "Maximum "; + case "COUNT_DISTINCT" -> "Distinct "; + default -> "Count of "; + } + fieldLabel(field); + } + + int max = context.policy() == null ? PolicyService.ToolPolicy.DEFAULT_MAX_ROWS_PER_CALL : context.policy().maxRowsPerCall(); + int cap = Math.max(1, Math.min(max, MAX_GROUPS)); + int limit = in.get("limit") instanceof Number n ? Math.max(1, Math.min(cap, n.intValue())) : Math.min(DEFAULT_GROUPS, cap); + return (new Plan(table, groupBy, labels, measure, measureLabel, toolbox.buildFilter(table, in.get("filter")), limit)); + } + + + + /******************************************************************************* + ** Run the plan; the groups as the model and the confirmation card read + ** them, or null when the server refused (the outcome then says why). + *******************************************************************************/ + Map run(UiApiToolbox.Context context, UiApiToolbox.Outcome outcome, Plan plan) throws QException + { + String tableName = plan.table().getName(); + UiApiResult response = toolbox.call(context, outcome, new UiApiCall("POST", PIVOT_PATH).withPathParam("table", tableName).withJsonBody(plan.body())); + if(!response.isSuccess()) + { + return (null); + } + Map pivot = UiApiToolbox.asMap(response.json()); + List headers = UiApiToolbox.asList(pivot.get("rowHeaders")); + Set redactions = new LinkedHashSet<>(); + + Map> metrics = new LinkedHashMap<>(); + for(Object item : UiApiToolbox.asList(pivot.get("cells"))) + { + Map cell = UiApiToolbox.asMap(item); + List values = UiApiToolbox.asList(cell.get("values")); + if(cell.get("rowIndex") instanceof Number index && !values.isEmpty()) + { + metrics.putIfAbsent(index.intValue(), UiApiToolbox.asMap(values.get(0))); + } + } + + /////////////////////////////////////////////////////////////////////// + // two group-by fields can make more rows than the per-field limit: // + // the policy's row cap still bounds what reaches the model // + /////////////////////////////////////////////////////////////////////// + int max = context.policy() == null ? PolicyService.ToolPolicy.DEFAULT_MAX_ROWS_PER_CALL : context.policy().maxRowsPerCall(); + List> rows = new ArrayList<>(); + boolean cut = false; + for(int r = 0; r < headers.size(); r++) + { + if(rows.size() >= max) + { + cut = true; + break; + } + List keys = new ArrayList<>(); + List labels = new ArrayList<>(); + List header = UiApiToolbox.asList(headers.get(r)); + for(int d = 0; d < header.size(); d++) + { + Map group = UiApiToolbox.asMap(header.get(d)); + Object value = group.get("value"); + String shown = toolbox.sanitize(group.get("label") == null ? (value == null ? null : String.valueOf(value)) : String.valueOf(group.get("label")), context, redactions); + String field = d < plan.groupBy().size() ? String.valueOf(plan.groupBy().get(d).get("field")) : null; + if(TranscriptRedactor.current() != null && field != null && value != null) + { + TranscriptRedactor.current().observe(tableName, field, null, value, shown); + } + keys.add(value instanceof String text ? toolbox.sanitize(text, context, redactions) : value); + labels.add(shown == null ? "(none)" : shown); + } + Map metric = metrics.getOrDefault(r, Map.of()); + Map row = new LinkedHashMap<>(); + row.put("keys", keys); + row.put("labels", labels); + row.put("value", Boolean.TRUE.equals(metric.get("suppressed")) ? null : metric.get("value")); + row.put("display", metric.get("formatted")); + String href = drillHref(tableName, metric); + if(href != null) + { + row.put("href", href); + } + rows.add(row); + } + outcome.getResult().getRedactions().addAll(redactions); + + List totals = UiApiToolbox.asList(pivot.get("grandTotals")); + boolean hasMore = cut || Boolean.TRUE.equals(pivot.get("truncatedRows")); + List warnings = new ArrayList<>(); + for(Object warning : UiApiToolbox.asList(pivot.get("warnings"))) + { + warnings.add(UiApiToolbox.shortText(String.valueOf(warning))); + } + + Map data = new LinkedHashMap<>(); + data.put("table", tableName); + data.put("tableLabel", UiApiToolbox.label(plan.table())); + data.put("groupBy", plan.groupBy()); + data.put("groupLabels", plan.groupLabels()); + data.put("measure", plan.measure()); + data.put("measureLabel", plan.measureLabel()); + data.put("rowCount", rows.size()); + data.put("total", totals.isEmpty() ? null : UiApiToolbox.asMap(totals.get(0)).get("value")); + data.put("hasMore", hasMore); + if(hasMore) + { + data.put("note", "Not every group is shown: smaller groups are merged or left out. Say so, or narrow the filter."); + } + if(Boolean.TRUE.equals(pivot.get("estimate"))) + { + data.put("estimate", true); + } + if(!warnings.isEmpty()) + { + data.put("warnings", warnings); + } + data.put("openHref", AiLinkBuilder.filteredTableHref(tableName, plan.filter())); + data.put("rows", rows); + return (data); + } + + + + /******************************************************************************* + ** The link that opens the records behind one group: the server's own + ** drill-through filter on this table, or none. + *******************************************************************************/ + private static String drillHref(String tableName, Map metric) + { + Map drill = UiApiToolbox.asMap(metric.get("drill")); + if(!tableName.equals(drill.get("tableName")) || !(drill.get("filter") instanceof Map)) + { + return (null); + } + try + { + return (AiLinkBuilder.filteredTableHref(tableName, JsonUtils.toObject(JsonUtils.toJson(drill.get("filter")), QQueryFilter.class))); + } + catch(Exception e) + { + return (null); + } + } + + + + /******************************************************************************* + ** A field the assistant may group or measure by: one whose values it may + ** see. A hidden field is refused as an unknown one is. + *******************************************************************************/ + private static QFieldMetaData usableField(QTableMetaData table, Map usable, String name) throws AiToolException + { + QFieldMetaData field = name == null ? null : usable.get(name); + if(field == null) + { + throw (AiToolException.invalidInput(name == null ? "Each groupBy entry and any measure other than COUNT needs a field." : "Unknown field " + name + " on " + table.getName() + ". Use describe_table.")); + } + return (field); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String fieldLabel(QFieldMetaData field) + { + return (field.getLabel() == null ? field.getName() : field.getLabel()); + } + + +} diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java index 07d4290..719feac 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java @@ -93,6 +93,7 @@ import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials; ** | create_record | POST /table/{tableName} | ** | update_record | PATCH /table/{tableName}/{primaryKey} | ** | delete_record | DELETE /table/{tableName}/{primaryKey} | + ** | analyze_records | POST /qbit/data-analysis/tables/{table}/pivot (AnalysisTools; offered only when the data-analysis extension is registered) | *******************************************************************************/ public class UiApiToolbox { @@ -124,7 +125,8 @@ public class UiApiToolbox Map.entry(RECORD_HISTORY, new String[] {"POST", "/table/{tableName}/query"}), Map.entry(CREATE_RECORD, new String[] {"POST", "/table/{tableName}"}), Map.entry(UPDATE_RECORD, new String[] {"PATCH", "/table/{tableName}/{primaryKey}"}), - Map.entry(DELETE_RECORD, new String[] {"DELETE", "/table/{tableName}/{primaryKey}"})); + Map.entry(DELETE_RECORD, new String[] {"DELETE", "/table/{tableName}/{primaryKey}"}), + Map.entry(AnalysisTools.ANALYZE_RECORDS, new String[] {"POST", AnalysisTools.PIVOT_PATH})); public static final int DEFAULT_LIMIT = 20; @@ -133,6 +135,7 @@ public class UiApiToolbox private final AiExposurePolicy exposurePolicy; private final AiRedactor redactor; private final int maxCellChars; + private final AnalysisTools analysis; @@ -146,6 +149,7 @@ public class UiApiToolbox this.exposurePolicy = exposurePolicy; this.redactor = redactor; this.maxCellChars = maxCellChars; + this.analysis = new AnalysisTools(this, qInstance); } @@ -377,6 +381,10 @@ public class UiApiToolbox { byName.remove(RECORD_HISTORY); } + if(analysis.offered()) + { + byName.put(AnalysisTools.ANALYZE_RECORDS, analysis.analyzeDefinition()); + } return (new ArrayList<>(byName.values())); } @@ -485,6 +493,7 @@ public class UiApiToolbox case SEARCH_RECORDS -> searchRecords(context, outcome, in); case RENDER_WIDGET -> renderWidget(context, outcome, in); case RECORD_HISTORY -> recordHistory(context, outcome, in); + case AnalysisTools.ANALYZE_RECORDS -> analysis.analyze(context, outcome, in); case CREATE_RECORD, UPDATE_RECORD, DELETE_RECORD -> write(context, outcome, toolName, in); default -> throw (AiToolException.invalidInput("Unknown tool \"" + toolName + "\".")); } @@ -1334,7 +1343,7 @@ public class UiApiToolbox ** Make the call with the user's session; map a non-2xx status to the ** outcome the UI's status means, with the API's own message. *******************************************************************************/ - private UiApiResult call(Context context, Outcome outcome, UiApiCall call) throws QException + UiApiResult call(Context context, Outcome outcome, UiApiCall call) throws QException { context.prepare(call).withInvokedBy("assistant"); outcome.operation = call.operation(); @@ -1363,7 +1372,7 @@ public class UiApiToolbox ** The table, when the assistant may read it (else a DENIED outcome; the ** message does not reveal whether the table exists). *******************************************************************************/ - private QTableMetaData requireReadable(Outcome outcome, String tableName) + QTableMetaData requireReadable(Outcome outcome, String tableName) { QTableMetaData table = tableName == null ? null : qInstance.getTable(tableName); DataAccessService.Decision decision = dataAccess.forTable(tableName); @@ -1380,7 +1389,7 @@ public class UiApiToolbox /******************************************************************************* ** *******************************************************************************/ - private static void deny(AiToolResult result, String check, String message) + static void deny(AiToolResult result, String check, String message) { result.withOutcome(AiToolOutcome.DENIED).withSummary("Not allowed").withPermissionCheck(check).withModelContent(message); } @@ -1390,7 +1399,7 @@ public class UiApiToolbox /******************************************************************************* ** The model's filter, on fields whose values the assistant may see. *******************************************************************************/ - private QQueryFilter buildFilter(QTableMetaData table, Object filterInput) throws QException + QQueryFilter buildFilter(QTableMetaData table, Object filterInput) throws QException { QQueryFilter filter = AiFilterBuilder.build(filterInput, filterable(table)); return (filter == null ? new QQueryFilter() : filter); @@ -1411,7 +1420,7 @@ public class UiApiToolbox /******************************************************************************* ** Fields the model may filter and sort on (exposure VALUES). *******************************************************************************/ - private Map filterable(QTableMetaData table) + Map filterable(QTableMetaData table) { Map rs = new LinkedHashMap<>(); table.getFields().forEach((name, field) -> @@ -1474,7 +1483,7 @@ public class UiApiToolbox /******************************************************************************* ** Redact (when the policy says so) and truncate one value. *******************************************************************************/ - private String sanitize(String value, Context context, Set redactions) + String sanitize(String value, Context context, Set redactions) { if(value == null) { @@ -1536,7 +1545,7 @@ public class UiApiToolbox /******************************************************************************* ** *******************************************************************************/ - private static String label(QTableMetaData table) + static String label(QTableMetaData table) { return (table.getLabel() == null ? table.getName() : table.getLabel()); } @@ -1546,7 +1555,7 @@ public class UiApiToolbox /******************************************************************************* ** *******************************************************************************/ - private static String shortText(String text) + static String shortText(String text) { if(text == null) { @@ -1565,7 +1574,7 @@ public class UiApiToolbox /******************************************************************************* ** *******************************************************************************/ - private static String requiredString(Map in, String name) throws AiToolException + static String requiredString(Map in, String name) throws AiToolException { Object value = in.get(name); if(value == null || !StringUtils.hasContent(String.valueOf(value))) @@ -1581,7 +1590,7 @@ public class UiApiToolbox ** *******************************************************************************/ @SuppressWarnings("unchecked") - private static Map asMap(Object value) + static Map asMap(Object value) { return (value instanceof Map map ? (Map) map : Map.of()); } @@ -1592,7 +1601,7 @@ public class UiApiToolbox ** *******************************************************************************/ @SuppressWarnings("unchecked") - private static List asList(Object value) + static List asList(Object value) { return (value instanceof List list ? (List) list : List.of()); } @@ -1602,7 +1611,7 @@ public class UiApiToolbox /******************************************************************************* ** *******************************************************************************/ - private static ModelToolDefinition tool(String name, String description, Map schema) + static ModelToolDefinition tool(String name, String description, Map schema) { return (new ModelToolDefinition().withName(name).withDescription(description).withInputSchema(schema).withStrict(false)); } diff --git a/qbit-assistant/src/main/resources/help/qbit-assistant.yaml b/qbit-assistant/src/main/resources/help/qbit-assistant.yaml index ea0dad2..982a499 100644 --- a/qbit-assistant/src/main/resources/help/qbit-assistant.yaml +++ b/qbit-assistant/src/main/resources/help/qbit-assistant.yaml @@ -445,7 +445,7 @@ entries: - key: table:assistant_policy;field:allowedTools summary: Limits the assistant to the tools named here, separated by commas; leave empty to allow all of them. content: | - Examples are list_tables, query_records and count_records. Names must match the tools exactly. Check **{{label:assistant_tool_call}}** to see the names the assistant has used. + Examples are list_tables, query_records, count_records and analyze_records. Names must match the tools exactly. Check **{{label:assistant_tool_call}}** to see the names the assistant has used. - key: table:assistant_policy;field:redactPii summary: When on, or empty, keys, tokens, card numbers and social security numbers are hidden from the model's view of results. content: | diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AnalysisTestFixture.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AnalysisTestFixture.java new file mode 100644 index 0000000..ec65fc6 --- /dev/null +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AnalysisTestFixture.java @@ -0,0 +1,133 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.testing; + + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitProducer; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + + +/******************************************************************************* + ** The fixture app with the released data-analysis qbit and a print_job table + ** (format PDF or ZPL, a real date-time, a hidden note), for the analysis + ** tools. Users (AssistantTestFixture.permissionsFor): + ** - reporter-*: read and analyze print_job, and their own saved analyses; + ** - viewer-*: read print_job, without the analyze permission. + *******************************************************************************/ +public final class AnalysisTestFixture +{ + public static final String TABLE = "print_job"; + public static final String REPORTER = "reporter-1"; + public static final String VIEWER = "viewer-1"; + + public static final String NOTE_SENTINEL = "OPERATOR_NOTE_SENTINEL_5d1e"; + + public static final int PDF_LAST_DAY = 3; + public static final int ZPL_LAST_DAY = 5; + public static final int PDF_OLDER = 2; + + + + /******************************************************************************* + ** Utility class - no instances. + *******************************************************************************/ + private AnalysisTestFixture() + { + } + + + + /******************************************************************************* + ** Add print_job and the data-analysis qbit (memory backend, no scheduler). + *******************************************************************************/ + public static void customize(QInstance qInstance) throws Exception + { + customize(qInstance, new DataAnalysisQBitConfig()); + } + + + + /******************************************************************************* + ** The same, with a given data-analysis configuration (e.g. a scheduler). + *******************************************************************************/ + public static void customize(QInstance qInstance, DataAnalysisQBitConfig analysis) throws Exception + { + qInstance.addTable(AssistantTestFixture.table(TABLE, "Print Job", "title") + .withField(new QFieldMetaData("title", QFieldType.STRING).withLabel("Title")) + .withField(new QFieldMetaData("format", QFieldType.STRING).withLabel("Format")) + .withField(new QFieldMetaData("status", QFieldType.STRING).withLabel("Status")) + .withField(new QFieldMetaData("pages", QFieldType.INTEGER).withLabel("Pages")) + .withField(new QFieldMetaData("printedAt", QFieldType.DATE_TIME).withLabel("Printed At")) + .withField(new QFieldMetaData("operatorNote", QFieldType.STRING).withLabel("Operator Note").withIsHidden(true))); + new DataAnalysisQBitProducer().withQBitConfig(analysis.withBackendName(AssistantTestFixture.BACKEND)).produce(qInstance); + } + + + + /******************************************************************************* + ** Seed the print jobs: 3 PDF and 5 ZPL in the last day, 2 PDF ten days ago. + *******************************************************************************/ + public static void seed(QInstance qInstance) throws Exception + { + QContext.init(qInstance, AssistantTestFixture.sessionFor("seeder-0")); + try + { + Instant now = Instant.now(); + List jobs = new ArrayList<>(); + for(int i = 0; i < PDF_LAST_DAY; i++) + { + jobs.add(job("Packing slip " + i, "PDF", now.minus(1 + i, ChronoUnit.HOURS), 2)); + } + for(int i = 0; i < ZPL_LAST_DAY; i++) + { + jobs.add(job("Shipping label " + i, "ZPL", now.minus(2 + i, ChronoUnit.HOURS), 1)); + } + for(int i = 0; i < PDF_OLDER; i++) + { + jobs.add(job("Old invoice " + i, "PDF", now.minus(10, ChronoUnit.DAYS), 4)); + } + new InsertAction().execute(new InsertInput(TABLE).withRecords(jobs)); + } + finally + { + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QRecord job(String title, String format, Instant printedAt, int pages) + { + return (new QRecord().withValue("title", title).withValue("format", format).withValue("status", "SPOOLED").withValue("pages", pages) + .withValue("printedAt", printedAt).withValue("operatorNote", NOTE_SENTINEL)); + } + +} diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AssistantTestFixture.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AssistantTestFixture.java index 2f9cfe2..6627022 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AssistantTestFixture.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/AssistantTestFixture.java @@ -98,6 +98,17 @@ public class AssistantTestFixture ** set here), validate it, reset the memory store and seed the data. *******************************************************************************/ public static QInstance build(AssistantQBitConfig config) throws Exception + { + return (build(config, null)); + } + + + + /******************************************************************************* + ** The same, with more metadata added (and seeded by the caller afterwards) + ** before the QBit is produced and the instance validated: e.g. another qbit. + *******************************************************************************/ + public static QInstance build(AssistantQBitConfig config, InstanceCustomizer customizer) throws Exception { config.setRegisterShutdownHook(false); MemoryRecordStore.fullReset(); @@ -158,6 +169,11 @@ public class AssistantTestFixture .withChild(qInstance.getTable("media")) .withChild(qInstance.getTable("analytics_daily_stat"))); + if(customizer != null) + { + customizer.customize(qInstance); + } + config.setBackendName(BACKEND); new AssistantQBitProducer().withQBitConfig(config).produce(qInstance); @@ -170,9 +186,23 @@ public class AssistantTestFixture /******************************************************************************* - ** + ** More metadata for one test class's instance. *******************************************************************************/ - private static QTableMetaData table(String name, String label, String labelField) + @FunctionalInterface + public interface InstanceCustomizer + { + /******************************************************************************* + ** + *******************************************************************************/ + void customize(QInstance qInstance) throws Exception; + } + + + + /******************************************************************************* + ** A fixture table: memory backend, CRUD permissions, id and dates. + *******************************************************************************/ + public static QTableMetaData table(String name, String label, String labelField) { return (new QTableMetaData() .withName(name) @@ -301,6 +331,13 @@ public class AssistantTestFixture { session.withSecurityKeyValue(AssistantMetaDataBuilder.allAccessKeyName(AssistantQBitConfig.DEFAULT_OWNER_KEY_TYPE), true); } + if(userId.startsWith("reporter") || userId.startsWith("viewer")) + { + /////////////////////////////////////////////////////////////////////// + // the data-analysis qbit's owner key, as a host's sign-in sets it // + /////////////////////////////////////////////////////////////////////// + session.withSecurityKeyValue("data_analysis_user", userId); + } return (session); } @@ -312,6 +349,8 @@ public class AssistantTestFixture ** analyst-* assistant + read on analytics_daily_stat only ** limited-* assistant only (no table reads at all) ** admin-* assistant + assistant.admin + everything + ** reporter-* assistant + read and analyze on print_job + their own saved analyses (AnalysisTestFixture) + ** viewer-* assistant + read on print_job, without analyze ** seeder-* everything (fixture seeding) ** anything else: nothing *******************************************************************************/ @@ -319,10 +358,19 @@ public class AssistantTestFixture { Set permissions = new HashSet<>(); boolean all = userId.startsWith("seeder") || userId.startsWith("admin"); - if(all || userId.startsWith("editor") || userId.startsWith("analyst") || userId.startsWith("limited")) + if(all || userId.startsWith("editor") || userId.startsWith("analyst") || userId.startsWith("limited") || userId.startsWith("reporter") || userId.startsWith("viewer")) { permissions.add(AssistantQBitConfig.ACCESS_PERMISSION); } + if(userId.startsWith("reporter") || userId.startsWith("viewer")) + { + permissions.add("print_job.read"); + } + if(userId.startsWith("reporter")) + { + permissions.add("print_job.analyze"); + permissions.addAll(List.of("data_analysis.saved.read", "data_analysis.saved.insert", "data_analysis.saved.edit", "data_analysis.saved.delete")); + } if(all) { permissions.add(AssistantQBitConfig.ADMIN_PERMISSION); diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/BaseAssistantTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/BaseAssistantTest.java index ab8a84b..75e4508 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/BaseAssistantTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/BaseAssistantTest.java @@ -53,12 +53,23 @@ public abstract class BaseAssistantTest .withProviderName(ScriptedModelProvider.DEFAULT_NAME) .withModelProvider(provider); customizeConfig(config); - qInstance = AssistantTestFixture.build(config); + qInstance = AssistantTestFixture.build(config, instanceCustomizer()); runner = new TurnRunner(qInstance, config); } + /******************************************************************************* + ** Hook to add metadata (e.g. another qbit) to the fixture instance; null + ** for the plain fixture. + *******************************************************************************/ + protected AssistantTestFixture.InstanceCustomizer instanceCustomizer() + { + return (null); + } + + + /******************************************************************************* ** Hook to change the config before production. *******************************************************************************/ diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java new file mode 100644 index 0000000..35211f7 --- /dev/null +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java @@ -0,0 +1,338 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.tools; + + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.assistant.access.DataAccessMode; +import com.kingsrook.qbits.assistant.engine.PolicyService; +import com.kingsrook.qbits.assistant.engine.TurnEvent; +import com.kingsrook.qbits.assistant.model.AssistantDataAccess; +import com.kingsrook.qbits.assistant.model.AssistantToolCall; +import com.kingsrook.qbits.assistant.testing.AnalysisTestFixture; +import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; +import com.kingsrook.qbits.assistant.testing.BaseAssistantTest; +import com.kingsrook.qbits.assistant.testing.EvalSupport; +import com.kingsrook.qbits.assistant.testing.TurnRunner; +import com.kingsrook.qbits.assistant.testing.UiApiTestServer; +import com.kingsrook.qbits.assistant.uiapi.UiApiResult; +import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** analyze_records against the released data-analysis qbit, served by the real + ** UI API: the grouped numbers are right, relative dates filter, and the + ** assistant is refused exactly where the Analyses screen is (the same status + ** and message), and only ever narrower. + *******************************************************************************/ +class AnalysisToolsTest extends BaseAssistantTest +{ + private final HttpClient client = HttpClient.newHttpClient(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected AssistantTestFixture.InstanceCustomizer instanceCustomizer() + { + return (AnalysisTestFixture::customize); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void seed() throws Exception + { + AnalysisTestFixture.seed(qInstance); + } + + + + /******************************************************************************* + ** "How many documents were printed, by format?": one pivot call, the right + ** counts, a total, and a link that opens each group's records. + *******************************************************************************/ + @Test + void countsGroupedByOneField() throws Exception + { + UiApiToolbox.Outcome outcome = tool(AnalysisTestFixture.REPORTER, null, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND"))); + assertThat(outcome.getResult().getOutcome()).as(outcome.getResult().getModelContent()).isEqualTo(AiToolOutcome.OK); + assertThat(outcome.getOperation()).isEqualTo("POST /qbit/data-analysis/tables/{table}/pivot"); + assertThat(outcome.getHttpStatus()).isEqualTo(200); + + Map data = EvalSupport.unwrap(outcome.getResult().getModelContent()); + assertThat(counts(data)).containsOnly(Map.entry("PDF", AnalysisTestFixture.PDF_LAST_DAY + AnalysisTestFixture.PDF_OLDER), Map.entry("ZPL", AnalysisTestFixture.ZPL_LAST_DAY)); + assertThat(((Number) data.get("total")).intValue()).isEqualTo(AnalysisTestFixture.PDF_LAST_DAY + AnalysisTestFixture.PDF_OLDER + AnalysisTestFixture.ZPL_LAST_DAY); + assertThat(data).containsEntry("measureLabel", "Count").containsEntry("hasMore", false); + assertThat(String.valueOf(data.get("openHref"))).startsWith("/app/print_job"); + assertThat(rows(data)).allSatisfy(row -> assertThat(String.valueOf(row.get("href"))).startsWith("/app/print_job?filter=")); + assertThat(outcome.getResult().getSummary()).contains("Print Job").contains("Count by Format").contains("2 groups"); + assertThat(outcome.getResult().getModelContent()).doesNotContain(AnalysisTestFixture.NOTE_SENTINEL); + } + + + + /******************************************************************************* + ** "...from the last day": the relative date expression is evaluated by the + ** server on the real date-time field, so the older jobs are left out. + *******************************************************************************/ + @Test + void relativeDateFilterNarrowsTheGroups() throws Exception + { + UiApiToolbox.Outcome outcome = tool(AnalysisTestFixture.REPORTER, null, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), + "filter", filter("AND", "printedAt", "GREATER_THAN_OR_EQUALS", List.of("NOW-1 DAYS")))); + assertThat(outcome.getResult().getOutcome()).as(outcome.getResult().getModelContent()).isEqualTo(AiToolOutcome.OK); + Map data = EvalSupport.unwrap(outcome.getResult().getModelContent()); + assertThat(counts(data)).containsOnly(Map.entry("PDF", AnalysisTestFixture.PDF_LAST_DAY), Map.entry("ZPL", AnalysisTestFixture.ZPL_LAST_DAY)); + } + + + + /******************************************************************************* + ** A sum per group, and two group-by fields (the second a date bucket). + *******************************************************************************/ + @Test + void sumsAndTwoGroupByFields() throws Exception + { + UiApiToolbox.Outcome sum = tool(AnalysisTestFixture.REPORTER, null, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), + "measure", Map.of("operator", "SUM", "field", "pages"), "filter", filter("AND"))); + assertThat(sum.getResult().getOutcome()).as(sum.getResult().getModelContent()).isEqualTo(AiToolOutcome.OK); + Map sums = EvalSupport.unwrap(sum.getResult().getModelContent()); + assertThat(counts(sums)).containsOnly(Map.entry("PDF", AnalysisTestFixture.PDF_LAST_DAY * 2 + AnalysisTestFixture.PDF_OLDER * 4), Map.entry("ZPL", AnalysisTestFixture.ZPL_LAST_DAY)); + assertThat(sums).containsEntry("measureLabel", "Sum of Pages"); + + UiApiToolbox.Outcome two = tool(AnalysisTestFixture.REPORTER, null, input("table", AnalysisTestFixture.TABLE, + "groupBy", List.of(Map.of("field", "format"), Map.of("field", "printedAt", "dateBucket", "YEAR")), "filter", filter("AND"))); + assertThat(two.getResult().getOutcome()).as(two.getResult().getModelContent()).isEqualTo(AiToolOutcome.OK); + Map data = EvalSupport.unwrap(two.getResult().getModelContent()); + assertThat(rows(data)).isNotEmpty().allSatisfy(row -> assertThat((List) row.get("keys")).hasSize(2)); + assertThat(rows(data).stream().mapToInt(row -> ((Number) row.get("value")).intValue()).sum()) + .isEqualTo(AnalysisTestFixture.PDF_LAST_DAY + AnalysisTestFixture.PDF_OLDER + AnalysisTestFixture.ZPL_LAST_DAY); + } + + + + /******************************************************************************* + ** No analyze permission: the Analyses screen's own request is refused, and + ** the assistant gets the identical status and message. A user who cannot + ** read the table at all gets the UI's denial as well. + *******************************************************************************/ + @Test + void theSameDenialAsTheAnalysesScreen() throws Exception + { + String body = "{\"rows\":[{\"field\":\"format\"}],\"values\":[{\"operator\":\"COUNT\"}]}"; + for(String user : List.of(AnalysisTestFixture.VIEWER, AssistantTestFixture.EDITOR)) + { + HttpResponse ui = ui("/qbit/data-analysis/tables/print_job/pivot", user, body); + assertThat(ui.statusCode()).as(user + " via the UI").isBetween(400, 499); + + UiApiToolbox.Outcome outcome = tool(user, null, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND"))); + assertThat(outcome.getHttpStatus()).as(user + " via the assistant").isEqualTo(ui.statusCode()); + assertThat(outcome.getResult().getOutcome()).isIn(AiToolOutcome.DENIED, AiToolOutcome.NOT_FOUND); + String message = new UiApiResult(ui.statusCode(), ui.body()).errorMessage(); + if(message != null) + { + assertThat(outcome.getResult().getModelContent()).contains(message); + } + assertThat(outcome.getResult().getModelContent()).doesNotContain("PDF").doesNotContain("ZPL"); + } + } + + + + /******************************************************************************* + ** The assistant only narrows: a hidden field cannot be grouped, measured or + ** filtered on (it is refused as an unknown field, before any request); bad + ** shapes are refused; the policy's allowed tools and row cap hold; data + ** access NONE hides the table. + *******************************************************************************/ + @Test + void theAssistantOnlyNarrows() throws Exception + { + for(Map hidden : List.of( + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "operatorNote")), "filter", filter("AND")), + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "measure", Map.of("operator", "COUNT_DISTINCT", "field", "operatorNote"), "filter", filter("AND")), + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND", "operatorNote", "EQUALS", List.of(AnalysisTestFixture.NOTE_SENTINEL))))) + { + UiApiToolbox.Outcome outcome = tool(AnalysisTestFixture.REPORTER, null, hidden); + assertThat(outcome.getResult().getOutcome()).isEqualTo(AiToolOutcome.INVALID_INPUT); + assertThat(outcome.getHttpStatus()).as("no request is made").isNull(); + assertThat(outcome.getResult().getModelContent()).doesNotContain(AnalysisTestFixture.NOTE_SENTINEL); + } + + for(Map invalid : List.of( + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(), "filter", filter("AND")), + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format"), Map.of("field", "status"), Map.of("field", "title")), "filter", filter("AND")), + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format", "dateBucket", "DAY")), "filter", filter("AND")), + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "measure", Map.of("operator", "SUM"), "filter", filter("AND")), + input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "measure", Map.of("operator", "MEDIAN", "field", "pages"), "filter", filter("AND")))) + { + UiApiToolbox.Outcome outcome = tool(AnalysisTestFixture.REPORTER, null, invalid); + assertThat(outcome.getResult().getOutcome()).as(String.valueOf(invalid)).isEqualTo(AiToolOutcome.INVALID_INPUT); + assertThat(outcome.getHttpStatus()).as("no request is made").isNull(); + } + + PolicyService.ToolPolicy oneRow = new PolicyService.ToolPolicy(true, Set.of(), 1, true); + UiApiToolbox.Outcome capped = tool(AnalysisTestFixture.REPORTER, oneRow, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND"), "limit", 100)); + assertThat(capped.getResult().getOutcome()).as(capped.getResult().getModelContent()).isEqualTo(AiToolOutcome.TRUNCATED); + assertThat(capped.getResult().getRowCount()).isEqualTo(1); + assertThat(capped.getResult().getModelContent()).contains("\"hasMore\":true"); + + PolicyService.ToolPolicy onlyCount = new PolicyService.ToolPolicy(true, Set.of(UiApiToolbox.COUNT_RECORDS), 50, true); + UiApiToolbox.Outcome refused = tool(AnalysisTestFixture.REPORTER, onlyCount, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND"))); + assertThat(refused.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED); + assertThat(refused.getHttpStatus()).isNull(); + + setAccess(AnalysisTestFixture.TABLE, DataAccessMode.NONE); + UiApiToolbox.Outcome none = tool(AnalysisTestFixture.REPORTER, null, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND"))); + assertThat(none.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED); + assertThat(none.getHttpStatus()).as("no request is made").isNull(); + } + + + + /******************************************************************************* + ** The tool is offered where the data-analysis extension is registered, and a + ** whole turn uses it: the model's request lists it, the call is audited with + ** its operation and status, and the faithful model answers from the groups. + *******************************************************************************/ + @Test + void aTurnAnswersByFormat() throws Exception + { + assertThat(config.getRuntime().getContextBuilder().toolNames()).contains(AnalysisTools.ANALYZE_RECORDS); + + provider.thenToolUse(AnalysisTools.ANALYZE_RECORDS, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), + "filter", filter("AND", "printedAt", "GREATER_THAN_OR_EQUALS", List.of("NOW-1 DAYS")))) + .thenRespond(EvalSupport.answer(request -> + { + StringBuilder text = new StringBuilder("Printed in the last day: "); + for(Map row : rows(EvalSupport.firstResult(request))) + { + text.append(((List) row.get("labels")).get(0)).append(" ").append(row.get("value")).append("; "); + } + return (text.toString()); + })); + + TurnRunner.Outcome outcome = runner.run(AnalysisTestFixture.REPORTER, "How many documents were printed in the last day, by format?"); + assertThat(outcome.end()).containsEntry("status", "COMPLETE"); + assertThat(provider.getRequests().get(0).getTools()).extracting(tool -> tool.getName()).contains(AnalysisTools.ANALYZE_RECORDS); + assertThat(outcome.text()).contains("PDF " + AnalysisTestFixture.PDF_LAST_DAY).contains("ZPL " + AnalysisTestFixture.ZPL_LAST_DAY); + + AssistantToolCall call = outcome.toolCalls().get(0); + assertThat(call.getToolName()).isEqualTo(AnalysisTools.ANALYZE_RECORDS); + assertThat(call.getOutcome()).isEqualTo("OK"); + assertThat(call.getUiApiOperation()).isEqualTo("POST /qbit/data-analysis/tables/{table}/pivot"); + assertThat(call.getHttpStatus()).isEqualTo(200); + assertThat(call.getTableName()).isEqualTo(AnalysisTestFixture.TABLE); + assertThat(outcome.events(TurnEvent.TOOL_RESULT).get(0)).containsEntry("name", AnalysisTools.ANALYZE_RECORDS).containsEntry("outcome", "OK"); + } + + + + /******************************************************************************* + ** The Analyses screen's own request (the user's session cookie). + *******************************************************************************/ + private HttpResponse ui(String path, String user, String body) throws Exception + { + String base = UiApiTestServer.serve(qInstance, config); + return (client.send(HttpRequest.newBuilder(URI.create(base + "/qqq/v1" + path)).header("Cookie", "sessionId=" + user).header("Content-Type", "application/json") + .timeout(Duration.ofSeconds(30)).POST(HttpRequest.BodyPublishers.ofString(body)).build(), HttpResponse.BodyHandlers.ofString())); + } + + + + /******************************************************************************* + ** The assistant's tool call, as the turn engine makes it. + *******************************************************************************/ + private UiApiToolbox.Outcome tool(String user, PolicyService.ToolPolicy policy, Map in) + { + UiApiTestServer.serve(qInstance, config); + QContext.init(qInstance, AssistantTestFixture.ownerSession(user, config)); + try + { + return (config.getRuntime().getToolbox().execute(new UiApiToolbox.Context(user, null, config.getRuntime().getUiApiGateway(), policy, null), "call-1", AnalysisTools.ANALYZE_RECORDS, in)); + } + finally + { + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setAccess(String tableName, DataAccessMode mode) throws QException + { + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + try + { + config.getRuntime().getStore().insert(new AssistantDataAccess().withObjectType(AssistantDataAccess.TYPE_TABLE).withObjectName(tableName).withMode(mode.name())); + config.getRuntime().getDataAccessService().invalidate(); + } + finally + { + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static List> rows(Map data) + { + return ((List>) data.get("rows")); + } + + + + /******************************************************************************* + ** Group label to value, for one group-by field. + *******************************************************************************/ + private static Map counts(Map data) + { + Map counts = new LinkedHashMap<>(); + for(Map row : rows(data)) + { + counts.put(String.valueOf(((List) row.get("labels")).get(0)), ((Number) row.get("value")).intValue()); + } + return (counts); + } + +} -- 2.54.0 From ad33fbcf4f2e22a544c7d6df34242b25096394d6 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 11 Nov 2026 07:11:33 -0600 Subject: [PATCH 7/8] fix(tools): an extension tool's operation is not a core operation OPERATIONS stays the list of core /qqq/v1 operations (every one is invocable on any instance); analyze_records moves to EXTENSION_OPERATIONS, offered and invocable only where the data-analysis extension is registered. A test pins that it is neither offered nor invocable without the extension. Co-Authored-By: Claude Fable 5.2 --- .../qbits/assistant/tools/UiApiToolbox.java | 10 ++++++-- .../assistant/tools/AnalysisToolsTest.java | 4 ++++ .../tools/UiApiToolsSecurityTest.java | 23 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java index 719feac..a1aabda 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java @@ -125,7 +125,13 @@ public class UiApiToolbox Map.entry(RECORD_HISTORY, new String[] {"POST", "/table/{tableName}/query"}), Map.entry(CREATE_RECORD, new String[] {"POST", "/table/{tableName}"}), Map.entry(UPDATE_RECORD, new String[] {"PATCH", "/table/{tableName}/{primaryKey}"}), - Map.entry(DELETE_RECORD, new String[] {"DELETE", "/table/{tableName}/{primaryKey}"}), + Map.entry(DELETE_RECORD, new String[] {"DELETE", "/table/{tableName}/{primaryKey}"})); + + ///////////////////////////////////////////////////////////////////////// + // tools whose operation belongs to an optional UI API extension: they // + // are offered, and invocable, only where that extension is registered // + ///////////////////////////////////////////////////////////////////////// + public static final Map EXTENSION_OPERATIONS = Map.ofEntries( Map.entry(AnalysisTools.ANALYZE_RECORDS, new String[] {"POST", AnalysisTools.PIVOT_PATH})); public static final int DEFAULT_LIMIT = 20; @@ -458,7 +464,7 @@ public class UiApiToolbox long started = System.nanoTime(); try { - String[] operation = OPERATIONS.get(toolName); + String[] operation = OPERATIONS.containsKey(toolName) ? OPERATIONS.get(toolName) : EXTENSION_OPERATIONS.get(toolName); if(operation == null) { throw (AiToolException.invalidInput("Unknown tool \"" + toolName + "\". Use only the tools provided.")); diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java index 35211f7..fe938ad 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/AnalysisToolsTest.java @@ -232,6 +232,10 @@ class AnalysisToolsTest extends BaseAssistantTest void aTurnAnswersByFormat() throws Exception { assertThat(config.getRuntime().getContextBuilder().toolNames()).contains(AnalysisTools.ANALYZE_RECORDS); + for(String[] operation : UiApiToolbox.EXTENSION_OPERATIONS.values()) + { + assertThat(config.getRuntime().getUiApiGateway().isInvocable(operation[0], operation[1])).as(operation[0] + " " + operation[1]).isTrue(); + } provider.thenToolUse(AnalysisTools.ANALYZE_RECORDS, input("table", AnalysisTestFixture.TABLE, "groupBy", List.of(Map.of("field", "format")), "filter", filter("AND", "printedAt", "GREATER_THAN_OR_EQUALS", List.of("NOW-1 DAYS")))) diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/UiApiToolsSecurityTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/UiApiToolsSecurityTest.java index 1975ba5..4b21ee5 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/UiApiToolsSecurityTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/tools/UiApiToolsSecurityTest.java @@ -406,6 +406,29 @@ class UiApiToolsSecurityTest extends BaseAssistantTest + /******************************************************************************* + ** A tool of an optional UI API extension (analyze_records: the data-analysis + ** pivot) is neither offered nor invocable where the extension is not + ** registered: the model never sees it, and a call is refused before any request. + *******************************************************************************/ + @Test + void extensionToolsNeedTheirExtension() + { + UiApiTestServer.serve(qInstance, config); + assertThat(config.getRuntime().getContextBuilder().toolNames()).doesNotContainAnyElementsOf(UiApiToolbox.EXTENSION_OPERATIONS.keySet()); + assertThat(UiApiToolbox.OPERATIONS.keySet()).doesNotContainAnyElementsOf(UiApiToolbox.EXTENSION_OPERATIONS.keySet()); + for(String[] operation : UiApiToolbox.EXTENSION_OPERATIONS.values()) + { + assertThat(config.getRuntime().getUiApiGateway().isInvocable(operation[0], operation[1])).as(operation[0] + " " + operation[1]).isFalse(); + } + UiApiToolbox.Outcome outcome = tool(AssistantTestFixture.ADMIN, null, AnalysisTools.ANALYZE_RECORDS, input("table", "blog_post", "groupBy", List.of(Map.of("field", "status")), "filter", filter("AND"))); + assertThat(outcome.getResult().getOutcome()).isEqualTo(AiToolOutcome.DENIED); + assertThat(outcome.getHttpStatus()).as("no request is made").isNull(); + assertThat(outcome.getResult().getModelContent()).contains("not a spec'd UI API operation"); + } + + + /******************************************************************************* ** The policy narrows the tools, and caps rows. *******************************************************************************/ -- 2.54.0 From 10d80f6bcfa843c2d77cc48fd0efcc15bb697f31 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 11 Nov 2026 07:27:31 -0600 Subject: [PATCH 8/8] test(tools): run the analysis tests against data-analysis 0.3.0-RC.2 (relative dates in filters) Co-Authored-By: Claude Fable 5.2 --- qbit-assistant/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qbit-assistant/pom.xml b/qbit-assistant/pom.xml index d1e6660..b9ccc28 100644 --- a/qbit-assistant/pom.xml +++ b/qbit-assistant/pom.xml @@ -81,11 +81,11 @@ test - + com.kingsrook.qbits qbit-data-analysis - 0.1.0-RC.7 + 0.3.0-RC.2 test -- 2.54.0