fix(assistant): store the transcript before message.end so the panel shows the answer

The panel reloads the conversation the moment the stream ends. The
redacted transcript (message parts, pending actions, stored events) was
flushed only after message.end was sent, so that reload read the "[]"
placeholders and showed the newest question and answer as empty bubbles
until the panel was closed and reopened. Persist and flush first, then
emit message.end.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.circleci/config.yml

@@ -1,60 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.15
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.15
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,15 @@
+profile: qbit
+orb_version: 0.7.15
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## Unreleased - Answer visible at the end of the stream
+
+- The turn's redacted transcript (the question and answer parts, pending actions, stored events) is now written before `message.end` is sent. The panel reloads the conversation the moment the stream ends, and used to read the empty placeholders (`[]`) the transcript redactor stores until its flush, so the newest question and answer showed as empty bubbles until the panel was closed and reopened.
+
+## Unreleased - Optional provider health
+
+- An enabled provider that is not the default and not the only route for any configured routing (an enabled profile's simple/complex model, a budget downgrade model or a knowledge source's embedding model, with no fallback on another usable provider) is optional: when it is unreachable, its `assistant.provider.<id>` and `assistant.model.<id>` checks report DEGRADED with a message naming the provider requests use instead, so one unfinished local server no longer turns the whole `assistant` component DOWN. The default provider and sole routes still report DOWN. A failed provider probe still carries `reachable=false`, so the circuit breaker keeps refusing the provider.
+- New `assistant_provider.setup_state` (Setup state: Active / Setting up, empty counts as Active). A provider that is Setting up is never routed to (default, profile models, fallbacks, budget downgrades, embeddings) and gets no provider or model health checks; Test connection and Discover models still work. Apply the Provider setup state DDL (UPGRADE.md).
+
+## Unreleased - Fail soft (UI API extension 1.5.0)
+
+- Every message part, tool render and presentation is validated before it is stored or streamed, and again when stored parts and events are read: all render types (records, record, count, widget, history, pending_action, bulk_load_dry_run) and presentation blocks always carry their lists. A render that cannot be repaired is replaced by an error notice ("This result could not be displayed."); unknown part and render types pass through. Problems are logged with the type and a fixed code only.
+- Tool calls have their own limit (`toolCallTimeoutSeconds`, default 60) besides the turn's (`maxTurnSeconds`, default 300). A timed-out or failing tool gets an ERROR (`is_error`) result for the model, an ERROR tool part and an error notice for the user; internal details go only to the log.
+- Every turn ends: terminal events are emitted independently, so a failing emit never skips `message.end` or the terminal turn row. Conversation detail repairs assistant messages left `STREAMING` (an orphaned run is reaped; a run that already ended gives its status), and the retention process finalizes runs still `RUNNING` past the hard stream limit (e.g. after a pod restart).
+- `POST /qqq/v1/assistant/client-errors` (extension 1.5.0): the panel reports errors it hit (`scope`, `message`, optional stacks, route, ids, part/render type, app version, `occurredAt`). Answers `202 {"accepted": true}`; over `clientErrorsPerUserPerMinute` (default 30) it answers `202 {"accepted": false}` and drops the report. Fields are bounded, message and stacks scrubbed (secrets, e-mails, long numbers, tokens, query strings), ids kept only when they are the user's own. Rows go to the new owner-locked `assistant_client_error` table (Assistant app nav, retention after `toolResultRetentionDays`), and the `assistant.client-errors` health check turns degraded at `clientErrorWarnPerHour` (default 20) reports in an hour. Apply the Fail soft DDL (UPGRADE.md).
+
+## Unreleased - Empty renders and stream context (UI API extension 1.4.0)
+
+- Tool result renders always carry their lists: a `records` render for a query that matched nothing now has `"rows": []` and `"columns": []` (QQQ's JSON omitted empty lists, and Next's Assistant crashed the page reading `rows.length`). Parts and events stored before the fix are completed when read. Extension 1.4.0 documents the shape as `AssistantRecordsRenderV1`.
+- Streamed text arrives on the provider's own (virtual) thread; the turn's progress updates now run in the turn's QContext, so job progress no longer fails with "Could not flush job progress" (NPE on the missing instance).
+
+## Unreleased - UI API 1.6.0
+
+- UI API extension assistant 1.3.0 (requires QQQ 4.1.0-RC.15, UI API 1.6.0): operationIds, standard error responses with BasicErrorResponseV1, extension tags and trace IDs. Preserve HTTP statuses and assistant-specific error codes.
+- Retain AssistantErrorV1 as a deprecated component for compatibility. Remove dangling `null` schema references from the two bodyless 204 responses (delete conversation and feedback), with an empty-content workaround for core schema traversal. Success payloads are unchanged; the contract test retains the release baseline with explicit exceptions for the error schema migration and these documentation corrections.
+
 ## Unreleased - Central health
 
 ### Added

UPGRADE.md

@@ -1,5 +1,31 @@
 # Upgrading qbit-assistant
 
+## Provider setup state (Unreleased)
+
+Apply the **Provider setup state** block at the end of `docs/schema-postgres.sql`
+**before** deploying: the provider table now reads `setup_state`, so the new jar
+fails on `assistant_provider` without it. It adds a nullable `VARCHAR(16)` with
+default `'ACTIVE'` and a check for `ACTIVE` / `SETTING_UP`; existing rows behave as
+before. It is `IF NOT EXISTS` and safe to re-run.
+
+To stop an unfinished provider from being routed to or health-checked, edit it in
+Assistant > Assistant Provider (Provider section) and set **Setup state** to **Setting up** (SQL:
+`UPDATE assistant_provider SET setup_state = 'SETTING_UP' WHERE id = <id>;`).
+Enabled providers that are neither the default nor the only route for a configured
+model now report DEGRADED, not DOWN, when unreachable. No configuration change.
+
+## Fail soft (Unreleased, UI API extension assistant 1.5.0)
+
+Apply the **Fail soft** block at the end of `docs/schema-postgres.sql` before
+deploying. It creates `assistant_client_error` (errors the panel reports through
+`POST /qqq/v1/assistant/client-errors`) with its indexes and `SET NULL` foreign keys
+to the conversation, message and turn. It is `IF NOT EXISTS` and safe to re-run.
+
+New configuration, all optional: `toolCallTimeoutSeconds` (default 60),
+`clientErrorsPerUserPerMinute` (default 30) and `clientErrorWarnPerHour` (default
+20, when the `assistant.client-errors` health check turns degraded). The new table
+is in the Assistant app's nav for administrators.
+
 ## Transcript redaction (Unreleased)
 
 Apply the transcript-redaction column widening block in `docs/schema-postgres.sql`

docs/schema-postgres.sql

@@ -713,3 +713,40 @@ ALTER TABLE assistant_data_audit ALTER COLUMN reason SET DATA TYPE TEXT;
 ALTER TABLE assistant_budget_override ALTER COLUMN reason SET DATA TYPE TEXT;
 ALTER TABLE assistant_scheduled_run ALTER COLUMN summary SET DATA TYPE TEXT;
 ALTER TABLE assistant_scheduled_run ALTER COLUMN error SET DATA TYPE TEXT;
+
+-- Fail soft (UI API extension assistant 1.5.0): errors the panel reports through
+-- POST /qqq/v1/assistant/client-errors. Owner-locked like the other user rows; the
+-- message and stacks are scrubbed and bounded before they are stored. The ids are
+-- kept only when they are the reporting user's own rows, and SET NULL when those go.
+-- Retention deletes rows older than toolResultRetentionDays. Safe to re-run.
+CREATE TABLE IF NOT EXISTS assistant_client_error
+(
+   id SERIAL PRIMARY KEY,
+   user_id VARCHAR(255),
+   scope VARCHAR(64),
+   message TEXT,
+   error_name VARCHAR(100),
+   stack TEXT,
+   component_stack TEXT,
+   request_id VARCHAR(128),
+   route VARCHAR(300),
+   conversation_id INTEGER CONSTRAINT fk_assistant_client_error_conversation_id REFERENCES assistant_conversation (id) ON DELETE SET NULL,
+   message_id INTEGER CONSTRAINT fk_assistant_client_error_message_id REFERENCES assistant_message (id) ON DELETE SET NULL,
+   turn_id INTEGER CONSTRAINT fk_assistant_client_error_turn_id REFERENCES assistant_turn (id) ON DELETE SET NULL,
+   part_type VARCHAR(64),
+   render_type VARCHAR(64),
+   app_version VARCHAR(64),
+   occurred_at TIMESTAMP WITH TIME ZONE,
+   create_date TIMESTAMP WITH TIME ZONE
+);
+CREATE INDEX IF NOT EXISTS assistant_client_error_user_id ON assistant_client_error (user_id);
+CREATE INDEX IF NOT EXISTS assistant_client_error_create_date ON assistant_client_error (create_date);
+CREATE INDEX IF NOT EXISTS assistant_client_error_conversation ON assistant_client_error (conversation_id);
+CREATE INDEX IF NOT EXISTS assistant_client_error_message ON assistant_client_error (message_id);
+CREATE INDEX IF NOT EXISTS assistant_client_error_turn ON assistant_client_error (turn_id);
+
+-- Provider setup state: SETTING_UP marks a provider an admin is still building. The
+-- assistant never routes to it and its provider and model health checks do not run.
+-- Existing rows (and NULL) behave as ACTIVE, exactly as before. Safe to re-run.
+ALTER TABLE assistant_provider ADD COLUMN IF NOT EXISTS setup_state VARCHAR(16) DEFAULT 'ACTIVE'
+   CONSTRAINT ck_assistant_provider_setup_state CHECK (setup_state IN ('ACTIVE', 'SETTING_UP'));

pom.xml

@@ -20,7 +20,7 @@
    <!-- Multi-module build. The qqq-orb sets the revision here; the modules -->
    <!-- inherit it, and the flatten plugin resolves it in the published POMs. -->
    <properties>
-      <revision>0.1.0-SNAPSHOT</revision>
+      <revision>0.1.0-RC.24</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -217,7 +217,7 @@
          <properties>
             <!-- RC.7 provides classified-value tokens, log redaction and native help bundles; -->
             <!-- RC.8 provides core job tracking (JobTracking, the Jobs tray).                   -->
-            <qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.15</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitConfig.java

@@ -110,6 +110,9 @@ public class AssistantQBitConfig implements QBitConfig
    private AssistantRunTracker runTracker = new JobTrackingRunTracker();
 
    private Integer               maxTurnSeconds             = 300;
+   private Integer               toolCallTimeoutSeconds     = 60;
+   private Integer               clientErrorsPerUserPerMinute = 30;
+   private Integer               clientErrorWarnPerHour     = 20;
    private Integer               maxUserMessageChars        = 8000;
    private Integer               historyMessageLimit        = 40;
    private AssistantBudgets      budgets                    = new AssistantBudgets();
@@ -233,6 +236,9 @@ public class AssistantQBitConfig implements QBitConfig
          errors.add("turnHeartbeatSeconds must be less than turnLeaseSeconds.");
       }
 
+      requirePositive(errors, "toolCallTimeoutSeconds", toolCallTimeoutSeconds);
+      requirePositive(errors, "clientErrorsPerUserPerMinute", clientErrorsPerUserPerMinute);
+      requirePositive(errors, "clientErrorWarnPerHour", clientErrorWarnPerHour);
       requirePositive(errors, "maxUserMessageChars", maxUserMessageChars);
       requirePositive(errors, "historyMessageLimit", historyMessageLimit);
       requirePositive(errors, "maxTurnsPerUserPerMinute", maxTurnsPerUserPerMinute);
@@ -747,6 +753,99 @@ public class AssistantQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Getter for toolCallTimeoutSeconds (how long one tool call may run before it gets an error result; default 60)
+    *******************************************************************************/
+   public Integer getToolCallTimeoutSeconds()
+   {
+      return (this.toolCallTimeoutSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for toolCallTimeoutSeconds
+    *******************************************************************************/
+   public void setToolCallTimeoutSeconds(Integer toolCallTimeoutSeconds)
+   {
+      this.toolCallTimeoutSeconds = toolCallTimeoutSeconds;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for toolCallTimeoutSeconds
+    *******************************************************************************/
+   public AssistantQBitConfig withToolCallTimeoutSeconds(Integer toolCallTimeoutSeconds)
+   {
+      this.toolCallTimeoutSeconds = toolCallTimeoutSeconds;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for clientErrorsPerUserPerMinute (client error reports accepted per user and minute; more are acknowledged but dropped; default 30)
+    *******************************************************************************/
+   public Integer getClientErrorsPerUserPerMinute()
+   {
+      return (this.clientErrorsPerUserPerMinute);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for clientErrorsPerUserPerMinute
+    *******************************************************************************/
+   public void setClientErrorsPerUserPerMinute(Integer clientErrorsPerUserPerMinute)
+   {
+      this.clientErrorsPerUserPerMinute = clientErrorsPerUserPerMinute;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for clientErrorsPerUserPerMinute
+    *******************************************************************************/
+   public AssistantQBitConfig withClientErrorsPerUserPerMinute(Integer clientErrorsPerUserPerMinute)
+   {
+      this.clientErrorsPerUserPerMinute = clientErrorsPerUserPerMinute;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for clientErrorWarnPerHour (client error reports in the last hour at which the health check warns; default 20)
+    *******************************************************************************/
+   public Integer getClientErrorWarnPerHour()
+   {
+      return (this.clientErrorWarnPerHour);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for clientErrorWarnPerHour
+    *******************************************************************************/
+   public void setClientErrorWarnPerHour(Integer clientErrorWarnPerHour)
+   {
+      this.clientErrorWarnPerHour = clientErrorWarnPerHour;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for clientErrorWarnPerHour
+    *******************************************************************************/
+   public AssistantQBitConfig withClientErrorWarnPerHour(Integer clientErrorWarnPerHour)
+   {
+      this.clientErrorWarnPerHour = clientErrorWarnPerHour;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for maxUserMessageChars
     *******************************************************************************/

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantRuntime.java

@@ -30,6 +30,7 @@ import java.util.regex.Pattern;
 import com.kingsrook.qbits.assistant.access.AccessExplainer;
 import com.kingsrook.qbits.assistant.access.DataAccessService;
 import com.kingsrook.qbits.assistant.engine.BudgetService;
+import com.kingsrook.qbits.assistant.engine.ClientErrors;
 import com.kingsrook.qbits.assistant.engine.ContextBuilder;
 import com.kingsrook.qbits.assistant.engine.PolicyService;
 import com.kingsrook.qbits.assistant.engine.ProfileService;
@@ -86,6 +87,7 @@ public class AssistantRuntime
    private final ProviderService                  providerService;
    private final BudgetService                    budgetService;
    private final RateLimiter                      rateLimiter;
+   private final ClientErrors                     clientErrors;
    private final AiToolKernel                     kernel;
    private final ContextBuilder                   contextBuilder;
    private final DataAccessService                dataAccessService;
@@ -134,6 +136,7 @@ public class AssistantRuntime
       this.budgetService = new BudgetService(store, providerService, clock);
       this.policyService.withBudgetService(budgetService, providerService);
       this.rateLimiter = new RateLimiter(clock);
+      this.clientErrors = new ClientErrors(clock);
       AiToolKernelSettings settings = kernelSettings(config);
       this.kernel = new AiToolKernel(qInstance, settings);
       AiExposurePolicy exposurePolicy = new AiExposurePolicy(settings);
@@ -481,6 +484,17 @@ public class AssistantRuntime
 
 
 
+   /*******************************************************************************
+    ** Getter for clientErrors (bounds, scrubbing and rate limit of client
+    ** error reports)
+    *******************************************************************************/
+   public ClientErrors getClientErrors()
+   {
+      return (clientErrors);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for kernel
     *******************************************************************************/

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/api/AssistantApi.java

@@ -32,6 +32,8 @@ import com.kingsrook.qbits.assistant.AssistantRuntime;
 import com.kingsrook.qbits.assistant.AssistantSecurity;
 import com.kingsrook.qbits.assistant.access.AccessExplainer;
 import com.kingsrook.qbits.assistant.access.DataAccessMode;
+import com.kingsrook.qbits.assistant.engine.AbandonedTurns;
+import com.kingsrook.qbits.assistant.engine.ClientErrors;
 import com.kingsrook.qbits.assistant.engine.ContextBuilder;
 import com.kingsrook.qbits.assistant.engine.MessageParts;
 import com.kingsrook.qbits.assistant.engine.PolicyService;
@@ -40,6 +42,7 @@ import com.kingsrook.qbits.assistant.engine.ProviderService;
 import com.kingsrook.qbits.assistant.engine.RunningTurn;
 import com.kingsrook.qbits.assistant.engine.TurnEngine;
 import com.kingsrook.qbits.assistant.engine.TurnEvent;
+import com.kingsrook.qbits.assistant.model.AssistantClientError;
 import com.kingsrook.qbits.assistant.model.AssistantConversation;
 import com.kingsrook.qbits.assistant.model.AssistantFeedback;
 import com.kingsrook.qbits.assistant.model.AssistantMessage;
@@ -206,7 +209,7 @@ public class AssistantApi
       List<Map<String, Object>> messages = new ArrayList<>();
       for(AssistantMessage message : store.listMessages(userId, id, beforeMessageId, pageSize))
       {
-         messages.add(toJson(message));
+         messages.add(toJson(repairIfAbandoned(message)));
       }
 
       if(conversation.getUnreadCount() != null && conversation.getUnreadCount() > 0)
@@ -223,6 +226,45 @@ public class AssistantApi
 
 
 
+   /*******************************************************************************
+    ** A message still STREAMING whose run is not live here: an orphaned run is
+    ** recovered; a run that already ended (or is gone) gives the message its
+    ** terminal status. A run still alive on another replica is left alone.
+    ** Never throws: the conversation is shown either way.
+    *******************************************************************************/
+   private AssistantMessage repairIfAbandoned(AssistantMessage message)
+   {
+      if(!"STREAMING".equals(message.getStatus()) || (message.getTurnId() != null && runtime.getRunningTurn(message.getTurnId()) != null))
+      {
+         return (message);
+      }
+      try
+      {
+         AssistantTurn turn = message.getTurnId() == null ? null : store.getTurn(userId, message.getTurnId());
+         if(isRunning(turn))
+         {
+            turn = reapIfOrphaned(turn);
+            if(isRunning(turn))
+            {
+               return (message);
+            }
+            AssistantMessage reread = store.getMessage(userId, message.getId());
+            if(reread != null && !"STREAMING".equals(reread.getStatus()))
+            {
+               return (reread);
+            }
+         }
+         return (AbandonedTurns.repairMessage(store, userId, message, turn));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not check an unfinished assistant message", logPair("messageId", message.getId()));
+         return (message);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** PATCH /conversations/{id}
     *******************************************************************************/
@@ -669,53 +711,7 @@ public class AssistantApi
       }
       // Recovery only appends fixed notices to already redacted stored parts (or
       // crash placeholders). It never reloads model/tool values or raw user text.
-      String status = cancelReason == null ? "ERROR" : "CANCELLED";
-      String text = cancelReason == null ? "This answer was interrupted (the server restarted). Ask again."
-         : ("SUPERSEDED".equals(cancelReason) ? TurnEngine.SUPERSEDED_TEXT : "Stopped by you.");
-      if(current.getAssistantMessageId() != null)
-      {
-         try
-         {
-            AssistantMessage message = store.getMessage(userId, current.getAssistantMessageId());
-            if(message != null)
-            {
-               store.updateFields(AssistantMessage.TABLE_NAME, new QRecord().withValue("id", message.getId()).withValue("status", status)
-                  .withValue("partsJson", MessageParts.fromJson(message.getPartsJson()).addNotice(cancelReason == null ? "error" : "info", text).toJson()));
-            }
-         }
-         catch(Exception e)
-         {
-            LOG.warn("Could not persist recovered assistant message");
-         }
-      }
-      try
-      {
-         int seq = store.lastEventSeq(userId, current.getId());
-         List<AssistantTurnEvent> last = store.listEvents(userId, current.getId(), Math.max(0, seq - 1), 1);
-         if(last.isEmpty() || !TurnEvent.MESSAGE_END.equals(last.getFirst().getEventType()))
-         {
-            if(cancelReason == null)
-            {
-               seq++;
-               store.insertEvent(current.getId(), userId, seq, TurnEvent.ERROR,
-                  new TurnEvent(seq, TurnEvent.ERROR, Map.of("code", "INTERNAL", "message", text)).toJson());
-            }
-            seq++;
-            Map<String, Object> end = new LinkedHashMap<>();
-            end.put("messageId", current.getAssistantMessageId());
-            end.put("status", status);
-            end.put("stopReason", cancelReason == null ? "interrupted" : "cancelled");
-            store.insertEvent(current.getId(), userId, seq, TurnEvent.MESSAGE_END, new TurnEvent(seq, TurnEvent.MESSAGE_END, end).toJson());
-         }
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Could not persist recovered assistant events");
-      }
-      Instant now = runtime.getClock().instant();
-      store.updateRunningTurn(userId, new QRecord().withValue("id", current.getId()).withValue("status", status)
-         .withValue("errorCode", cancelReason == null ? "INTERNAL" : null).withValue("cancelReason", cancelReason)
-         .withValue("stopReason", cancelReason == null ? "interrupted" : "cancelled").withValue("phase", "done").withValue("endedAt", now));
+      AbandonedTurns.finalizeRun(store, runtime.getClock().instant(), userId, current, cancelReason);
       return (store.getTurn(userId, current.getId()));
    }
 
@@ -755,6 +751,71 @@ public class AssistantApi
 
 
 
+   /*******************************************************************************
+    ** POST /client-errors: an error the panel hit, reported by the user's own
+    ** browser. Any signed-in user may report (also when the assistant is off
+    ** for them); only bounded, scrubbed fields are stored (ClientErrors), and
+    ** conversation / message / turn ids only when they are the user's own.
+    ** Over the per-user rate limit the report is acknowledged but dropped
+    ** (accepted: false), never a 429, so a looping client is not spammed.
+    *******************************************************************************/
+   public Map<String, Object> reportClientError(Map<String, Object> body) throws QException, AssistantApiException
+   {
+      if(userId == null)
+      {
+         throw (new AssistantApiException(403, "NO_PERMISSION", "Sign in to report errors."));
+      }
+      if(body == null)
+      {
+         throw (new AssistantApiException(400, "A JSON object is required."));
+      }
+      String scope = ClientErrors.token(body.get("scope"), ClientErrors.SCOPE_MAX);
+      if(scope == null)
+      {
+         throw (new AssistantApiException(400, "scope is required."));
+      }
+      String message = ClientErrors.text(body.get("message"), ClientErrors.MESSAGE_MAX);
+      if(message == null)
+      {
+         throw (new AssistantApiException(400, "message is required."));
+      }
+      if(!runtime.getClientErrors().tryAcquire(userId, config.getClientErrorsPerUserPerMinute()))
+      {
+         return (Map.of("accepted", false));
+      }
+
+      Integer conversationId = ClientErrors.id(body.get("conversationId"));
+      Integer messageId      = ClientErrors.id(body.get("messageId"));
+      Integer turnId         = ClientErrors.id(body.get("turnId"));
+      AssistantClientError row = new AssistantClientError()
+         .withUserId(userId)
+         .withScope(scope)
+         .withMessage(message)
+         .withErrorName(ClientErrors.token(body.get("errorName"), ClientErrors.ERROR_NAME_MAX))
+         .withStack(ClientErrors.text(body.get("stack"), ClientErrors.STACK_MAX))
+         .withComponentStack(ClientErrors.text(body.get("componentStack"), ClientErrors.COMPONENT_STACK_MAX))
+         .withRequestId(ClientErrors.token(body.get("requestId"), ClientErrors.REQUEST_ID_MAX))
+         .withRoute(ClientErrors.route(body.get("route")))
+         .withConversationId(conversationId != null && store.getConversation(userId, conversationId) != null ? conversationId : null)
+         .withMessageId(messageId != null && store.getMessage(userId, messageId) != null ? messageId : null)
+         .withTurnId(turnId != null && store.getTurn(userId, turnId) != null ? turnId : null)
+         .withPartType(ClientErrors.token(body.get("partType"), ClientErrors.PART_TYPE_MAX))
+         .withRenderType(ClientErrors.token(body.get("renderType"), ClientErrors.RENDER_TYPE_MAX))
+         .withAppVersion(ClientErrors.token(body.get("appVersion"), ClientErrors.APP_VERSION_MAX))
+         .withOccurredAt(runtime.getClientErrors().occurredAt(body.get("occurredAt")));
+      AssistantClientError saved = store.insert(row);
+
+      //////////////////////////////////////////////////////////////////////
+      // structured, no reported text: the row holds the (scrubbed) text //
+      //////////////////////////////////////////////////////////////////////
+      LOG.info("Assistant client error reported", logPair("clientErrorId", saved.getId()), logPair("scope", row.getScope()),
+         logPair("partType", row.getPartType()), logPair("renderType", row.getRenderType()), logPair("conversationId", row.getConversationId()),
+         logPair("messageId", row.getMessageId()), logPair("turnId", row.getTurnId()));
+      return (Map.of("accepted", true));
+   }
+
+
+
    /*******************************************************************************
     ** GET /actions/{id}: one of the user's pending actions.
     *******************************************************************************/

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/api/specs/AssistantSpecs.java

@@ -37,9 +37,11 @@ import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
 import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
 import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension;
 import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
+import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
 import com.kingsrook.qqq.openapi.model.HttpMethod;
 import com.kingsrook.qqq.openapi.model.In;
 import com.kingsrook.qqq.openapi.model.Parameter;
+import com.kingsrook.qqq.openapi.model.Response;
 import com.kingsrook.qqq.openapi.model.Schema;
 import com.kingsrook.qqq.openapi.model.Type;
 import io.javalin.http.ContentType;
@@ -54,14 +56,14 @@ import io.javalin.http.HttpStatus;
  ** routes, with the same URLs and bodies, so they appear in the served
  ** /qqq/v1/openapi.yaml and a generated client covers them.
  **
- ** Errors keep the assistant's shape: HTTP status plus {"error", "code"}.
+ ** Errors use BasicErrorResponseV1 and retain assistant-specific codes.
  ** Turns and events are text/event-stream; each event type is documented as
  ** AssistantTurnEventV1.
  *******************************************************************************/
 public final class AssistantSpecs
 {
    public static final String EXTENSION_NAME    = "assistant";
-   public static final String EXTENSION_VERSION = "1.2.0";
+   public static final String EXTENSION_VERSION = "1.5.0";
    public static final String PREFIX            = "/assistant";
 
 
@@ -99,7 +101,8 @@ public final class AssistantSpecs
          .withSpec(GetActionSpec.class)
          .withSpec(ConfirmActionSpec.class)
          .withSpec(RejectActionSpec.class)
-         .withSpec(AccessSpec.class));
+         .withSpec(AccessSpec.class)
+         .withSpec(ClientErrorsSpec.class));
    }
 
 
@@ -208,9 +211,31 @@ public final class AssistantSpecs
       rs.put("AssistantMessageV1", object("A message: the user's text or the assistant's reply as parts (text, tool, record_refs, navigate, notice).",
          "id", integer(null), "role", string("USER or ASSISTANT."), "status", string(null), "turnId", integer(null).withNullable(true),
          "createDate", string(null).withNullable(true), "parts", array(freeObject("A message part."), null)));
+      rs.put("AssistantRecordsRenderV1", object("A tool part's render of type records (query results). columns and rows are always present - empty when the query matched nothing (since 1.4.0; older stored parts are completed when read).",
+         "type", string("records"), "tableName", string(null), "tableLabel", string(null),
+         "columns", array(object("A column.", "name", string(null), "label", string(null)), "Always present; empty when nothing matched."),
+         "rows", array(object("A row.", "id", string("The primary key."), "href", string("The record's /app link."), "values", freeObject("Display values by column name.")), "Always present; empty when nothing matched."),
+         "totalCount", integer("All matching rows.").withNullable(true), "truncated", bool("More rows match than are shown."), "openHref", string("The Query screen link.")));
       rs.put("AssistantConversationDetailResponseV1", object("A conversation and a page of its messages, oldest first.",
          "conversation", ref("AssistantConversationV1"), "messages", array(ref("AssistantMessageV1"), null)));
       rs.put("AssistantStatusResponseV1", object("An acknowledgement.", "status", string(null)));
+      rs.put("AssistantClientErrorReportV1", object("An error the client hit while showing the assistant (since 1.5.0). scope and message are required; "
+            + "longer values are truncated, message and stacks are scrubbed of e-mail addresses, long numbers, tokens and query strings, and ids "
+            + "are kept only when they are the user's own.",
+         "scope", string("Required. Where it happened: assistant-panel, assistant-message, assistant-part, assistant-render, assistant-stream, widget, record-section, page...").withMaxLength(64),
+         "message", string("Required. The error message.").withMaxLength(500),
+         "errorName", string("The error's name, e.g. TypeError.").withMaxLength(100).withNullable(true),
+         "stack", string("The JavaScript stack.").withMaxLength(4000).withNullable(true),
+         "componentStack", string("The React component stack.").withMaxLength(4000).withNullable(true),
+         "requestId", string("The request or trace id the client saw.").withMaxLength(128).withNullable(true),
+         "route", string("The page path (no host, query or fragment).").withMaxLength(300).withNullable(true),
+         "conversationId", integer(null).withNullable(true), "messageId", integer(null).withNullable(true), "turnId", integer(null).withNullable(true),
+         "partType", string("The message part type being shown.").withMaxLength(64).withNullable(true),
+         "renderType", string("The render type being shown.").withMaxLength(64).withNullable(true),
+         "appVersion", string("The client build.").withMaxLength(64).withNullable(true),
+         "occurredAt", string("ISO-8601 instant.").withFormat("date-time").withNullable(true)));
+      rs.put("AssistantClientErrorResponseV1", object("Whether the report was stored: false when the user's reports are over the per-minute limit (still 202).",
+         "accepted", bool(null)));
       rs.put("AssistantActionV1", object("A write the assistant proposed and its state.",
          "id", integer(null), "status", string("PENDING, RUNNING, EXECUTED, FAILED, REJECTED or EXPIRED."), "toolName", string(null), "tableName", string(null),
          "recordId", string(null).withNullable(true), "summary", string(null), "httpStatus", integer("The UI API status of the write.").withNullable(true),
@@ -227,7 +252,7 @@ public final class AssistantSpecs
       rs.put("AssistantTurnEventV1", object("One server-sent event of a turn (event: <type>, id: <seq>, data: this object). Types: message.start, turn.phase, text.delta, tool.start, tool.result, ui.navigate, notice, usage, message.end, error, ping.",
          "seq", integer("Sequence number; resume with ?after=<seq>."), "type", string("The event type (also the SSE event name)."),
          "data", freeObject("The event's fields, by type. message.start includes startedAt (ISO-8601). turn.phase includes phase (thinking, tool, writing), optional toolName and label.")));
-      rs.put("AssistantErrorV1", object("An error.", "error", string("A message for the user."), "code", string("BUDGET, RATE_LIMIT, NOT_CONFIGURED, KILL_SWITCH, DISABLED, NO_PERMISSION...").withNullable(true)));
+      rs.put("AssistantErrorV1", object("Deprecated since 1.3.0: errors use BasicErrorResponseV1.", "error", string("A message for the user."), "code", string("BUDGET, RATE_LIMIT, NOT_CONFIGURED, KILL_SWITCH, DISABLED, NO_PERMISSION...").withNullable(true)));
       return (rs);
    }
 
@@ -260,7 +285,7 @@ public final class AssistantSpecs
    /*******************************************************************************
     ** Base of the assistant's JSON specs: the user's own AssistantApi (with the
     ** request's credentials, forwarded to the UI API by the tools), errors in
-    ** the assistant's {"error","code"} shape with their status.
+    ** the standard error shape with their status and assistant-specific code.
     *******************************************************************************/
    public abstract static class AssistantJsonSpec extends AbstractJsonEndpointSpec
    {
@@ -336,6 +361,26 @@ public final class AssistantSpecs
 
 
 
+      /*******************************************************************************
+       ** A bodyless response has empty content. Core 1.6's schema traversal
+       ** requires a non-null map even when the response has no schema.
+       *******************************************************************************/
+      @Override
+      public Map<Integer, Response> defineResponses()
+      {
+         Map<Integer, Response> responses = super.defineResponses();
+         for(Response response : responses.values())
+         {
+            if(response.getContent() == null)
+            {
+               response.withContent(Map.of());
+            }
+         }
+         return (responses);
+      }
+
+
+
       /*******************************************************************************
        ** Keep the request (for its credentials and the server's local port).
        *******************************************************************************/
@@ -403,7 +448,7 @@ public final class AssistantSpecs
 
 
       /*******************************************************************************
-       ** Assistant errors keep their status and {"error","code"}; others go
+       ** Assistant errors keep their status and code in the standard shape; others go
        ** through QQQ's mapping (400 / 401 / 403 / 404 / 500).
        *******************************************************************************/
       @Override
@@ -413,15 +458,16 @@ public final class AssistantSpecs
          AssistantApiException apiException = ExceptionUtils.findClassInRootChain(e, AssistantApiException.class);
          if(apiException != null)
          {
-            Map<String, Object> body = new LinkedHashMap<>();
-            body.put("error", apiException.getMessage());
+            context.contentType(ContentType.APPLICATION_JSON);
+            QJavalinUtils.respondWithError(context, org.eclipse.jetty.http.HttpStatus.getCode(apiException.getStatus()), apiException.getMessage());
             if(apiException.getCode() != null)
             {
-               body.put("code", apiException.getCode());
+               context.result(JsonUtils.toJson(new BasicErrorResponseV1()
+                  .withError(apiException.getMessage())
+                  .withCode(apiException.getCode())
+                  .withTraceId(QJavalinUtils.getTraceId(context))
+                  .toMap()));
             }
-            context.status(apiException.getStatus());
-            context.contentType(ContentType.APPLICATION_JSON);
-            context.result(JsonUtils.toJson(body));
             return;
          }
          QJavalinUtils.handleException(null, context, e);
@@ -436,12 +482,12 @@ public final class AssistantSpecs
       public List<BasicResponse> defineAdditionalBasicResponses()
       {
          return (List.of(
-            new BasicResponse(HttpStatus.BAD_REQUEST, "Invalid input", "AssistantErrorV1"),
-            new BasicResponse(HttpStatus.FORBIDDEN, "No access to the assistant", "AssistantErrorV1"),
-            new BasicResponse(HttpStatus.NOT_FOUND, "Not found (or not the user's own)", "AssistantErrorV1"),
-            new BasicResponse(HttpStatus.CONFLICT, "Already finished or decided", "AssistantErrorV1"),
-            new BasicResponse(HttpStatus.TOO_MANY_REQUESTS, "Budget or rate limit", "AssistantErrorV1"),
-            new BasicResponse(HttpStatus.SERVICE_UNAVAILABLE, "Disabled, not configured, or the kill switch is on", "AssistantErrorV1")));
+            new BasicResponse(HttpStatus.BAD_REQUEST, "Invalid input", BasicErrorResponseV1.class.getSimpleName()),
+            new BasicResponse(HttpStatus.FORBIDDEN, "No access to the assistant", BasicErrorResponseV1.class.getSimpleName()),
+            new BasicResponse(HttpStatus.NOT_FOUND, "Not found (or not the user's own)", BasicErrorResponseV1.class.getSimpleName()),
+            new BasicResponse(HttpStatus.CONFLICT, "Already finished or decided", BasicErrorResponseV1.class.getSimpleName()),
+            new BasicResponse(HttpStatus.TOO_MANY_REQUESTS, "Budget or rate limit", BasicErrorResponseV1.class.getSimpleName()),
+            new BasicResponse(HttpStatus.SERVICE_UNAVAILABLE, "Disabled, not configured, or the kill switch is on", BasicErrorResponseV1.class.getSimpleName())));
       }
 
 
@@ -1361,6 +1407,89 @@ public final class AssistantSpecs
 
 
 
+   /*******************************************************************************
+    ** POST /assistant/client-errors (since 1.5.0)
+    *******************************************************************************/
+   public static class ClientErrorsSpec extends AssistantJsonSpec
+   {
+      /*******************************************************************************
+       ** HTTP method.
+       *******************************************************************************/
+      @Override
+      protected HttpMethod method()
+      {
+         return (HttpMethod.POST);
+      }
+
+
+
+      /*******************************************************************************
+       ** Route path.
+       *******************************************************************************/
+      @Override
+      protected String path()
+      {
+         return (PREFIX + "/client-errors");
+      }
+
+
+
+      /*******************************************************************************
+       ** Operation summary.
+       *******************************************************************************/
+      @Override
+      protected String summary()
+      {
+         return ("Report an error the client hit while showing the assistant (stored for administrators, rate limited per user)");
+      }
+
+
+
+      /*******************************************************************************
+       ** Request body.
+       *******************************************************************************/
+      @Override
+      protected Schema defineRequestBodySchema()
+      {
+         return (ref("AssistantClientErrorReportV1"));
+      }
+
+
+
+      /*******************************************************************************
+       ** Response schema.
+       *******************************************************************************/
+      @Override
+      protected String getResponseSchemaName()
+      {
+         return ("AssistantClientErrorResponseV1");
+      }
+
+
+
+      /*******************************************************************************
+       ** 202: accepted (or over the limit and dropped).
+       *******************************************************************************/
+      @Override
+      protected HttpStatus successStatus()
+      {
+         return (HttpStatus.ACCEPTED);
+      }
+
+
+
+      /*******************************************************************************
+       ** Runs in the invoker's authenticated user session.
+       *******************************************************************************/
+      @Override
+      protected Object handle(AssistantApi api, JsonInput input) throws Exception
+      {
+         return (api.reportClientError(input.getBody()));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** GET /assistant/access
     *******************************************************************************/

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/AbandonedTurns.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.engine;
+
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.assistant.AssistantQBitConfig;
+import com.kingsrook.qbits.assistant.model.AssistantMessage;
+import com.kingsrook.qbits.assistant.model.AssistantTurn;
+import com.kingsrook.qbits.assistant.model.AssistantTurnEvent;
+import com.kingsrook.qbits.assistant.persistence.AssistantStore;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Terminal state for runs whose engine is gone (a pod restart, a crash, a
+ ** finalizer that could not write): the turn row, its assistant message and a
+ ** terminal replay (error + message.end) are written, so no reader ever sees
+ ** an answer "in progress" forever and the "answer already in progress"
+ ** lockout cannot persist.
+ **
+ ** Used by the API when it reads an orphaned run (active turns, events,
+ ** conversation detail, the next question) and by the retention process,
+ ** which sweeps every run older than the hard stream limit. Only fixed
+ ** notices are appended to already stored (redacted) parts; nothing is
+ ** reloaded from the model, tools or the user's text.
+ *******************************************************************************/
+public final class AbandonedTurns
+{
+   private static final QLogger LOG = QLogger.getLogger(AbandonedTurns.class);
+
+   public static final String INTERRUPTED_TEXT = "This answer was interrupted (the server restarted). Ask again.";
+   public static final String UNFINISHED_TEXT  = "This answer did not finish. Ask again.";
+
+   private static final int BATCH = 200;
+
+
+
+   /*******************************************************************************
+    ** not instantiable
+    *******************************************************************************/
+   private AbandonedTurns()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Finalize one abandoned run as ERROR (no cancel reason: interrupted) or
+    ** CANCELLED (a stop nobody acknowledged). Each step is independent; the
+    ** turn row is written last and only while it still says it runs.
+    *******************************************************************************/
+   public static void finalizeRun(AssistantStore store, Instant now, String userId, AssistantTurn current, String cancelReason) throws QException
+   {
+      String status = cancelReason == null ? "ERROR" : "CANCELLED";
+      String text = cancelReason == null ? INTERRUPTED_TEXT
+         : ("SUPERSEDED".equals(cancelReason) ? TurnEngine.SUPERSEDED_TEXT : "Stopped by you.");
+      if(current.getAssistantMessageId() != null)
+      {
+         try
+         {
+            AssistantMessage message = store.getMessage(userId, current.getAssistantMessageId());
+            if(message != null)
+            {
+               store.updateFields(AssistantMessage.TABLE_NAME, new QRecord().withValue("id", message.getId()).withValue("status", status)
+                  .withValue("partsJson", MessageParts.fromJson(message.getPartsJson()).addNotice(cancelReason == null ? "error" : "info", text).toJson()));
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Could not persist recovered assistant message", logPair("turnId", current.getId()));
+         }
+      }
+      try
+      {
+         int seq = store.lastEventSeq(userId, current.getId());
+         List<AssistantTurnEvent> last = store.listEvents(userId, current.getId(), Math.max(0, seq - 1), 1);
+         if(last.isEmpty() || !TurnEvent.MESSAGE_END.equals(last.getFirst().getEventType()))
+         {
+            if(cancelReason == null)
+            {
+               seq++;
+               store.insertEvent(current.getId(), userId, seq, TurnEvent.ERROR,
+                  new TurnEvent(seq, TurnEvent.ERROR, Map.of("code", "INTERNAL", "message", text)).toJson());
+            }
+            seq++;
+            Map<String, Object> end = new LinkedHashMap<>();
+            end.put("messageId", current.getAssistantMessageId());
+            end.put("status", status);
+            end.put("stopReason", cancelReason == null ? "interrupted" : "cancelled");
+            store.insertEvent(current.getId(), userId, seq, TurnEvent.MESSAGE_END, new TurnEvent(seq, TurnEvent.MESSAGE_END, end).toJson());
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not persist recovered assistant events", logPair("turnId", current.getId()));
+      }
+      store.updateRunningTurn(userId, new QRecord().withValue("id", current.getId()).withValue("status", status)
+         .withValue("errorCode", cancelReason == null ? "INTERNAL" : null).withValue("cancelReason", cancelReason)
+         .withValue("stopReason", cancelReason == null ? "interrupted" : "cancelled").withValue("phase", "done").withValue("endedAt", now));
+   }
+
+
+
+   /*******************************************************************************
+    ** An assistant message still STREAMING whose run is over (or gone): give it
+    ** the run's terminal status (ERROR when the run is unknown) and, unless the
+    ** run completed, an error notice. Returns the repaired message (or the
+    ** given one when nothing changed or the repair could not be written).
+    *******************************************************************************/
+   public static AssistantMessage repairMessage(AssistantStore store, String userId, AssistantMessage message, AssistantTurn turn)
+   {
+      if(message == null || !"STREAMING".equals(message.getStatus()))
+      {
+         return (message);
+      }
+      String status = turn == null || turn.getStatus() == null || "RUNNING".equals(turn.getStatus()) || "CANCELLING".equals(turn.getStatus()) ? "ERROR" : turn.getStatus();
+      try
+      {
+         MessageParts parts = MessageParts.fromJson(message.getPartsJson());
+         if(!"COMPLETE".equals(status))
+         {
+            parts.addNotice("ERROR".equals(status) ? "error" : "info", "ERROR".equals(status) ? UNFINISHED_TEXT : "Stopped.");
+         }
+         String partsJson = parts.toJson();
+         store.updateFields(AssistantMessage.TABLE_NAME, new QRecord().withValue("id", message.getId()).withValue("status", status).withValue("partsJson", partsJson));
+         message.setStatus(status);
+         message.setPartsJson(partsJson);
+         LOG.info("Repaired an unfinished assistant message", logPair("messageId", message.getId()), logPair("turnId", message.getTurnId()), logPair("status", status));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not repair an unfinished assistant message", logPair("messageId", message.getId()), logPair("turnId", message.getTurnId()));
+      }
+      return (message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Retention sweep: every run (of any user the running session can see)
+    ** still RUNNING or CANCELLING although it started before the hard stream
+    ** limit (maxTurnSeconds + force-finalize grace + lease) is finalized as
+    ** interrupted. Returns how many.
+    *******************************************************************************/
+   public static int sweep(AssistantQBitConfig config, AssistantStore store, Instant now) throws QException
+   {
+      Instant cutoff = now.minusSeconds(config.getMaxTurnSeconds() + (long) config.getTurnForceFinalizeGraceSeconds() + config.getTurnLeaseSeconds());
+      int     count  = 0;
+      Integer afterId = 0;
+      while(true)
+      {
+         List<AssistantTurn> batch = store.listRunningTurnsStartedBefore(cutoff, afterId, BATCH);
+         for(AssistantTurn turn : batch)
+         {
+            afterId = turn.getId();
+            try
+            {
+               finalizeRun(store, now, turn.getUserId(), turn, null);
+               count++;
+            }
+            catch(Exception e)
+            {
+               LOG.warn("Could not finalize an abandoned assistant turn", logPair("turnId", turn.getId()));
+            }
+         }
+         if(batch.size() < BATCH)
+         {
+            break;
+         }
+      }
+      return (count);
+   }
+}

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ClientErrors.java

@@ -0,0 +1,226 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.engine;
+
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+import com.kingsrook.qqq.backend.core.actions.ai.output.AiRedactor;
+
+
+/*******************************************************************************
+ ** Client error reports (POST /assistant/client-errors): the bounds of every
+ ** field, the scrubbing of free text, and the per-user rate limit.
+ **
+ ** Reported text may contain personal data (an error message can quote a
+ ** value), so message and stacks are scrubbed before they are stored: secrets
+ ** through the kernel's AiRedactor, then e-mail addresses, long digit runs
+ ** (more than 6 digits: phone, account and card numbers, ids), long opaque
+ ** tokens (32 or more letters and digits, mixed) and query strings. Short identifiers keep only printable characters.
+ ** A route keeps only its path.
+ *******************************************************************************/
+public class ClientErrors
+{
+   public static final int SCOPE_MAX           = 64;
+   public static final int MESSAGE_MAX         = 500;
+   public static final int ERROR_NAME_MAX      = 100;
+   public static final int STACK_MAX           = 4000;
+   public static final int COMPONENT_STACK_MAX = 4000;
+   public static final int REQUEST_ID_MAX      = 128;
+   public static final int ROUTE_MAX           = 300;
+   public static final int PART_TYPE_MAX       = 64;
+   public static final int RENDER_TYPE_MAX     = 64;
+   public static final int APP_VERSION_MAX     = 64;
+
+   private static final AiRedactor SECRETS = new AiRedactor(List.of());
+
+   private static final Pattern EMAIL        = Pattern.compile("[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}");
+   private static final Pattern LONG_DIGITS  = Pattern.compile("\\d(?:[\\s.-]?\\d){6,}");
+   private static final Pattern LONG_TOKEN   = Pattern.compile("(?=[A-Za-z0-9_+=-]*\\d)(?=[A-Za-z0-9_+=-]*[A-Za-z])[A-Za-z0-9_+=-]{32,}");
+   private static final Pattern QUERY_STRING = Pattern.compile("\\?[^\\s)\"']*");
+   private static final Pattern CONTROL      = Pattern.compile("[\\p{Cntrl}&&[^\\n\\t]]");
+   private static final Pattern NOT_TOKEN    = Pattern.compile("[^\\p{Graph} ]");
+
+   private final Clock                       clock;
+   private final Map<String, Deque<Instant>> windows = new ConcurrentHashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public ClientErrors(Clock clock)
+   {
+      this.clock = clock;
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether one more report from this user fits in the last minute (and
+    ** count it if so).
+    *******************************************************************************/
+   public boolean tryAcquire(String userId, int perMinute)
+   {
+      Instant        now    = clock.instant();
+      Deque<Instant> window = windows.computeIfAbsent(userId, key -> new ArrayDeque<>());
+      synchronized(window)
+      {
+         while(!window.isEmpty() && !window.peekFirst().isAfter(now.minusSeconds(60)))
+         {
+            window.pollFirst();
+         }
+         if(window.size() >= perMinute)
+         {
+            return (false);
+         }
+         window.addLast(now);
+         return (true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Free text (message, stacks): secrets, e-mails, long numbers, tokens and
+    ** query strings removed, control characters dropped, then truncated.
+    ** Null for a non-string or blank value.
+    *******************************************************************************/
+   public static String text(Object value, int max)
+   {
+      if(!(value instanceof String text) || text.isBlank())
+      {
+         return (null);
+      }
+      String scrubbed = text.length() > max * 4 ? text.substring(0, max * 4) : text;
+      scrubbed = SECRETS.redact(scrubbed, new HashSet<>());
+      scrubbed = EMAIL.matcher(scrubbed).replaceAll("[email]");
+      scrubbed = QUERY_STRING.matcher(scrubbed).replaceAll("?[query]");
+      scrubbed = LONG_TOKEN.matcher(scrubbed).replaceAll("[token]");
+      scrubbed = LONG_DIGITS.matcher(scrubbed).replaceAll("[number]");
+      scrubbed = CONTROL.matcher(scrubbed).replaceAll("").strip();
+      return (truncate(scrubbed, max));
+   }
+
+
+
+   /*******************************************************************************
+    ** A short identifier (scope, error name, request id, types, version):
+    ** printable characters only, truncated. Null for a non-string or blank value.
+    *******************************************************************************/
+   public static String token(Object value, int max)
+   {
+      if(!(value instanceof String text))
+      {
+         return (null);
+      }
+      String cleaned = NOT_TOKEN.matcher(text).replaceAll("").strip();
+      cleaned = EMAIL.matcher(cleaned).replaceAll("[email]");
+      return (cleaned.isEmpty() ? null : truncate(cleaned, max));
+   }
+
+
+
+   /*******************************************************************************
+    ** The path of a route: scheme, host, query and fragment removed; long
+    ** numbers and e-mails scrubbed. Null unless it is a path ("/...").
+    *******************************************************************************/
+   public static String route(Object value)
+   {
+      if(!(value instanceof String text))
+      {
+         return (null);
+      }
+      String path = text.strip().replaceFirst("^[A-Za-z][A-Za-z0-9+.-]*://[^/]*", "");
+      int    cut  = path.indexOf('?');
+      path = cut < 0 ? path : path.substring(0, cut);
+      cut = path.indexOf('#');
+      path = cut < 0 ? path : path.substring(0, cut);
+      if(!path.startsWith("/") || path.startsWith("//"))
+      {
+         return (null);
+      }
+      path = NOT_TOKEN.matcher(path).replaceAll("").replace(" ", "");
+      path = EMAIL.matcher(path).replaceAll("[email]");
+      path = LONG_DIGITS.matcher(path).replaceAll("[number]");
+      return (truncate(path, ROUTE_MAX));
+   }
+
+
+
+   /*******************************************************************************
+    ** A positive integer id, or null.
+    *******************************************************************************/
+   public static Integer id(Object value)
+   {
+      if(value instanceof Number number && number.doubleValue() == number.longValue() && number.longValue() > 0 && number.longValue() <= Integer.MAX_VALUE)
+      {
+         return (number.intValue());
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** An ISO-8601 instant (with Z or an offset), or null; times in the future
+    ** become now.
+    *******************************************************************************/
+   public Instant occurredAt(Object value)
+   {
+      if(!(value instanceof String text) || text.length() > 64)
+      {
+         return (null);
+      }
+      Instant instant;
+      try
+      {
+         instant = OffsetDateTime.parse(text.strip()).toInstant();
+      }
+      catch(RuntimeException e)
+      {
+         try
+         {
+            instant = Instant.parse(text.strip());
+         }
+         catch(RuntimeException e2)
+         {
+            return (null);
+         }
+      }
+      Instant now = clock.instant();
+      return (instant.isAfter(now) ? now : instant);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String truncate(String value, int max)
+   {
+      return (value.length() <= max ? value : value.substring(0, max));
+   }
+}

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/MessageParts.java

@@ -50,11 +50,12 @@ public class MessageParts
       {
          try
          {
-            List<Map<String, Object>> list = JsonUtils.toObject(json, new TypeReference<List<Map<String, Object>>>() {});
-            if(list != null)
-            {
-               result.parts.addAll(list);
-            }
+            List<Object> list = JsonUtils.toObject(json, new TypeReference<List<Object>>() {});
+            //////////////////////////////////////////////////////////////////////
+            // stored parts are repaired on read: older rows may lack empty     //
+            // render lists, and a part that cannot be shown becomes a notice   //
+            //////////////////////////////////////////////////////////////////////
+            result.parts.addAll(RenderShapes.parts(list));
          }
          catch(Exception e)
          {
@@ -240,7 +241,7 @@ public class MessageParts
     *******************************************************************************/
    public synchronized String toJson()
    {
-      return (JsonUtils.toJson(parts));
+      return (RenderShapes.toJson(RenderShapes.parts(parts)));
    }
 
 

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderHealth.java

@@ -27,6 +27,7 @@ import com.kingsrook.qbits.assistant.spi.HealthResult;
 import com.kingsrook.qbits.assistant.spi.ModelProviderException;
 import com.kingsrook.qbits.assistant.spi.ProviderSettings;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
 import com.kingsrook.qqq.backend.core.health.HealthRunner;
 import com.kingsrook.qqq.backend.core.health.HealthStatus;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -37,6 +38,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  *******************************************************************************/
 public class ProviderHealth
 {
+   /*******************************************************************************
+    ** Health detail set to false when a provider probe failed, even when the
+    ** outcome is only DEGRADED because the provider is optional.
+    *******************************************************************************/
+   public static final String DETAIL_REACHABLE = "reachable";
+
    private final AssistantRuntime runtime;
 
 
@@ -65,6 +72,10 @@ public class ProviderHealth
       {
          throw (new ModelProviderException(ModelProviderException.Kind.AUTHENTICATION, null, "Provider is disabled."));
       }
+      if(row.isSettingUp())
+      {
+         throw (new ModelProviderException(ModelProviderException.Kind.AUTHENTICATION, null, "Provider is not set up yet."));
+      }
       if(isDown(row))
       {
          throw (new ModelProviderException(ModelProviderException.Kind.OVERLOADED, null, "Provider circuit is open."));
@@ -80,7 +91,7 @@ public class ProviderHealth
       var outcome = runner == null ? null : runner.getLastOutcome(checkName);
       if(checked != null && !checked.isAfter(now) && checked.plusSeconds(60).isAfter(now) && outcome != null)
       {
-         if(outcome.getStatus() == HealthStatus.DOWN)
+         if(outcome.getStatus() == HealthStatus.DOWN || isUnreachable(outcome))
          {
             throw (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, "Provider health check failed."));
          }
@@ -158,4 +169,15 @@ public class ProviderHealth
          .withValue("healthState", state).withValue("healthCheckedAt", runtime.getClock().instant()).withValue("downUntil", downUntil));
       runtime.getProviderService().invalidate();
    }
+
+
+
+   /*******************************************************************************
+    ** A central outcome that says the provider could not be reached, whatever
+    ** its status (an optional provider's failed probe is DEGRADED, not DOWN).
+    *******************************************************************************/
+   public static boolean isUnreachable(HealthCheckOutcome outcome)
+   {
+      return (outcome != null && outcome.getDetails() != null && Boolean.FALSE.equals(outcome.getDetails().getValues().get(DETAIL_REACHABLE)));
+   }
 }

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java

@@ -158,14 +158,15 @@ public class ProviderService
 
 
    /*******************************************************************************
-    ** The default enabled provider row, or null.
+    ** The default usable (enabled and set up) provider row, else the first
+    ** usable one, or null.
     *******************************************************************************/
    public AssistantProvider activeRow() throws QException
    {
       AssistantProvider first = null;
       for(AssistantProvider row : rows())
       {
-         if(Boolean.TRUE.equals(row.getEnabled()))
+         if(row.isUsable())
          {
             if(Boolean.TRUE.equals(row.getIsDefault()))
             {
@@ -782,7 +783,7 @@ public class ProviderService
          return (null);
       }
       AssistantProvider row = store.getProvider(model.getProviderId());
-      if(row == null || !Boolean.TRUE.equals(row.getEnabled()) || ((requiresHipaa || Boolean.TRUE.equals(config.getHipaaMode())) && !Boolean.TRUE.equals(row.getHipaaEligible())))
+      if(row == null || !row.isUsable() || ((requiresHipaa || Boolean.TRUE.equals(config.getHipaaMode())) && !Boolean.TRUE.equals(row.getHipaaEligible())))
       {
          return (null);
       }
@@ -854,9 +855,9 @@ public class ProviderService
          throw (new ProviderSetupException("Select an enabled EMBEDDING model for this knowledge source."));
       }
       AssistantProvider row = store.getProvider(model.getProviderId());
-      if(row == null || !Boolean.TRUE.equals(row.getEnabled()))
+      if(row == null || !row.isUsable())
       {
-         throw (new ProviderSetupException("The embedding provider is disabled or unavailable."));
+         throw (new ProviderSetupException("The embedding provider is disabled, not set up yet or unavailable."));
       }
       if((requiresHipaa || Boolean.TRUE.equals(config.getHipaaMode())) && !Boolean.TRUE.equals(row.getHipaaEligible()))
       {

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/RenderShapes.java

@@ -0,0 +1,502 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.engine;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The shapes clients rely on for message parts, tool result renders and
+ ** presentations (extension "assistant", AssistantMessageV1 parts and the
+ ** tool.result / presentation events). Every part, render and presentation
+ ** is checked before it is persisted or streamed, and again when stored
+ ** parts and events are read back (older rows are repaired on read):
+ **
+ ** - required lists are always present, empty when there is nothing (a query
+ **   that matched no rows has "rows": [] and "columns": []);
+ ** - list entries that are not objects are dropped, required strings are
+ **   filled with safe defaults;
+ ** - a render that cannot be repaired (no type, a pending action without an
+ **   action id, a presentation with nothing to show) is dropped and a notice
+ **   part ({@link #UNDISPLAYABLE_TEXT}) takes its place;
+ ** - unknown part and render types pass through untouched.
+ **
+ ** Problems are logged with the render type and a fixed problem code only;
+ ** never with values. QQQ's default JSON omits empty lists, so parts and
+ ** stream events are written with {@link #toJson(Object)}.
+ *******************************************************************************/
+public final class RenderShapes
+{
+   private static final QLogger LOG = QLogger.getLogger(RenderShapes.class);
+
+   /** what the user sees instead of a result that cannot be shown */
+   public static final String UNDISPLAYABLE_TEXT = "This result could not be displayed.";
+
+   private static final Pattern    SAFE_TYPE     = Pattern.compile("[a-z][a-z0-9_.-]{0,63}");
+   private static final Set<String> NOTICE_LEVELS = Set.of("info", "warning", "error");
+   private static final Set<String> ITEM_BLOCKS   = Set.of("metrics", "filterLinks", "recordLinks", "actions");
+
+
+
+   /*******************************************************************************
+    ** not instantiable
+    *******************************************************************************/
+   private RenderShapes()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** JSON that keeps empty lists (nulls are still left out).
+    *******************************************************************************/
+   public static String toJson(Object value)
+   {
+      return (JsonUtils.toJsonCustomized(value, builder -> builder.serializationInclusion(JsonInclude.Include.NON_NULL)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The notice part shown in place of a part or render that cannot be shown.
+    *******************************************************************************/
+   public static Map<String, Object> undisplayableNotice()
+   {
+      Map<String, Object> notice = new LinkedHashMap<>();
+      notice.put("type", "notice");
+      notice.put("level", "error");
+      notice.put("text", UNDISPLAYABLE_TEXT);
+      return (notice);
+   }
+
+
+
+   /*******************************************************************************
+    ** Normalized copies of message parts (never throws; never null). A part
+    ** that is not an object, or has no type, becomes an error notice; a tool
+    ** part whose render cannot be repaired loses its render and is followed by
+    ** an error notice; unknown part types pass through.
+    *******************************************************************************/
+   public static List<Map<String, Object>> parts(Object parts)
+   {
+      List<Map<String, Object>> result = new ArrayList<>();
+      if(!(parts instanceof List<?> list))
+      {
+         return (result);
+      }
+      for(Object entry : list)
+      {
+         try
+         {
+            addPart(result, entry);
+         }
+         catch(RuntimeException e)
+         {
+            problem(entry instanceof Map<?, ?> map ? map.get("type") : null, "part-exception");
+            result.add(undisplayableNotice());
+         }
+      }
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    ** Normalize one part into the result.
+    *******************************************************************************/
+   private static void addPart(List<Map<String, Object>> result, Object entry)
+   {
+      if(!(entry instanceof Map<?, ?> map) || !(map.get("type") instanceof String type))
+      {
+         problem(null, "part-without-type");
+         result.add(undisplayableNotice());
+         return;
+      }
+      Map<String, Object> part = copy(map);
+      switch(type)
+      {
+         case "text" -> part.put("text", string(part.get("text"), ""));
+         case "notice" ->
+         {
+            part.put("level", part.get("level") instanceof String level && NOTICE_LEVELS.contains(level) ? level : "info");
+            part.put("text", string(part.get("text"), ""));
+         }
+         case "navigate" ->
+         {
+            if(!(part.get("href") instanceof String href) || !TurnEngine.isAppHref(href))
+            {
+               problem(type, "navigate-without-app-href");
+               return;
+            }
+         }
+         case "record_refs" -> part.put("refs", objects(part.get("refs")));
+         case "tool" ->
+         {
+            part.put("name", string(part.get("name"), "tool"));
+            part.put("summary", string(part.get("summary"), ""));
+            part.put("outcome", string(part.get("outcome"), "ERROR"));
+            if(part.containsKey("render"))
+            {
+               Map<String, Object> render = render(part.get("render"));
+               if(render == null)
+               {
+                  part.remove("render");
+                  result.add(part);
+                  result.add(undisplayableNotice());
+                  return;
+               }
+               part.put("render", render);
+            }
+         }
+         case "presentation" ->
+         {
+            Map<String, Object> presentation = presentation(part);
+            result.add(presentation == null ? undisplayableNotice() : presentation);
+            return;
+         }
+         default ->
+         {
+            //////////////////////////////////////////////////
+            // unknown part types pass through untouched    //
+            //////////////////////////////////////////////////
+         }
+      }
+      result.add(part);
+   }
+
+
+
+   /*******************************************************************************
+    ** A normalized copy of a tool result render, or null when it cannot be
+    ** repaired (the problem is logged without values). Unknown render types
+    ** pass through as a shallow copy.
+    *******************************************************************************/
+   public static Map<String, Object> render(Object value)
+   {
+      try
+      {
+         if(!(value instanceof Map<?, ?> map) || !(map.get("type") instanceof String type))
+         {
+            problem(null, "render-without-type");
+            return (null);
+         }
+         Map<String, Object> render = copy(map);
+         switch(type)
+         {
+            case "records" ->
+            {
+               tableLabel(render);
+               List<Map<String, Object>> columns = new ArrayList<>();
+               for(Map<String, Object> column : objects(render.get("columns")))
+               {
+                  if(column.get("name") instanceof String name)
+                  {
+                     column.put("label", string(column.get("label"), name));
+                     columns.add(column);
+                  }
+               }
+               render.put("columns", columns);
+               List<Map<String, Object>> rows = objects(render.get("rows"));
+               for(Map<String, Object> row : rows)
+               {
+                  row.put("values", row.get("values") instanceof Map<?, ?> values ? copy(values) : new LinkedHashMap<>());
+               }
+               render.put("rows", rows);
+               render.put("totalCount", number(render.get("totalCount")));
+               render.put("truncated", Boolean.TRUE.equals(render.get("truncated")));
+            }
+            case "record" ->
+            {
+               tableLabel(render);
+               List<Map<String, Object>> fields = new ArrayList<>();
+               for(Map<String, Object> field : objects(render.get("fields")))
+               {
+                  if(field.get("name") instanceof String name)
+                  {
+                     field.put("label", string(field.get("label"), name));
+                     Object fieldValue = field.get("value");
+                     field.put("value", fieldValue == null || fieldValue instanceof String ? fieldValue : String.valueOf(fieldValue));
+                     fields.add(field);
+                  }
+               }
+               render.put("fields", fields);
+            }
+            case "count" ->
+            {
+               tableLabel(render);
+               render.put("count", number(render.get("count")));
+            }
+            case "widget" ->
+            {
+               render.put("label", string(render.get("label"), string(render.get("widgetName"), "Widget")));
+               render.put("data", render.get("data") instanceof Map<?, ?> data ? copy(data) : new LinkedHashMap<>());
+            }
+            case "history" ->
+            {
+               List<Map<String, Object>> entries = objects(render.get("entries"));
+               for(Map<String, Object> entry : entries)
+               {
+                  entry.put("details", objects(entry.get("details")));
+               }
+               render.put("entries", entries);
+            }
+            case "pending_action" ->
+            {
+               Object actionId = number(render.get("actionId"));
+               if(actionId == null)
+               {
+                  problem(type, "pending-action-without-id");
+                  return (null);
+               }
+               render.put("actionId", actionId);
+               render.put("summary", string(render.get("summary"), ""));
+               render.put("changes", objects(render.get("changes")));
+            }
+            case "bulk_load_dry_run" -> render.put("errors", objects(render.get("errors")));
+            default ->
+            {
+               //////////////////////////////////////////////////
+               // unknown render types pass through untouched  //
+               //////////////////////////////////////////////////
+            }
+         }
+         return (render);
+      }
+      catch(RuntimeException e)
+      {
+         problem(value instanceof Map<?, ?> map ? map.get("type") : null, "render-exception");
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A normalized copy of a presentation part, or null when it has nothing to
+    ** show. Blocks that are not objects or have no kind are dropped; table,
+    ** chart and item blocks always carry their lists.
+    *******************************************************************************/
+   public static Map<String, Object> presentation(Object value)
+   {
+      try
+      {
+         if(!(value instanceof Map<?, ?> map))
+         {
+            problem("presentation", "presentation-not-object");
+            return (null);
+         }
+         Map<String, Object> part = copy(map);
+         part.put("type", "presentation");
+         part.put("summary", string(part.get("summary"), ""));
+         List<Map<String, Object>> blocks = new ArrayList<>();
+         for(Map<String, Object> block : objects(part.get("blocks")))
+         {
+            if(!(block.get("kind") instanceof String kind))
+            {
+               problem("presentation", "block-without-kind");
+               continue;
+            }
+            if("table".equals(kind))
+            {
+               block.put("title", string(block.get("title"), ""));
+               List<Map<String, Object>> columns = new ArrayList<>();
+               for(Map<String, Object> column : objects(block.get("columns")))
+               {
+                  if(column.get("key") instanceof String key)
+                  {
+                     column.put("label", string(column.get("label"), key));
+                     columns.add(column);
+                  }
+               }
+               block.put("columns", columns);
+               List<Map<String, Object>> rows = objects(block.get("rows"));
+               for(Map<String, Object> row : rows)
+               {
+                  row.put("cells", row.get("cells") instanceof Map<?, ?> cells ? copy(cells) : new LinkedHashMap<>());
+                  row.put("display", row.get("display") instanceof Map<?, ?> display ? copy(display) : new LinkedHashMap<>());
+               }
+               block.put("rows", rows);
+            }
+            else if("chart".equals(kind))
+            {
+               block.put("title", string(block.get("title"), ""));
+               block.put("chartType", string(block.get("chartType"), "bar"));
+               block.put("points", objects(block.get("points")));
+            }
+            else if(ITEM_BLOCKS.contains(kind))
+            {
+               block.put("items", objects(block.get("items")));
+            }
+            blocks.add(block);
+         }
+         part.put("blocks", blocks);
+         part.put("markdown", string(part.get("markdown"), (String) part.get("summary")));
+         if(blocks.isEmpty() && ((String) part.get("summary")).isBlank() && ((String) part.get("markdown")).isBlank())
+         {
+            problem("presentation", "presentation-empty");
+            return (null);
+         }
+         return (part);
+      }
+      catch(RuntimeException e)
+      {
+         problem("presentation", "presentation-exception");
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Normalize an event's data in place: a tool.result render that cannot be
+    ** repaired is removed; a presentation event's part that cannot be shown is
+    ** replaced by the error notice part.
+    *******************************************************************************/
+   public static void normalizeEvent(String type, Map<String, Object> data)
+   {
+      if(data == null)
+      {
+         return;
+      }
+      if(data.containsKey("render"))
+      {
+         Map<String, Object> render = render(data.get("render"));
+         if(render == null)
+         {
+            data.remove("render");
+         }
+         else
+         {
+            data.put("render", render);
+         }
+      }
+      if(TurnEvent.PRESENTATION.equals(type) && data.containsKey("part"))
+      {
+         Map<String, Object> part = presentation(data.get("part"));
+         data.put("part", part == null ? undisplayableNotice() : part);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Fill in a single part (in place) - kept for callers of the 1.4.0 helper.
+    ** A render that cannot be repaired is removed.
+    *******************************************************************************/
+   public static Map<String, Object> complete(Map<String, Object> part)
+   {
+      if(part != null)
+      {
+         normalizeEvent(null, part);
+      }
+      return (part);
+   }
+
+
+
+   /*******************************************************************************
+    ** tableLabel defaults to the table name.
+    *******************************************************************************/
+   private static void tableLabel(Map<String, Object> render)
+   {
+      render.put("tableLabel", string(render.get("tableLabel"), string(render.get("tableName"), "")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Copies of the object entries of a list (others dropped); empty when the
+    ** value is not a list.
+    *******************************************************************************/
+   private static List<Map<String, Object>> objects(Object value)
+   {
+      List<Map<String, Object>> result = new ArrayList<>();
+      if(value instanceof List<?> list)
+      {
+         for(Object entry : list)
+         {
+            if(entry instanceof Map<?, ?> map)
+            {
+               result.add(copy(map));
+            }
+         }
+      }
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    ** A mutable copy with string keys.
+    *******************************************************************************/
+   private static Map<String, Object> copy(Map<?, ?> map)
+   {
+      Map<String, Object> copy = new LinkedHashMap<>();
+      map.forEach((key, value) -> copy.put(String.valueOf(key), value));
+      return (copy);
+   }
+
+
+
+   /*******************************************************************************
+    ** The value when it is a string; otherwise the default.
+    *******************************************************************************/
+   private static String string(Object value, String defaultValue)
+   {
+      return (value instanceof String s ? s : defaultValue);
+   }
+
+
+
+   /*******************************************************************************
+    ** A number (numeric strings are converted), or null.
+    *******************************************************************************/
+   private static Number number(Object value)
+   {
+      if(value instanceof Number n)
+      {
+         return (n);
+      }
+      if(value instanceof String s && s.matches("-?\\d{1,18}"))
+      {
+         return (Long.parseLong(s));
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Log a shape problem: the type (when it looks like a type) and a fixed
+    ** code, never values.
+    *******************************************************************************/
+   private static void problem(Object type, String code)
+   {
+      String safeType = type instanceof String s && SAFE_TYPE.matcher(s).matches() ? s : null;
+      LOG.warn("Assistant part or render was not displayable", logPair("renderType", safeType), logPair("problem", code));
+   }
+}

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEngine.java

@@ -29,12 +29,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLongArray;
 import com.kingsrook.qbits.assistant.AssistantQBitConfig;
 import com.kingsrook.qbits.assistant.AssistantRuntime;
 import com.kingsrook.qbits.assistant.model.AssistantConversation;
@@ -67,6 +69,7 @@ import com.kingsrook.qqq.backend.core.actions.ai.AiToolCall;
 import com.kingsrook.qqq.backend.core.actions.ai.AiToolOutcome;
 import com.kingsrook.qqq.backend.core.actions.ai.AiToolResult;
 import com.kingsrook.qqq.backend.core.actions.ai.output.AiRedactor;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -770,13 +773,28 @@ public class TurnEngine implements Runnable
       {
          throw (new BudgetService.Blocked("The assistant has reached this provider's monthly cost cap. Ask an administrator."));
       }
+      ///////////////////////////////////////////////////////////////////////
+      // providers stream on their own (virtual) threads, which have no     //
+      // QContext: run the delta handler in the turn's context, or progress //
+      // writes (JobLogWriter) fail with no instance                        //
+      ///////////////////////////////////////////////////////////////////////
+      CapturedContext turnContext = QContext.capture();
       ModelEventListener listener = new ModelEventListener()
       {
          @Override
          public void onTextDelta(String text)
          {
-            phase("writing", null);
-            bufferText(text);
+            if(QContext.getQInstance() != null)
+            {
+               phase("writing", null);
+               bufferText(text);
+               return;
+            }
+            QContext.withTemporaryContext(turnContext, () ->
+            {
+               phase("writing", null);
+               bufferText(text);
+            });
          }
       };
 
@@ -855,6 +873,10 @@ public class TurnEngine implements Runnable
       AiToolResult[]   results      = new AiToolResult[toolUses.size()];
       UiApiToolbox.Outcome[] outcomes = new UiApiToolbox.Outcome[toolUses.size()];
       List<Future<?>>  futures      = new ArrayList<>();
+      List<Integer>    futureIndexes = new ArrayList<>();
+      AtomicLongArray  startedNanos = new AtomicLongArray(toolUses.size());
+      boolean[]        failed       = new boolean[toolUses.size()];
+      boolean[]        timedOut     = new boolean[toolUses.size()];
       Semaphore        permits      = new Semaphore(Math.max(1, config.getMaxParallelToolCalls()));
 
       for(int i = 0; i < toolUses.size(); i++)
@@ -879,6 +901,7 @@ public class TurnEngine implements Runnable
 
          final int    index = i;
          AiToolCall   call  = new AiToolCall().withId(use.getToolUseId()).withName(use.getToolName()).withInput(use.getInput());
+         futureIndexes.add(index);
          futures.add(runtime.getExecutor().submit(() ->
          {
             if(!transcript.retainWorker())
@@ -890,6 +913,7 @@ public class TurnEngine implements Runnable
             {
                QContext.init(runtime.getQInstance(), session);
                permits.acquire();
+               startedNanos.set(index, System.nanoTime());
                try
                {
                   if(!running.isCancelled())
@@ -944,11 +968,15 @@ public class TurnEngine implements Runnable
             {
                Thread.currentThread().interrupt();
             }
-            catch(RuntimeException e)
+            catch(RuntimeException | Error e)
             {
-               LOG.warn("Tool kernel threw", hipaaSafe(e), logPair("tool", call.getName()));
-               results[index] = new AiToolResult().withToolCallId(call.getId()).withToolName(call.getName()).withOutcome(AiToolOutcome.ERROR)
-                  .withSummary("Failed").withModelContent("The tool failed unexpectedly.");
+               ///////////////////////////////////////////////////////////////////
+               // internal details go to the log only; the model and the user  //
+               // get a fixed, safe text                                       //
+               ///////////////////////////////////////////////////////////////////
+               LOG.warn("Tool kernel threw", hipaaSafe(e), logPair("turnId", turn.getId()), logPair("tool", call.getName()), logPair("errorClass", e.getClass().getSimpleName()));
+               failed[index] = true;
+               results[index] = failedResult(call.getId(), call.getName());
             }
             finally
             {
@@ -960,20 +988,39 @@ public class TurnEngine implements Runnable
       }
       persistPendingEvents();
 
-      for(Future<?> future : futures)
+      long toolTimeoutNanos = TimeUnit.SECONDS.toNanos(config.getToolCallTimeoutSeconds() == null ? 60 : config.getToolCallTimeoutSeconds());
+      for(int f = 0; f < futures.size(); f++)
       {
-         long waitMillis = Math.max(1, Duration.between(runtime.getClock().instant(), deadline).toMillis());
+         Future<?> future = futures.get(f);
+         int       index  = futureIndexes.get(f);
+         ///////////////////////////////////////////////////////////////////////
+         // each call has its own limit (from when it got a permit), and no  //
+         // call outlives the turn's deadline                                 //
+         ///////////////////////////////////////////////////////////////////////
+         long now        = System.nanoTime();
+         long started    = startedNanos.get(index);
+         long toolMillis = TimeUnit.NANOSECONDS.toMillis((started == 0 ? now : started) + toolTimeoutNanos - now);
+         long turnMillis = Duration.between(runtime.getClock().instant(), deadline).toMillis();
+         long waitMillis = Math.max(1, Math.min(toolMillis, turnMillis));
          try
          {
             future.get(waitMillis, TimeUnit.MILLISECONDS);
          }
          catch(TimeoutException e)
          {
+            timedOut[index] = !future.isDone();
             future.cancel(true);
+            LOG.warn("Assistant tool call timed out", logPair("turnId", turn.getId()), logPair("tool", toolUses.get(index).getToolName()),
+               logPair("limitSeconds", config.getToolCallTimeoutSeconds()), logPair("turnDeadline", toolMillis > turnMillis));
          }
          catch(ExecutionException e)
          {
-            LOG.debug("tool future failed", hipaaSafe(e));
+            failed[index] = true;
+            LOG.warn("Assistant tool call failed", hipaaSafe(e), logPair("turnId", turn.getId()), logPair("tool", toolUses.get(index).getToolName()));
+         }
+         catch(CancellationException e)
+         {
+            LOG.debug("tool future cancelled", logPair("turnId", turn.getId()));
          }
       }
 
@@ -987,15 +1034,30 @@ public class TurnEngine implements Runnable
             continue;
          }
 
-         AiToolResult result = results[i];
-         if(result == null)
+         AiToolResult         result = timedOut[i] ? null : results[i];
+         UiApiToolbox.Outcome extras = timedOut[i] ? null : outcomes[i];
+         if(result == null && failed[i])
          {
+            result = failedResult(use.getToolUseId(), use.getToolName());
+         }
+         else if(result == null)
+         {
+            boolean stopped = running.isCancelled() && !timedOut[i];
             result = new AiToolResult().withToolCallId(use.getToolUseId()).withToolName(use.getToolName()).withOutcome(AiToolOutcome.ERROR)
-               .withSummary(running.isCancelled() ? "Not run (stopped)" : "Timed out")
-               .withModelContent(running.isCancelled() ? "Not run: the user stopped the answer." : "The tool did not finish in time.");
+               .withSummary(stopped ? "Not run (stopped)" : "Timed out")
+               .withModelContent(stopped ? "Not run: the user stopped the answer." : "The tool did not finish in time. Answer with what you have, or try a narrower request.");
+            timedOut[i] = !stopped;
          }
          classification.observe(use, result);
-         recordToolResult(use, result, outcomes[i]);
+         recordToolResult(use, result, extras);
+         if(timedOut[i])
+         {
+            notice("error", "A step took too long and was stopped (" + ToolLabels.running(use.getToolName()) + ").");
+         }
+         else if(failed[i] || (extras != null && extras.isFailed()))
+         {
+            notice("error", "A step failed (" + ToolLabels.running(use.getToolName()) + ").");
+         }
 
          String content = result.getModelContent() == null ? "" : result.getModelContent();
          modelResults.add(ModelPart.toolResult(use.getToolUseId(), content, result.getOutcome() != null && ERROR_OUTCOMES.contains(result.getOutcome())));
@@ -1008,7 +1070,21 @@ public class TurnEngine implements Runnable
 
 
    /*******************************************************************************
-    ** Audit row, events and message parts of one tool result.
+    ** The result of a tool call that threw: an ERROR (is_error) result with a
+    ** fixed text; what went wrong is only in the log.
+    *******************************************************************************/
+   private static AiToolResult failedResult(String toolCallId, String toolName)
+   {
+      return (new AiToolResult().withToolCallId(toolCallId).withToolName(toolName).withOutcome(AiToolOutcome.ERROR)
+         .withSummary("Failed").withModelContent("The tool failed unexpectedly. Answer with what you have, or try another way."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Audit row, events and message parts of one tool result. The render is
+    ** checked first (RenderShapes): one that cannot be shown is left out and an
+    ** error notice says so.
     *******************************************************************************/
    private void recordToolResult(ModelPart use, AiToolResult result, UiApiToolbox.Outcome extras)
    {
@@ -1026,9 +1102,23 @@ public class TurnEngine implements Runnable
 
       Map<String, Object> data = map("messageId", assistantMessage.getId(), "toolCallId", use.getToolUseId(), "name", use.getToolName(), "outcome", outcome, "summary", summary);
       Boolean presentation = PresentResultsTool.NAME.equals(use.getToolName()) && AiToolOutcome.OK.equals(result.getOutcome()) && result.getRender() != null;
-      if(result.getRender() != null && !presentation)
+      Map<String, Object> render           = null;
+      Map<String, Object> presentationPart = null;
+      if(result.getRender() != null)
       {
-         data.put("render", result.getRender());
+         if(presentation)
+         {
+            presentationPart = RenderShapes.presentation(result.getRender());
+         }
+         else
+         {
+            render = RenderShapes.render(result.getRender());
+         }
+      }
+      boolean undisplayable = result.getRender() != null && render == null && presentationPart == null;
+      if(render != null)
+      {
+         data.put("render", render);
       }
       if(!refs.isEmpty())
       {
@@ -1036,11 +1126,16 @@ public class TurnEngine implements Runnable
       }
       emit(TurnEvent.TOOL_RESULT, data);
 
-      parts.addTool(use.getToolUseId(), use.getToolName(), summary, outcome, presentation ? null : result.getRender());
-      if(presentation)
+      parts.addTool(use.getToolUseId(), use.getToolName(), summary, outcome, render);
+      if(presentationPart != null)
       {
-         parts.addPresentation(result.getRender());
-         emit(TurnEvent.PRESENTATION, map("messageId", assistantMessage.getId(), "part", result.getRender()));
+         parts.addPresentation(presentationPart);
+         emit(TurnEvent.PRESENTATION, map("messageId", assistantMessage.getId(), "part", presentationPart));
+      }
+      if(undisplayable)
+      {
+         LOG.warn("Assistant tool result could not be displayed", logPair("turnId", turn.getId()), logPair("tool", use.getToolName()));
+         notice("error", RenderShapes.UNDISPLAYABLE_TEXT);
       }
       parts.addRecordRefs(refs);
 
@@ -1215,13 +1310,29 @@ public class TurnEngine implements Runnable
                .withValue("model", model).withValue("partsJson", parts.toJson())));
          }
 
-         emit(TurnEvent.USAGE, map("turnId", turn.getId(), "inputTokens", usage.getInputTokens(), "outputTokens", usage.getOutputTokens(),
-            "cacheReadTokens", usage.getCacheReadTokens(), "cacheWriteTokens", usage.getCacheWriteTokens(), "costMicros", costMicros));
+         //////////////////////////////////////////////////////////////////////
+         // each terminal event is independent: a failing emit never skips  //
+         // message.end or the terminal turn row below                      //
+         //////////////////////////////////////////////////////////////////////
+         safely(() -> emit(TurnEvent.USAGE, map("turnId", turn.getId(), "inputTokens", usage.getInputTokens(), "outputTokens", usage.getOutputTokens(),
+            "cacheReadTokens", usage.getCacheReadTokens(), "cacheWriteTokens", usage.getCacheWriteTokens(), "costMicros", costMicros)));
          if(errorCode != null)
          {
-            emit(TurnEvent.ERROR, map("code", errorCode, "message", errorText));
+            String finalErrorText = errorText;
+            safely(() -> emit(TurnEvent.ERROR, map("code", finalErrorCode, "message", finalErrorText)));
          }
-         emit(TurnEvent.MESSAGE_END, map("messageId", assistantMessage.getId(), "status", status, "stopReason", stopReason));
+
+         ///////////////////////////////////////////////////////////////////////
+         // the stream ends at message.end and the panel reloads the          //
+         // conversation at once, so the redacted transcript (message parts,  //
+         // pending actions, events) must be stored before that event is sent //
+         ///////////////////////////////////////////////////////////////////////
+         if(!alreadyTerminal)
+         {
+            safely(this::persistPendingEvents);
+            safely(() -> transcript.flush(store));
+         }
+         safely(() -> emit(TurnEvent.MESSAGE_END, map("messageId", assistantMessage.getId(), "status", finalStatus, "stopReason", finalStopReason)));
          if(!alreadyTerminal)
          {
             safely(this::persistPendingEvents);

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/TurnEvent.java

@@ -67,6 +67,11 @@ public class TurnEvent
             }
          });
       }
+      ////////////////////////////////////////////////////////////////////
+      // renders and presentations are checked before they are streamed //
+      // or stored, and again when stored events are replayed           //
+      ////////////////////////////////////////////////////////////////////
+      RenderShapes.normalizeEvent(type, this.data);
    }
 
 
@@ -110,7 +115,7 @@ public class TurnEvent
     *******************************************************************************/
    public String toJson()
    {
-      return (JsonUtils.toJson(data));
+      return (RenderShapes.toJson(data));
    }
 
 

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java

@@ -20,16 +20,26 @@ package com.kingsrook.qbits.assistant.health;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import com.kingsrook.qbits.assistant.AssistantRuntime;
 import com.kingsrook.qbits.assistant.AssistantSecurity;
 import com.kingsrook.qbits.assistant.engine.ConnectionTest;
+import com.kingsrook.qbits.assistant.engine.ProviderHealth;
 import com.kingsrook.qbits.assistant.engine.ProviderService;
+import com.kingsrook.qbits.assistant.model.AssistantBudget;
 import com.kingsrook.qbits.assistant.model.AssistantModel;
+import com.kingsrook.qbits.assistant.model.AssistantProfile;
 import com.kingsrook.qbits.assistant.model.AssistantProvider;
+import com.kingsrook.qbits.assistant.model.AssistantRagSource;
 import com.kingsrook.qbits.assistant.spi.HealthResult;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -44,6 +54,8 @@ import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
 import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
 import com.kingsrook.qqq.backend.core.health.HealthDetails;
 import com.kingsrook.qqq.backend.core.health.HealthStatus;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 
@@ -53,6 +65,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
  *******************************************************************************/
 public final class AssistantHealthChecks
 {
+   public static final String CLIENT_ERRORS = "assistant.client-errors";
+   public static final String NOT_SET_UP    = "Not set up yet; checks are skipped";
+
+   private static final QLogger LOG = QLogger.getLogger(AssistantHealthChecks.class);
+
    /*******************************************************************************
     ** Utility class.
     *******************************************************************************/
@@ -70,6 +87,33 @@ public final class AssistantHealthChecks
       HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance);
       register(registry, "assistant.provider", Providers.class);
       register(registry, "assistant.model", Models.class);
+      if(registry.getChecks().stream().noneMatch(existing -> CLIENT_ERRORS.equals(existing.getName())))
+      {
+         registry.withCheck(new HealthCheckMetaData().withName(CLIENT_ERRORS).withLabel("Assistant client errors").withComponent("assistant")
+            .withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.AI)
+            .withIntervalSeconds(300).withTimeoutMs(10000)
+            .withCheckInstance(input -> bounded(input, () -> clientErrors(AssistantRuntime.of(input.getQInstance())))));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Errors the panel reported in the last hour (POST /assistant/client-errors):
+    ** DEGRADED at clientErrorWarnPerHour or more, so administrators see in
+    ** System Health that users are hitting broken screens. Only the count.
+    *******************************************************************************/
+   public static HealthCheckOutcome clientErrors(AssistantRuntime runtime) throws QException
+   {
+      if(runtime == null)
+      {
+         return (HealthCheckOutcome.unknown("Assistant not installed"));
+      }
+      int count = runtime.getStore().countClientErrorsSince(runtime.getClock().instant().minusSeconds(3600));
+      Integer threshold = runtime.getConfig().getClientErrorWarnPerHour();
+      String  text      = count + " client error" + (count == 1 ? "" : "s") + " reported in the last hour";
+      HealthCheckOutcome outcome = threshold != null && count >= threshold ? HealthCheckOutcome.degraded(text) : HealthCheckOutcome.up(text);
+      return (outcome.withDetails(new HealthDetails().withValue("lastHour", count).withValue("warnAt", threshold)));
    }
 
 
@@ -100,7 +144,8 @@ public final class AssistantHealthChecks
 
 
    /*******************************************************************************
-    ** One definition per enabled provider; no I/O to providers during resolution.
+    ** One definition per enabled provider that is set up; no I/O to providers
+    ** during resolution.
     *******************************************************************************/
    public static class Providers implements HealthCheckSupplier
    {
@@ -114,10 +159,10 @@ public final class AssistantHealthChecks
          List<HealthCheckMetaData> checks = new ArrayList<>();
          for(AssistantProvider row : runtime.getStore().listProviders())
          {
-            if(Boolean.TRUE.equals(row.getEnabled()))
+            if(row.isUsable())
             {
                checks.add(definition("assistant.provider." + row.getId(), "LOCAL".equals(runtime.getProviderService().location(row)) ? 60 : 300, 10000,
-                  input -> bounded(input, () -> provider(runtime, row.getId(), input))));
+                  input -> optional(runtime, row.getId(), bounded(input, () -> provider(runtime, row.getId(), input)), false)));
             }
          }
          return (checks);
@@ -127,7 +172,8 @@ public final class AssistantHealthChecks
 
 
    /*******************************************************************************
-    ** One definition per enabled model, including models of disabled providers.
+    ** One definition per enabled model, including models of disabled providers,
+    ** but none for a provider that is still being set up.
     *******************************************************************************/
    public static class Models implements HealthCheckSupplier
    {
@@ -139,12 +185,13 @@ public final class AssistantHealthChecks
       {
          AssistantRuntime runtime = AssistantRuntime.of(instance);
          List<HealthCheckMetaData> checks = new ArrayList<>();
+         Set<Integer> settingUp = runtime.getStore().listProviders().stream().filter(AssistantProvider::isSettingUp).map(AssistantProvider::getId).collect(Collectors.toSet());
          for(AssistantModel row : runtime.getStore().listModels())
          {
-            if(Boolean.TRUE.equals(row.getEnabled()))
+            if(Boolean.TRUE.equals(row.getEnabled()) && !settingUp.contains(row.getProviderId()))
             {
                checks.add(definition("assistant.model." + row.getId(), 21600, 30000,
-                  input -> bounded(input, () -> model(runtime, row.getId(), input))));
+                  input -> optional(runtime, row.getProviderId(), bounded(input, () -> model(runtime, row.getId(), input)), true)));
             }
          }
          return (checks);
@@ -213,6 +260,10 @@ public final class AssistantHealthChecks
       {
          return (HealthCheckOutcome.unknown("Provider disabled or unavailable"));
       }
+      if(row.isSettingUp())
+      {
+         return (HealthCheckOutcome.up(NOT_SET_UP).withDetails(new HealthDetails().withValue("providerId", id)));
+      }
       var type = runtime.getConfig().getProviderTypes().get(row.getProviderType());
       if(type == null)
       {
@@ -261,6 +312,10 @@ public final class AssistantHealthChecks
       {
          return (HealthCheckOutcome.unknown("Provider disabled or unavailable"));
       }
+      if(provider.isSettingUp())
+      {
+         return (HealthCheckOutcome.up(NOT_SET_UP).withDetails(new HealthDetails().withValue("modelId", id).withValue("providerId", provider.getId())));
+      }
       Long cap = ProviderService.capMicros(provider);
       if(cap != null && runtime.getProviderService().monthToDateCostMicros(provider.getId(), true) >= cap)
       {
@@ -276,4 +331,149 @@ public final class AssistantHealthChecks
          .withDetails(new HealthDetails().withValue("modelId", id).withValue("providerId", provider.getId()))
          .withLatencyMs(result.latencyMs() == null ? null : result.latencyMs().longValue()));
    }
+
+
+
+   /*******************************************************************************
+    ** A failed provider (or model) check is DOWN only when the provider matters:
+    ** it is the default, or the only route for a configured profile model, budget
+    ** downgrade or knowledge source. Any other enabled provider is optional, so its
+    ** failure is DEGRADED with a message that says where requests go instead.
+    ** Provider failures also carry reachable=false, so the circuit breaker still
+    ** refuses the provider. If the routes cannot be read, the outcome stays DOWN.
+    *******************************************************************************/
+   static HealthCheckOutcome optional(AssistantRuntime runtime, Integer providerId, HealthCheckOutcome outcome, boolean model)
+   {
+      if(outcome == null || outcome.getStatus() != HealthStatus.DOWN)
+      {
+         return (outcome);
+      }
+      HealthDetails details = outcome.getDetails() == null ? new HealthDetails() : outcome.getDetails();
+      outcome.withDetails(details);
+      if(!model)
+      {
+         details.withValue(ProviderHealth.DETAIL_REACHABLE, false);
+      }
+      try
+      {
+         AssistantProvider row = providerId == null ? null : runtime.getStore().getProvider(providerId);
+         if(row == null || !row.isUsable())
+         {
+            return (outcome);
+         }
+         String required = requiredBecause(runtime, row);
+         if(required != null)
+         {
+            details.withValue("required", required);
+            return (outcome);
+         }
+         AssistantProvider active  = runtime.getProviderService().activeRow();
+         String            instead = active == null ? "the other providers" : "'" + active.getName() + "'";
+         String            what    = model ? "Model connection failed; provider '" + row.getName() + "' is optional" : "Provider '" + row.getName() + "' is unreachable; it is optional";
+         details.withValue("optional", true);
+         return (outcome.withStatus(HealthStatus.DEGRADED).withMessage(what + " (not the default and not the only route), so requests use " + instead));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not tell whether an assistant provider is optional; reporting it as down", e);
+         return (outcome);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Why a provider is required (null when it is optional): it is the provider
+    ** the assistant uses by default, or one of its enabled models is assigned to
+    ** a routing profile, a budget downgrade or a knowledge source and that model
+    ** has no fallback on another usable provider.
+    *******************************************************************************/
+   static String requiredBecause(AssistantRuntime runtime, AssistantProvider row) throws QException
+   {
+      AssistantProvider active = runtime.getProviderService().activeRow();
+      if(active == null || Objects.equals(active.getId(), row.getId()))
+      {
+         return ("default provider");
+      }
+
+      Map<Integer, AssistantModel>    models    = runtime.getStore().listModels().stream().collect(Collectors.toMap(AssistantModel::getId, Function.identity(), (a, b) -> a));
+      Map<Integer, AssistantProvider> providers = runtime.getStore().listProviders().stream().collect(Collectors.toMap(AssistantProvider::getId, Function.identity(), (a, b) -> a));
+      Set<Integer>                    routed    = new HashSet<>();
+      for(AssistantProfile profile : runtime.getStore().query(AssistantProfile.TABLE_NAME, new QQueryFilter(), AssistantProfile.class))
+      {
+         if(!Boolean.FALSE.equals(profile.getEnabled()) && profile.getRoutingMode() != null && !"OFF".equals(profile.getRoutingMode()))
+         {
+            routed.add(profile.getSimpleModelId());
+            routed.add(profile.getComplexModelId());
+         }
+      }
+      for(AssistantBudget budget : runtime.getStore().budgets())
+      {
+         routed.add(budget.getDowngradeModelId());
+      }
+      for(Integer id : routed)
+      {
+         AssistantModel model = id == null ? null : models.get(id);
+         if(onProvider(model, row) && !hasFallback(runtime, model, row, models, providers))
+         {
+            return ("only route for a configured model");
+         }
+      }
+      for(AssistantRagSource source : runtime.getStore().query(AssistantRagSource.TABLE_NAME, new QQueryFilter(), AssistantRagSource.class))
+      {
+         if(!Boolean.FALSE.equals(source.getEnabled()) && source.getEmbeddingModelId() != null && onProvider(models.get(source.getEmbeddingModelId()), row))
+         {
+            return ("only route for a knowledge source");
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** An enabled model of this provider.
+    *******************************************************************************/
+   private static boolean onProvider(AssistantModel model, AssistantProvider row)
+   {
+      return (model != null && Boolean.TRUE.equals(model.getEnabled()) && Objects.equals(model.getProviderId(), row.getId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The fallback FallbackChain would try: the explicit chain when the model has
+    ** one, else (local providers only) another local provider of the same type.
+    ** Only enabled models of other usable providers count.
+    *******************************************************************************/
+   private static boolean hasFallback(AssistantRuntime runtime, AssistantModel model, AssistantProvider row, Map<Integer, AssistantModel> models, Map<Integer, AssistantProvider> providers)
+   {
+      if(model.getFallbackModelId() != null)
+      {
+         Set<Integer> seen = new HashSet<>();
+         Integer      next = model.getFallbackModelId();
+         while(next != null && seen.add(next))
+         {
+            AssistantModel candidate = models.get(next);
+            if(candidate == null)
+            {
+               return (false);
+            }
+            AssistantProvider owner = providers.get(candidate.getProviderId());
+            if(Boolean.TRUE.equals(candidate.getEnabled()) && owner != null && owner.isUsable() && !Objects.equals(owner.getId(), row.getId()))
+            {
+               return (true);
+            }
+            next = candidate.getFallbackModelId();
+         }
+         return (false);
+      }
+      if(!"LOCAL".equals(runtime.getProviderService().location(row)))
+      {
+         return (false);
+      }
+      return (providers.values().stream().anyMatch(other -> other.isUsable() && !Objects.equals(other.getId(), row.getId())
+         && "LOCAL".equals(runtime.getProviderService().location(other)) && Objects.equals(other.getProviderType(), row.getProviderType())
+         && models.values().stream().anyMatch(candidate -> Boolean.TRUE.equals(candidate.getEnabled()) && Objects.equals(candidate.getProviderId(), other.getId()))));
+   }
 }

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthWidget.java

@@ -42,6 +42,24 @@ public class AssistantHealthWidget extends AbstractWidgetRenderer
 {
    public static final String NAME = "assistantHealth";
 
+   public static final String NOT_CHECKED = "Not checked yet";
+
+
+
+   /*******************************************************************************
+    ** Human words for a health status; unknown means no check has reported yet.
+    *******************************************************************************/
+   static String wording(HealthStatus status)
+   {
+      return (switch(status)
+      {
+         case UP -> "Healthy";
+         case DEGRADED -> "Degraded";
+         case DOWN -> "Down";
+         case UNKNOWN -> NOT_CHECKED;
+      });
+   }
+
 
 
    /*******************************************************************************
@@ -72,8 +90,8 @@ public class AssistantHealthWidget extends AbstractWidgetRenderer
             .map(check -> runner.getLastOutcome(check.getName()))
             .map(outcome -> outcome == null ? HealthStatus.UNKNOWN : outcome.getStatus()).toArray(HealthStatus[]::new));
       }
-      return (new RenderWidgetOutput(new StatisticsData().withCount(status.name())
-         .withCountContext("System Health")
+      return (new RenderWidgetOutput(new StatisticsData().withCount(wording(status))
+         .withCountContext("System Health. Provider checks run every 1 to 5 minutes; model checks every 6 hours")
          .withCountURL(FilterLinkBuilder.href("health_check", List.of(Map.of("fieldName", "component", "operator", "EQUALS", "values", List.of("assistant")))))));
    }
 }

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantMetaDataBuilder.java

@@ -27,6 +27,7 @@ import com.kingsrook.qbits.assistant.model.AssistantBudget;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetEvent;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetOverride;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetUsage;
+import com.kingsrook.qbits.assistant.model.AssistantClientError;
 import com.kingsrook.qbits.assistant.model.AssistantConversation;
 import com.kingsrook.qbits.assistant.model.AssistantDataAccess;
 import com.kingsrook.qbits.assistant.model.AssistantDataAudit;
@@ -114,7 +115,7 @@ public class AssistantMetaDataBuilder
     ** calls on a conversation; model calls on a provider) or reached from the
     ** home. Processes are table and record actions, never nav entries.
     *******************************************************************************/
-   public static final List<String> NAV_TABLES = List.of(AssistantScheduledTask.TABLE_NAME, AssistantRagSource.TABLE_NAME, AssistantBudget.TABLE_NAME, AssistantProvider.TABLE_NAME, AssistantProfile.TABLE_NAME, AssistantSkill.TABLE_NAME, AssistantDataAccess.TABLE_NAME, AssistantConversation.TABLE_NAME, AssistantModelPrice.TABLE_NAME, AssistantPolicy.TABLE_NAME);
+   public static final List<String> NAV_TABLES = List.of(AssistantScheduledTask.TABLE_NAME, AssistantRagSource.TABLE_NAME, AssistantBudget.TABLE_NAME, AssistantProvider.TABLE_NAME, AssistantProfile.TABLE_NAME, AssistantSkill.TABLE_NAME, AssistantDataAccess.TABLE_NAME, AssistantConversation.TABLE_NAME, AssistantClientError.TABLE_NAME, AssistantModelPrice.TABLE_NAME, AssistantPolicy.TABLE_NAME);
 
    private final AssistantQBitConfig config;
 
@@ -179,6 +180,9 @@ public class AssistantMetaDataBuilder
 
       tables.add(text(readOnly(owned(base(AssistantFeedback.TABLE_NAME, "Assistant Feedback", "thumbs_up_down", AssistantFeedback.class), false)), "comment"));
 
+      tables.add(text(readOnly(owned(base(AssistantClientError.TABLE_NAME, "Assistant Client Error", "bug_report", AssistantClientError.class), false)), "message", "stack", "componentStack")
+         .withRecordLabelFormat("%s #%s").withRecordLabelFields("scope", "id"));
+
       tables.add(base(AssistantPolicy.TABLE_NAME, "Assistant Policy", "policy", AssistantPolicy.class)
          .withRecordLabelFormat("%s").withRecordLabelFields("name"));
 

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantRecordScreens.java

@@ -30,6 +30,7 @@ import com.kingsrook.qbits.assistant.engine.ProviderService;
 import com.kingsrook.qbits.assistant.model.AssistantBudget;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetEvent;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetOverride;
+import com.kingsrook.qbits.assistant.model.AssistantClientError;
 import com.kingsrook.qbits.assistant.model.AssistantConversation;
 import com.kingsrook.qbits.assistant.model.AssistantDataAccess;
 import com.kingsrook.qbits.assistant.model.AssistantDataAudit;
@@ -215,6 +216,7 @@ public final class AssistantRecordScreens
          case AssistantTurnEvent.TABLE_NAME -> turnEvent(table);
          case AssistantToolCall.TABLE_NAME -> toolCall(table);
          case AssistantFeedback.TABLE_NAME -> feedback(table);
+         case AssistantClientError.TABLE_NAME -> clientError(table);
          case AssistantPolicy.TABLE_NAME -> policy(table);
          case AssistantUsageDaily.TABLE_NAME -> usageDaily(table);
          case AssistantProvider.TABLE_NAME -> provider(table);
@@ -331,6 +333,7 @@ public final class AssistantRecordScreens
          settingsSource("assistant_price_model", modelValues, true, false),
          settingsSource("assistant_provider_kind", kinds, false, false),
          enumSource("assistant_provider_location", List.of(new QPossibleValue<>("LOCAL", "Local"), new QPossibleValue<>("CLOUD", "Cloud"))),
+         enumSource("assistant_provider_setup_state", List.of(new QPossibleValue<>("ACTIVE", "Active"), new QPossibleValue<>("SETTING_UP", "Setting up"))),
          enumSource("assistant_routing_mode", List.of(new QPossibleValue<>("OFF", "Off"), new QPossibleValue<>("HEURISTIC", "Heuristic"), new QPossibleValue<>("CLASSIFIER", "Classifier"))),
          enumSource("assistant_model_kind", List.of(new QPossibleValue<>("CHAT", "Chat"), new QPossibleValue<>("EMBEDDING", "Embedding"))),
          enumSource("assistant_model_source", List.of(new QPossibleValue<>("DISCOVERED", "Discovered"), new QPossibleValue<>("MANUAL", "Manual"))),
@@ -394,6 +397,7 @@ public final class AssistantRecordScreens
       pvs(table, "model", MODEL_PVS_NAME, "Model");
       pvs(table, "kind", "assistant_provider_kind", "Kind");
       pvs(table, "location", "assistant_provider_location", "Location");
+      pvs(table, "setupState", "assistant_provider_setup_state", "Setup state");
       table.getField("hipaaEligible").withLabel("HIPAA-eligible");
       table.getField("apiVersion").withLabel("API version");
       table.getField("connectTimeoutSeconds").withLabel("Connect timeout (s)");
@@ -412,6 +416,7 @@ public final class AssistantRecordScreens
       help(table, "providerType", "Which kind of provider this is. Only the types installed on this server are offered.");
       help(table, "model", "The model answers come from. Empty uses the policy's (or the application's) main model.");
       help(table, "enabled", "Off: the assistant never uses this provider.");
+      help(table, "setupState", "Setting up: the provider is still being built, so the assistant never routes to it and its health checks do not run. Use Active once it works.");
       help(table, "isDefault", "The assistant uses the default enabled provider (else the first enabled one). Marking one default unmarks the others.");
       help(table, "apiKeyEnvName", "The NAME of the environment variable that holds the API key (e.g. ANTHROPIC_API_KEY), never the key itself. "
          + "The key stays in the server's secret store; only allowed variable names are accepted. Empty uses the type's default variable. Bedrock and Vertex require this field to be empty and use server credentials. To replace the key, change it in the secret store.");
@@ -422,7 +427,7 @@ public final class AssistantRecordScreens
       help(table, "maxConcurrentRequests", "How many model calls may run at the same time on this server. Empty: no limit.");
       table.getField(ProviderService.KEY_STATUS_FIELD).withLabel("API Key").withHelpContent(new QHelpContent("Whether the key variable is set on this server, with the key's last four characters. The key itself is never shown or returned."));
       sections(table,
-         section("provider", "Provider", "settings_input_component", Tier.T1, "id", "name", "providerType", "model", "enabled", "isDefault"),
+         section("provider", "Provider", "settings_input_component", Tier.T1, "id", "name", "providerType", "model", "enabled", "setupState", "isDefault"),
          section("connection", "Connection", "key", Tier.T2, "apiKeyEnvName", ProviderService.KEY_STATUS_FIELD, "baseUrl", "tlsCaBundle", "cloudProject"),
          section("limits", "Limits", "speed", Tier.T2, "monthlyCostCapUsd", "maxOutputTokensPerRequest", "maxConcurrentRequests"),
          section("registry", "Provider settings", "settings", Tier.T2, "kind", "location", "hipaaEligible", "region", "apiVersion", "connectTimeoutSeconds", "requestTimeoutSeconds"),
@@ -695,6 +700,21 @@ public final class AssistantRecordScreens
 
 
 
+   /*******************************************************************************
+    ** Errors the panel reported: what and where first, the stacks last.
+    *******************************************************************************/
+   private static void clientError(QTableMetaData table)
+   {
+      link(table, "messageId", AssistantMessage.TABLE_NAME, "Message");
+      link(table, "turnId", AssistantTurn.TABLE_NAME, "Turn");
+      sections(table,
+         section("error", "Error", "bug_report", Tier.T1, "id", "scope", "errorName", "message", "occurredAt"),
+         section("where", "Where", "place", Tier.T2, "route", "partType", "renderType", "appVersion", "requestId", "userId", "conversationId", "messageId", "turnId"),
+         section("stacks", "Stacks", "code", Tier.T2, "stack", "componentStack"));
+   }
+
+
+
    /*******************************************************************************
     ** The one table people edit: limits are in tokens or micro-dollars, soft
     ** limits warn and hard limits stop.

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantSettingsCustomizer.java

@@ -137,6 +137,14 @@ public class AssistantSettingsCustomizer implements TableCustomizerInterface
          {
             record.setValue("location", service.location(provider));
          }
+         if(merged.getValue("setupState") == null)
+         {
+            record.setValue("setupState", AssistantProvider.SETUP_STATE_ACTIVE);
+         }
+         else if(!List.of(AssistantProvider.SETUP_STATE_ACTIVE, AssistantProvider.SETUP_STATE_SETTING_UP).contains(merged.getValueString("setupState")))
+         {
+            record.addError(new BadInputStatusMessage("Setup state must be Active or Setting up."));
+         }
          for(String field : List.of("hipaaEligible", "connectTimeoutSeconds", "requestTimeoutSeconds"))
          {
             if(merged.getValue(field) == null)

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantClientError.java

@@ -0,0 +1,634 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** An error the panel (Next UI) hit while showing the assistant, reported by
+ ** the user's browser through POST /assistant/client-errors. Only bounded,
+ ** scrubbed fields are kept (no free text beyond them); the ids are kept only
+ ** when they are the reporting user's own rows.
+ *******************************************************************************/
+public class AssistantClientError extends QRecordEntity
+{
+   public static final String TABLE_NAME = "assistant_client_error";
+
+   @QField(isEditable = false, isPrimaryKey = true)
+   private Integer id;
+
+   @QField(isEditable = false, maxLength = 255)
+   private String userId;
+
+   @QField(isEditable = false, maxLength = 64)
+   private String scope;
+
+   @QField(isEditable = false)
+   private String message;
+
+   @QField(isEditable = false, maxLength = 100)
+   private String errorName;
+
+   @QField(isEditable = false)
+   private String stack;
+
+   @QField(isEditable = false)
+   private String componentStack;
+
+   @QField(isEditable = false, maxLength = 128)
+   private String requestId;
+
+   @QField(isEditable = false, maxLength = 300)
+   private String route;
+
+   @QField(isEditable = false, possibleValueSourceName = AssistantConversation.TABLE_NAME)
+   private Integer conversationId;
+
+   @QField(isEditable = false)
+   private Integer messageId;
+
+   @QField(isEditable = false)
+   private Integer turnId;
+
+   @QField(isEditable = false, maxLength = 64)
+   private String partType;
+
+   @QField(isEditable = false, maxLength = 64)
+   private String renderType;
+
+   @QField(isEditable = false, maxLength = 64)
+   private String appVersion;
+
+   @QField(isEditable = false)
+   private Instant occurredAt;
+
+   @QField(isEditable = false)
+   private Instant createDate;
+
+
+
+   /*******************************************************************************
+    ** Default constructor.
+    *******************************************************************************/
+   public AssistantClientError()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor that populates from a QRecord.
+    *******************************************************************************/
+   public AssistantClientError(QRecord record) throws QException
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public AssistantClientError withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userId
+    *******************************************************************************/
+   public String getUserId()
+   {
+      return (this.userId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userId
+    *******************************************************************************/
+   public void setUserId(String userId)
+   {
+      this.userId = userId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userId
+    *******************************************************************************/
+   public AssistantClientError withUserId(String userId)
+   {
+      this.userId = userId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scope
+    *******************************************************************************/
+   public String getScope()
+   {
+      return (this.scope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scope
+    *******************************************************************************/
+   public void setScope(String scope)
+   {
+      this.scope = scope;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scope
+    *******************************************************************************/
+   public AssistantClientError withScope(String scope)
+   {
+      this.scope = scope;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for message
+    *******************************************************************************/
+   public String getMessage()
+   {
+      return (this.message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for message
+    *******************************************************************************/
+   public void setMessage(String message)
+   {
+      this.message = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for message
+    *******************************************************************************/
+   public AssistantClientError withMessage(String message)
+   {
+      this.message = message;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorName
+    *******************************************************************************/
+   public String getErrorName()
+   {
+      return (this.errorName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorName
+    *******************************************************************************/
+   public void setErrorName(String errorName)
+   {
+      this.errorName = errorName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for errorName
+    *******************************************************************************/
+   public AssistantClientError withErrorName(String errorName)
+   {
+      this.errorName = errorName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for stack
+    *******************************************************************************/
+   public String getStack()
+   {
+      return (this.stack);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for stack
+    *******************************************************************************/
+   public void setStack(String stack)
+   {
+      this.stack = stack;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for stack
+    *******************************************************************************/
+   public AssistantClientError withStack(String stack)
+   {
+      this.stack = stack;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for componentStack
+    *******************************************************************************/
+   public String getComponentStack()
+   {
+      return (this.componentStack);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for componentStack
+    *******************************************************************************/
+   public void setComponentStack(String componentStack)
+   {
+      this.componentStack = componentStack;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for componentStack
+    *******************************************************************************/
+   public AssistantClientError withComponentStack(String componentStack)
+   {
+      this.componentStack = componentStack;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for requestId
+    *******************************************************************************/
+   public String getRequestId()
+   {
+      return (this.requestId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for requestId
+    *******************************************************************************/
+   public void setRequestId(String requestId)
+   {
+      this.requestId = requestId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for requestId
+    *******************************************************************************/
+   public AssistantClientError withRequestId(String requestId)
+   {
+      this.requestId = requestId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for route
+    *******************************************************************************/
+   public String getRoute()
+   {
+      return (this.route);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for route
+    *******************************************************************************/
+   public void setRoute(String route)
+   {
+      this.route = route;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for route
+    *******************************************************************************/
+   public AssistantClientError withRoute(String route)
+   {
+      this.route = route;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for conversationId
+    *******************************************************************************/
+   public Integer getConversationId()
+   {
+      return (this.conversationId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for conversationId
+    *******************************************************************************/
+   public void setConversationId(Integer conversationId)
+   {
+      this.conversationId = conversationId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for conversationId
+    *******************************************************************************/
+   public AssistantClientError withConversationId(Integer conversationId)
+   {
+      this.conversationId = conversationId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for messageId
+    *******************************************************************************/
+   public Integer getMessageId()
+   {
+      return (this.messageId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for messageId
+    *******************************************************************************/
+   public void setMessageId(Integer messageId)
+   {
+      this.messageId = messageId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for messageId
+    *******************************************************************************/
+   public AssistantClientError withMessageId(Integer messageId)
+   {
+      this.messageId = messageId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for turnId
+    *******************************************************************************/
+   public Integer getTurnId()
+   {
+      return (this.turnId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for turnId
+    *******************************************************************************/
+   public void setTurnId(Integer turnId)
+   {
+      this.turnId = turnId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for turnId
+    *******************************************************************************/
+   public AssistantClientError withTurnId(Integer turnId)
+   {
+      this.turnId = turnId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for partType
+    *******************************************************************************/
+   public String getPartType()
+   {
+      return (this.partType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for partType
+    *******************************************************************************/
+   public void setPartType(String partType)
+   {
+      this.partType = partType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for partType
+    *******************************************************************************/
+   public AssistantClientError withPartType(String partType)
+   {
+      this.partType = partType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for renderType
+    *******************************************************************************/
+   public String getRenderType()
+   {
+      return (this.renderType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for renderType
+    *******************************************************************************/
+   public void setRenderType(String renderType)
+   {
+      this.renderType = renderType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for renderType
+    *******************************************************************************/
+   public AssistantClientError withRenderType(String renderType)
+   {
+      this.renderType = renderType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for appVersion
+    *******************************************************************************/
+   public String getAppVersion()
+   {
+      return (this.appVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for appVersion
+    *******************************************************************************/
+   public void setAppVersion(String appVersion)
+   {
+      this.appVersion = appVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appVersion
+    *******************************************************************************/
+   public AssistantClientError withAppVersion(String appVersion)
+   {
+      this.appVersion = appVersion;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for occurredAt
+    *******************************************************************************/
+   public Instant getOccurredAt()
+   {
+      return (this.occurredAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for occurredAt
+    *******************************************************************************/
+   public void setOccurredAt(Instant occurredAt)
+   {
+      this.occurredAt = occurredAt;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for occurredAt
+    *******************************************************************************/
+   public AssistantClientError withOccurredAt(Instant occurredAt)
+   {
+      this.occurredAt = occurredAt;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public AssistantClientError withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+}

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantProvider.java

@@ -71,6 +71,12 @@ public class AssistantProvider extends QRecordEntity
 
    public static final String TABLE_NAME = "assistant_provider";
 
+   public static final String SETUP_STATE_ACTIVE     = "ACTIVE";
+   public static final String SETUP_STATE_SETTING_UP = "SETTING_UP";
+
+   @QField(maxLength = 16)
+   private String setupState;
+
    @QField(isEditable = false, maxLength = 8)
    private String healthState;
 
@@ -1218,4 +1224,55 @@ public class AssistantProvider extends QRecordEntity
       this.downUntil = downUntil;
       return (this);
    }
+
+
+   /*******************************************************************************
+    ** Get setupState: ACTIVE (or empty) when in use, SETTING_UP while an admin
+    ** is still building it (never routed to, no health checks).
+    *******************************************************************************/
+   public String getSetupState()
+   {
+      return (this.setupState);
+   }
+
+
+
+   /*******************************************************************************
+    ** Set setupState.
+    *******************************************************************************/
+   public void setSetupState(String setupState)
+   {
+      this.setupState = setupState;
+   }
+
+
+
+   /*******************************************************************************
+    ** With setupState.
+    *******************************************************************************/
+   public AssistantProvider withSetupState(String setupState)
+   {
+      this.setupState = setupState;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an admin marked this provider as not set up yet.
+    *******************************************************************************/
+   public boolean isSettingUp()
+   {
+      return (SETUP_STATE_SETTING_UP.equals(setupState));
+   }
+
+
+
+   /*******************************************************************************
+    ** Enabled and set up: the only rows the assistant routes to or health-checks.
+    *******************************************************************************/
+   public boolean isUsable()
+   {
+      return (Boolean.TRUE.equals(enabled) && !isSettingUp());
+   }
 }

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/persistence/AssistantStore.java

@@ -36,6 +36,7 @@ import com.kingsrook.qbits.assistant.model.AssistantBudget;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetEvent;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetOverride;
 import com.kingsrook.qbits.assistant.model.AssistantBudgetUsage;
+import com.kingsrook.qbits.assistant.model.AssistantClientError;
 import com.kingsrook.qbits.assistant.model.AssistantConversation;
 import com.kingsrook.qbits.assistant.model.AssistantDataAudit;
 import com.kingsrook.qbits.assistant.model.AssistantFeedback;
@@ -54,6 +55,7 @@ import com.kingsrook.qbits.assistant.model.AssistantUsage;
 import com.kingsrook.qbits.assistant.model.AssistantUsageDaily;
 import com.kingsrook.qbits.assistant.scheduled.ScheduledTasks;
 import com.kingsrook.qbits.assistant.spi.ModelUsage;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -61,6 +63,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.ActionFlag;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@@ -437,6 +440,23 @@ public class AssistantStore
 
 
 
+   /*******************************************************************************
+    ** Runs of any user (that the running session can see) still RUNNING or
+    ** CANCELLING although they started before a time, by id after afterId
+    ** (the retention sweep of abandoned runs).
+    *******************************************************************************/
+   public List<AssistantTurn> listRunningTurnsStartedBefore(Instant cutoff, Integer afterId, int limit) throws QException
+   {
+      return (query(AssistantTurn.TABLE_NAME, new QQueryFilter()
+         .withCriteria("status", QCriteriaOperator.IN, List.of("RUNNING", "CANCELLING"))
+         .withCriteria("startedAt", QCriteriaOperator.LESS_THAN, cutoff)
+         .withCriteria("id", QCriteriaOperator.GREATER_THAN, afterId == null ? 0 : afterId)
+         .withOrderBy(new QFilterOrderBy("id", true))
+         .withLimit(limit), AssistantTurn.class));
+   }
+
+
+
    /*******************************************************************************
     ** Persist one streamed event.
     *******************************************************************************/
@@ -726,6 +746,39 @@ public class AssistantStore
 
 
 
+   /*******************************************************************************
+    ** How many client error reports (that the running session can see) were
+    ** stored since a time.
+    *******************************************************************************/
+   public int countClientErrorsSince(Instant since) throws QException
+   {
+      Integer count = new CountAction().execute(new CountInput(AssistantClientError.TABLE_NAME)
+         .withFilter(new QQueryFilter().withCriteria("createDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, since))).getCount();
+      return (count == null ? 0 : count);
+   }
+
+
+
+   /*******************************************************************************
+    ** Delete client error reports stored before a time; returns how many.
+    *******************************************************************************/
+   public int deleteClientErrorsBefore(Instant cutoff, int limit) throws QException
+   {
+      List<AssistantClientError> errors = query(AssistantClientError.TABLE_NAME, new QQueryFilter()
+         .withCriteria("createDate", QCriteriaOperator.LESS_THAN, cutoff)
+         .withLimit(limit), AssistantClientError.class);
+      if(errors.isEmpty())
+      {
+         return (0);
+      }
+      List<Serializable> ids = new ArrayList<>();
+      errors.forEach(e -> ids.add(e.getId()));
+      new DeleteAction().execute(new DeleteInput(AssistantClientError.TABLE_NAME).withPrimaryKeys(ids));
+      return (errors.size());
+   }
+
+
+
    /*******************************************************************************
     ** A page of assistant messages created before a time, by id after afterId.
     *******************************************************************************/

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/processes/AssistantRetentionStep.java

@@ -22,6 +22,7 @@ import java.time.Instant;
 import java.util.List;
 import com.kingsrook.qbits.assistant.AssistantQBitConfig;
 import com.kingsrook.qbits.assistant.AssistantRuntime;
+import com.kingsrook.qbits.assistant.engine.AbandonedTurns;
 import com.kingsrook.qbits.assistant.engine.MessageParts;
 import com.kingsrook.qbits.assistant.model.AssistantConversation;
 import com.kingsrook.qbits.assistant.model.AssistantMessage;
@@ -41,7 +42,10 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** - deletes conversations not active for conversationRetentionDays, with
  **   their messages, turns, events, tool calls and feedback;
  ** - nulls assistantToolCall.resultJson older than toolResultRetentionDays
- **   (the audit row stays).
+ **   (the audit row stays);
+ ** - finalizes abandoned runs (still RUNNING past the hard stream limit, e.g.
+ **   after a pod restart nobody looked at) as interrupted;
+ ** - deletes client error reports older than toolResultRetentionDays.
  ** It works on the rows the running session can see: run on a schedule (the
  ** scheduler's session) or by an admin holding the owner all-access key.
  *******************************************************************************/
@@ -68,6 +72,8 @@ public class AssistantRetentionStep implements BackendStep
       output.addValue("toolResultsCleared", result.toolResultsCleared());
       output.addValue("turnEventsDeleted", result.turnEventsDeleted());
       output.addValue("messagesStripped", result.messagesStripped());
+      output.addValue("turnsFinalized", result.turnsFinalized());
+      output.addValue("clientErrorsDeleted", result.clientErrorsDeleted());
    }
 
 
@@ -75,7 +81,7 @@ public class AssistantRetentionStep implements BackendStep
    /*******************************************************************************
     ** What was purged.
     *******************************************************************************/
-   public record Result(int conversationsDeleted, int toolResultsCleared, int turnEventsDeleted, int messagesStripped)
+   public record Result(int conversationsDeleted, int toolResultsCleared, int turnEventsDeleted, int messagesStripped, int turnsFinalized, int clientErrorsDeleted)
    {
    }
 
@@ -160,9 +166,23 @@ public class AssistantRetentionStep implements BackendStep
          }
       }
 
+      int turnsFinalized = AbandonedTurns.sweep(config, store, now);
+
+      int clientErrorsDeleted = 0;
+      while(true)
+      {
+         int count = store.deleteClientErrorsBefore(toolResultCutoff, BATCH);
+         clientErrorsDeleted += count;
+         if(count < BATCH)
+         {
+            break;
+         }
+      }
+
       LOG.info("Assistant retention ran", logPair("conversationsDeleted", deleted), logPair("toolResultsCleared", cleared),
-         logPair("turnEventsDeleted", eventsDeleted), logPair("messagesStripped", messagesStripped));
-      return (new Result(deleted, cleared, eventsDeleted, messagesStripped));
+         logPair("turnEventsDeleted", eventsDeleted), logPair("messagesStripped", messagesStripped), logPair("turnsFinalized", turnsFinalized),
+         logPair("clientErrorsDeleted", clientErrorsDeleted));
+      return (new Result(deleted, cleared, eventsDeleted, messagesStripped, turnsFinalized, clientErrorsDeleted));
    }
 
 }

qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/tools/UiApiToolbox.java

@@ -223,6 +223,7 @@ public class UiApiToolbox
       private Integer            httpStatus;
       private String             auditHref;
       private Integer            pendingActionId;
+      private boolean            failed;
 
 
 
@@ -251,6 +252,17 @@ public class UiApiToolbox
 
 
 
+      /*******************************************************************************
+       ** Whether the tool failed unexpectedly (an exception, not a refusal or a
+       ** bad input): the user is told a step failed.
+       *******************************************************************************/
+      public boolean isFailed()
+      {
+         return (failed);
+      }
+
+
+
       /*******************************************************************************
        ** Getter for result
        *******************************************************************************/
@@ -483,6 +495,7 @@ public class UiApiToolbox
       }
       catch(QException | RuntimeException e)
       {
+         outcome.failed = true;
          result.withOutcome(AiToolOutcome.ERROR).withSummary("Failed").withModelContent("The tool failed: " + shortText(e.getMessage()));
       }
       finally

qbit-assistant/src/main/resources/help/qbit-assistant.yaml

@@ -358,6 +358,52 @@ entries:
   summary: The conversation the rated answer belongs to.
 - key: table:assistant_feedback;field:rating
   summary: 1 for thumbs up, -1 for thumbs down.
+- key: table:assistant_client_error;slot:help
+  summary: Errors the assistant panel hit in users' browsers, such as a result it could not display.
+  content: |
+    When the assistant panel or one of its results fails to show, the user's browser reports the error here instead of breaking the page. Each row says where it happened (the scope, page and part or render type), the error and its stacks, and links to the conversation, message and turn when they are the user's own.
+
+    Reports are scrubbed before they are stored: e-mail addresses, long numbers, tokens and query strings are removed and every field is cut to a fixed length. Each user can report a limited number per minute; more are dropped. The **Assistant client errors** check in System Health turns degraded when many arrive in an hour.
+
+    Records here are read-only and are deleted after the tool result retention period by **{{label:assistantRetention}}**.
+- key: table:assistant_client_error;section:error
+  summary: What went wrong and when the browser saw it.
+- key: table:assistant_client_error;section:where
+  summary: Where it happened (page, part and render type, build) and the conversation, message and turn it was about.
+- key: table:assistant_client_error;section:stacks
+  summary: The JavaScript and component stacks, scrubbed and shortened.
+- key: table:assistant_client_error;section:dates
+  summary: When the report was stored.
+- key: table:assistant_client_error;field:scope
+  summary: The part of the screen that failed, such as the panel, a message, a part, a render, the stream or a widget.
+- key: table:assistant_client_error;field:message
+  summary: The error message, scrubbed and cut to 500 characters.
+- key: table:assistant_client_error;field:errorName
+  summary: The kind of error the browser raised, such as TypeError.
+- key: table:assistant_client_error;field:stack
+  summary: Where in the client code the error happened, scrubbed and cut to 4,000 characters.
+- key: table:assistant_client_error;field:componentStack
+  summary: Which screen components were showing when it happened, scrubbed and cut to 4,000 characters.
+- key: table:assistant_client_error;field:requestId
+  summary: The request or trace id the client saw, to find the matching server log.
+- key: table:assistant_client_error;field:route
+  summary: The page the user was on (its path only).
+- key: table:assistant_client_error;field:conversationId
+  summary: The conversation being shown, when it is the reporting user's own.
+- key: table:assistant_client_error;field:messageId
+  summary: The message being shown, when it is the reporting user's own.
+- key: table:assistant_client_error;field:turnId
+  summary: The question-and-answer cycle being shown, when it is the reporting user's own.
+- key: table:assistant_client_error;field:partType
+  summary: The type of message part being shown, such as tool or presentation.
+- key: table:assistant_client_error;field:renderType
+  summary: The type of result being shown, such as records or widget.
+- key: table:assistant_client_error;field:appVersion
+  summary: The client build that reported the error.
+- key: table:assistant_client_error;field:occurredAt
+  summary: When the browser says the error happened.
+- key: table:assistant_client_error;field:userId
+  summary: The user whose browser reported the error.
 - key: table:assistant_policy;slot:help
   summary: The one place to set the assistant's spending limits, the kill switch, and which tools and data it may use.
   content: |
@@ -516,6 +562,10 @@ entries:
   summary: LOCAL for a server on your own network, or CLOUD for an outside service.
   content: |
     Local providers are probed for health before use. Empty uses the usual value for the kind: Ollama, LM Studio, vLLM and llama.cpp are local; everything else is cloud.
+- key: table:assistant_provider;field:setupState
+  summary: Setting up while you are still building this provider; Active once it works. Empty counts as Active.
+  content: |
+    While a provider is Setting up the assistant never sends questions to it, it is not the default even if marked so, and its provider and model health checks do not run, so an unfinished server does not show the assistant as down. Test connection and Discover models still work, so you can check it before you switch it to Active.
 - key: table:assistant_provider;field:region
   summary: The cloud region to use; only needed for AWS Bedrock and Google Vertex AI.
   content: |
@@ -1351,6 +1401,11 @@ entries:
   content: |
     - **Local**: runs on your network. Its models cost 0 unless priced, and it is the preferred automatic backup.
     - **Cloud**: a hosted service. Costs follow the price list and the monthly cost cap.
+- key: possibleValueSource:assistant_provider_setup_state;slot:help
+  summary: Whether a provider is in use or still being set up.
+  content: |
+    - **Active**: in use. The assistant routes to it and health-checks it.
+    - **Setting up**: not used yet. No routing and no health checks until you switch it to Active.
 - key: possibleValueSource:assistant_routing_mode;slot:help
   summary: How a profile chooses between its simple and complex model for each question.
   content: |

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java

@@ -90,7 +90,7 @@ class AssistantQBitProducerTest extends BaseAssistantTest
       // the nav: the home plus the main tables; processes are actions    //
       //////////////////////////////////////////////////////////////////////
       assertThat(qInstance.getApp(AssistantMetaDataBuilder.APP_NAME).getChildren()).extracting(c -> c.getName())
-         .containsExactly("assistant_scheduled_task", "assistant_rag_source", "assistant_budget", "assistant_provider", "assistant_profile", "assistant_skill", "assistant_data_access", "assistant_conversation", "assistant_model_price", "assistant_policy");
+         .containsExactly("assistant_scheduled_task", "assistant_rag_source", "assistant_budget", "assistant_provider", "assistant_profile", "assistant_skill", "assistant_data_access", "assistant_conversation", "assistant_client_error", "assistant_model_price", "assistant_policy");
       assertThat(qInstance.getApp(AssistantMetaDataBuilder.APP_NAME).getWidgets()).containsExactly("assistantHealth", "assistantHome", "assistantAccess");
       assertThat(qInstance.getProcess(AssistantMetaDataBuilder.RETENTION_PROCESS_NAME).getTableName()).isEqualTo("assistant_conversation");
       assertThat(qInstance.getProcess(AssistantMetaDataBuilder.RETENTION_PROCESS_NAME)).isNotNull();

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantTablesGuardrailTest.java

@@ -71,6 +71,7 @@ class AssistantTablesGuardrailTest extends BaseAssistantTest
       Map.entry("assistant_tool_call", "runtime: every tool call"),
       Map.entry("assistant_pending_action", "runtime: a proposed change"),
       Map.entry("assistant_feedback", "user content: a rating"),
+      Map.entry("assistant_client_error", "runtime: the panel reports an error it hit"),
       Map.entry("assistant_usage", "runtime: every model call"),
       Map.entry("assistant_usage_daily", "runtime: after every answer"),
       Map.entry("assistant_data_audit", "runtime: every model call carrying data"));

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/api/AssistantRouteProviderTest.java

@@ -179,7 +179,11 @@ class AssistantRouteProviderTest extends BaseAssistantTest
       assertThat((Map<String, Object>) config.get("features")).containsEntry("writes", false);
 
       assertThat(json(send("GET", "/config", NO_ACCESS, null))).containsEntry("enabled", false).containsEntry("reason", "NO_PERMISSION");
-      assertThat(send("GET", "/conversations", NO_ACCESS, null).statusCode()).isEqualTo(403);
+      HttpResponse<String> denied = send("GET", "/conversations", NO_ACCESS, null);
+      assertThat(denied.statusCode()).isEqualTo(403);
+      assertThat(json(denied)).containsEntry("code", "NO_PERMISSION")
+         .containsEntry("message", json(denied).get("error"))
+         .containsEntry("traceId", denied.headers().firstValue("X-QQQ-Trace-Id").orElseThrow());
 
       setKillSwitch(true);
       assertThat(json(send("GET", "/config", EDITOR, null))).containsEntry("enabled", false).containsEntry("reason", "KILL_SWITCH");
@@ -217,6 +221,9 @@ class AssistantRouteProviderTest extends BaseAssistantTest
       HttpResponse<String> foreign = send("GET", "/conversations/" + id, OTHER_EDITOR, null);
       assertThat(foreign.statusCode()).isEqualTo(404);
       assertThat(foreign.body()).doesNotContain("Posts");
+      assertThat(json(foreign)).containsEntry("code", "NOT_FOUND")
+         .containsEntry("message", json(foreign).get("error"))
+         .containsEntry("traceId", foreign.headers().firstValue("X-QQQ-Trace-Id").orElseThrow());
       assertThat(send("PATCH", "/conversations/" + id, OTHER_EDITOR, "{\"title\":\"x\"}").statusCode()).isEqualTo(404);
       assertThat(send("DELETE", "/conversations/" + id, OTHER_EDITOR, null).statusCode()).isEqualTo(404);
       assertThat(send("POST", "/conversations/" + id + "/turns", OTHER_EDITOR, "{\"text\":\"hi\"}").statusCode()).isEqualTo(404);
@@ -238,7 +245,11 @@ class AssistantRouteProviderTest extends BaseAssistantTest
       HttpRequest form = HttpRequest.newBuilder(URI.create(base + "/conversations")).header("Cookie", "sessionId=" + EDITOR)
          .header("Content-Type", "application/x-www-form-urlencoded").POST(HttpRequest.BodyPublishers.ofString("title=x")).build();
       assertThat(client.send(form, HttpResponse.BodyHandlers.ofString()).statusCode()).isEqualTo(400);
-      assertThat(send("GET", "/conversations/abc", EDITOR, null).statusCode()).isEqualTo(400);
+      HttpResponse<String> invalid = send("GET", "/conversations/abc", EDITOR, null);
+      assertThat(invalid.statusCode()).isEqualTo(400);
+      assertThat(json(invalid)).containsEntry("code", "BAD_REQUEST")
+         .containsEntry("message", json(invalid).get("error"))
+         .containsEntry("traceId", invalid.headers().firstValue("X-QQQ-Trace-Id").orElseThrow());
       assertThat(send("POST", "/conversations/" + id + "/turns", EDITOR, "{}").statusCode()).isEqualTo(400);
 
       assertThat(send("DELETE", "/conversations/" + id, EDITOR, null).statusCode()).isEqualTo(204);

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/api/ClientErrorsTest.java

@@ -0,0 +1,261 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.api;
+
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.kingsrook.qbits.assistant.AssistantQBitConfig;
+import com.kingsrook.qbits.assistant.engine.ClientErrors;
+import com.kingsrook.qbits.assistant.health.AssistantHealthChecks;
+import com.kingsrook.qbits.assistant.metadata.AssistantMetaDataBuilder;
+import com.kingsrook.qbits.assistant.model.AssistantClientError;
+import com.kingsrook.qbits.assistant.model.AssistantConversation;
+import com.kingsrook.qbits.assistant.persistence.AssistantStore;
+import com.kingsrook.qbits.assistant.processes.AssistantRetentionStep;
+import com.kingsrook.qbits.assistant.testing.AssistantTestFixture;
+import com.kingsrook.qbits.assistant.testing.BaseAssistantTest;
+import com.kingsrook.qbits.assistant.testing.UiApiTestServer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthStatus;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.assistant.testing.AssistantTestFixture.EDITOR;
+import static com.kingsrook.qbits.assistant.testing.AssistantTestFixture.OTHER_EDITOR;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** POST /assistant/client-errors: bounds and scrubbing, owned ids only, the
+ ** per-user rate limit (202 accepted:false), owner security, the HTTP
+ ** contract, the health check and retention.
+ *******************************************************************************/
+class ClientErrorsTest extends BaseAssistantTest
+{
+   /*******************************************************************************
+    ** A small limit and warn threshold keep the tests short.
+    *******************************************************************************/
+   @Override
+   protected void customizeConfig(AssistantQBitConfig config)
+   {
+      config.withClientErrorsPerUserPerMinute(3).withClientErrorWarnPerHour(2);
+   }
+
+
+
+   /*******************************************************************************
+    ** The API bound to a user's own session (in QContext).
+    *******************************************************************************/
+   private AssistantApi as(String userId)
+   {
+      QSession session = AssistantTestFixture.ownerSession(userId, config);
+      QContext.init(qInstance, session);
+      return (new AssistantApi(config.getRuntime(), session));
+   }
+
+
+
+   /*******************************************************************************
+    ** The user's stored reports.
+    *******************************************************************************/
+   private List<AssistantClientError> rows(String userId) throws Exception
+   {
+      as(userId);
+      return (new AssistantStore().query(AssistantClientError.TABLE_NAME, new QQueryFilter(), AssistantClientError.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every field is bounded and the free text scrubbed; ids are kept only when
+    ** they are the user's own.
+    *******************************************************************************/
+   @Test
+   void storesBoundedScrubbedReports() throws Exception
+   {
+      AssistantApi          api   = as(EDITOR);
+      AssistantConversation mine  = config.getRuntime().getStore().createConversation(EDITOR, "mine", null);
+      as(OTHER_EDITOR);
+      AssistantConversation other = config.getRuntime().getStore().createConversation(OTHER_EDITOR, "other", null);
+      api = as(EDITOR);
+
+      Map<String, Object> body = new LinkedHashMap<>();
+      body.put("scope", "assistant-render" + "x".repeat(100));
+      body.put("message", "Cannot read properties of undefined (reading 'length') for jane.doe@example.com card 4111 1111 1111 1111 " + "y".repeat(800));
+      body.put("errorName", "TypeError");
+      body.put("stack", "at render (https://app.example.com/_next/chunk.js?token=abc:1:2)\n" + "z".repeat(5000));
+      body.put("componentStack", "in RecordsRender\n in AssistantPart key 9f8e7d6c5b4a3928170655e4d3c2b1a0AbCdEfQx");
+      body.put("requestId", "req-123");
+      body.put("route", "https://app.example.com/app/orders/12345678?filter=secret#top");
+      body.put("conversationId", mine.getId());
+      body.put("messageId", 999_999);
+      body.put("turnId", "12");
+      body.put("partType", "tool");
+      body.put("renderType", "records");
+      body.put("appVersion", "1.2.3");
+      body.put("occurredAt", "2026-10-15T12:00:00Z");
+      body.put("unknownField", "ignored");
+      assertThat(api.reportClientError(body)).isEqualTo(Map.of("accepted", true));
+
+      body.put("conversationId", other.getId());
+      body.put("route", "javascript:alert(1)");
+      body.put("occurredAt", "not a date");
+      assertThat(as(EDITOR).reportClientError(body)).containsEntry("accepted", true);
+
+      List<AssistantClientError> rows = rows(EDITOR);
+      assertThat(rows).hasSize(2);
+      AssistantClientError row = rows.get(0);
+      assertThat(row.getUserId()).isEqualTo(EDITOR);
+      assertThat(row.getScope()).hasSize(ClientErrors.SCOPE_MAX).startsWith("assistant-render");
+      assertThat(row.getMessage()).hasSize(ClientErrors.MESSAGE_MAX).startsWith("Cannot read properties of undefined (reading 'length') for [email] card [")
+         .doesNotContain("jane.doe").doesNotContain("4111");
+      assertThat(ClientErrors.text("order 12345678 for 555 123 4567, version 1.2.3", 100)).isEqualTo("order [number] for [number], version 1.2.3");
+      assertThat(row.getStack()).hasSize(ClientErrors.STACK_MAX).contains("chunk.js?[query]").doesNotContain("token=abc");
+      assertThat(row.getComponentStack()).contains("[token]").doesNotContain("9f8e7d6c");
+      assertThat(row.getRoute()).isEqualTo("/app/orders/[number]");
+      assertThat(row.getConversationId()).isEqualTo(mine.getId());
+      assertThat(row.getMessageId()).isNull();
+      assertThat(row.getTurnId()).isNull();
+      assertThat(row.getPartType()).isEqualTo("tool");
+      assertThat(row.getRenderType()).isEqualTo("records");
+      assertThat(row.getAppVersion()).isEqualTo("1.2.3");
+      assertThat(row.getOccurredAt()).isEqualTo(Instant.parse("2026-10-15T12:00:00Z"));
+
+      AssistantClientError second = rows.get(1);
+      assertThat(second.getConversationId()).as("another user's conversation is not linked").isNull();
+      assertThat(second.getRoute()).isNull();
+      assertThat(second.getOccurredAt()).isNull();
+
+      ///////////////////////////////////////////////////////////
+      // owner-locked: another user never sees these reports   //
+      ///////////////////////////////////////////////////////////
+      assertThat(rows(OTHER_EDITOR)).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** scope and message are required; everything else is optional.
+    *******************************************************************************/
+   @Test
+   void requiresScopeAndMessage()
+   {
+      AssistantApi api = as(EDITOR);
+      assertThatThrownBy(() -> api.reportClientError(null)).isInstanceOf(AssistantApiException.class).extracting("status").isEqualTo(400);
+      assertThatThrownBy(() -> api.reportClientError(Map.of("message", "x"))).extracting("status").isEqualTo(400);
+      assertThatThrownBy(() -> api.reportClientError(Map.of("scope", "page", "message", "   "))).extracting("status").isEqualTo(400);
+      assertThatThrownBy(() -> api.reportClientError(Map.of("scope", 7, "message", "x"))).extracting("status").isEqualTo(400);
+   }
+
+
+
+   /*******************************************************************************
+    ** Over the per-user limit: still acknowledged (accepted:false), nothing
+    ** stored; other users have their own limit.
+    *******************************************************************************/
+   @Test
+   void rateLimitedPerUser() throws Exception
+   {
+      Map<String, Object> body = Map.of("scope", "page", "message", "boom");
+      for(int i = 0; i < 3; i++)
+      {
+         assertThat(as(EDITOR).reportClientError(body)).containsEntry("accepted", true);
+      }
+      assertThat(as(EDITOR).reportClientError(body)).isEqualTo(Map.of("accepted", false));
+      assertThat(as(OTHER_EDITOR).reportClientError(body)).containsEntry("accepted", true);
+      assertThat(rows(EDITOR)).hasSize(3);
+   }
+
+
+
+   /*******************************************************************************
+    ** The health check is UP below the threshold and DEGRADED at it.
+    *******************************************************************************/
+   @Test
+   void healthCheckWarnsOnManyReports() throws Exception
+   {
+      assertThat(HealthChecksMetaData.ofOrWithNew(qInstance).getChecks()).anySatisfy(check -> assertThat(check.getName()).isEqualTo(AssistantHealthChecks.CLIENT_ERRORS));
+      as(EDITOR);
+      assertThat(AssistantHealthChecks.clientErrors(config.getRuntime()).getStatus()).isEqualTo(HealthStatus.UP);
+      as(EDITOR).reportClientError(Map.of("scope", "page", "message", "one"));
+      as(EDITOR).reportClientError(Map.of("scope", "page", "message", "two"));
+      as(EDITOR);
+      assertThat(AssistantHealthChecks.clientErrors(config.getRuntime()).getStatus()).isEqualTo(HealthStatus.DEGRADED);
+      assertThat(AssistantHealthChecks.clientErrors(null).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+   }
+
+
+
+   /*******************************************************************************
+    ** Retention deletes reports older than the tool result retention.
+    *******************************************************************************/
+   @Test
+   void retentionDeletesOldReports() throws Exception
+   {
+      as(EDITOR).reportClientError(Map.of("scope", "page", "message", "old"));
+      as(EDITOR);
+      AssistantRetentionStep.Result kept = AssistantRetentionStep.purge(config, config.getRuntime().getStore(), Instant.now());
+      assertThat(kept.clientErrorsDeleted()).isZero();
+      AssistantRetentionStep.Result result = AssistantRetentionStep.purge(config, config.getRuntime().getStore(), Instant.now().plus(Duration.ofDays(config.getToolResultRetentionDays() + 1)));
+      assertThat(result.clientErrorsDeleted()).isEqualTo(1);
+      assertThat(rows(EDITOR)).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Over HTTP: 202 {accepted}, a non-object body or a missing field is a 400;
+    ** the table is in the Assistant nav. (The session comes from the standard
+    ** UI API authentication; the fixture's MOCK module signs anyone in.)
+    *******************************************************************************/
+   @Test
+   void httpContract() throws Exception
+   {
+      QContext.clear();
+      String     base   = UiApiTestServer.serve(qInstance, config) + AssistantRouteProvider.BASE_PATH;
+      HttpClient client = HttpClient.newHttpClient();
+
+      HttpResponse<String> accepted = client.send(HttpRequest.newBuilder(URI.create(base + "/client-errors")).header("Cookie", "sessionId=" + EDITOR)
+         .header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString("{\"scope\":\"assistant-part\",\"message\":\"Cannot read properties of undefined\"}")).build(),
+         HttpResponse.BodyHandlers.ofString());
+      assertThat(accepted.statusCode()).isEqualTo(202);
+      assertThat(JsonUtils.toObject(accepted.body(), new TypeReference<Map<String, Object>>() {})).isEqualTo(Map.of("accepted", true));
+
+      HttpResponse<String> notObject = client.send(HttpRequest.newBuilder(URI.create(base + "/client-errors")).header("Cookie", "sessionId=" + EDITOR)
+         .header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString("[1,2]")).build(), HttpResponse.BodyHandlers.ofString());
+      assertThat(notObject.statusCode()).isEqualTo(400);
+
+      HttpResponse<String> missing = client.send(HttpRequest.newBuilder(URI.create(base + "/client-errors")).header("Cookie", "sessionId=" + EDITOR)
+         .header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString("{\"scope\":\"page\"}")).build(), HttpResponse.BodyHandlers.ofString());
+      assertThat(missing.statusCode()).isEqualTo(400);
+
+      assertThat(rows(EDITOR)).hasSize(1);
+      assertThat(AssistantMetaDataBuilder.NAV_TABLES).contains(AssistantClientError.TABLE_NAME);
+   }
+}

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/api/specs/AssistantSpecsContractTest.java

@@ -65,8 +65,17 @@ class AssistantSpecsContractTest extends BaseAssistantTest
 
       assertThat(generated).contains("/qqq/v1/assistant/conversations/{id}/turns").contains("text/event-stream").contains("/qqq/v1/assistant/actions/{id}/confirm")
          .contains("AssistantTurnEventV1").contains("/qqq/v1/assistant/access");
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // The standard error schema retains error/code and adds message/traceId/fieldErrors. //
+      // The two 204 responses never had bodies; their old refs pointed to schema "null".   //
+      // Keep the release baseline and accept only these documented migration corrections. //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      List<String> acceptedChanges = List.of(
+         "$ref changed from #/components/schemas/AssistantErrorV1 to #/components/schemas/BasicErrorResponseV1",
+         "DELETE /qqq/v1/assistant/conversations/{id} response 204: content type removed: application/json",
+         "POST /qqq/v1/assistant/messages/{id}/feedback response 204: content type removed: application/json");
       List<String> problems = UiApiContract.check(generated, Files.readString(COMMITTED, StandardCharsets.UTF_8),
-         Files.exists(BASELINE) ? Files.readString(BASELINE, StandardCharsets.UTF_8) : null, 1, "./mvnw test -pl qbit-assistant -Dtest=AssistantSpecsContractTest -Dassistant.regenerateOpenApi=true");
+         Files.exists(BASELINE) ? Files.readString(BASELINE, StandardCharsets.UTF_8) : null, 1, "./mvnw test -pl qbit-assistant -Dtest=AssistantSpecsContractTest -Dassistant.regenerateOpenApi=true", acceptedChanges);
       assertThat(problems).isEmpty();
    }
 

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/DurableTurnsTest.java

@@ -208,6 +208,42 @@ class DurableTurnsTest extends BaseAssistantTest
 
 
 
+   /*******************************************************************************
+    ** Streamed text arrives on the provider's own thread; the "writing" progress
+    ** it triggers still runs in the turn's QContext (the job tracker writes
+    ** through QQQ, which needs the instance).
+    *******************************************************************************/
+   @Test
+   void streamedProgressRunsInTheTurnContext() throws Exception
+   {
+      List<String> contexts = new CopyOnWriteArrayList<>();
+      config.setRunTracker(new AssistantRunTracker()
+      {
+         @Override
+         public String start(String userId, Integer turnId, Integer conversationId, String label)
+         {
+            return ("job-" + turnId);
+         }
+
+         @Override
+         public void progress(String trackingId, String phase, String message)
+         {
+            contexts.add(phase + ":" + (QContext.getQInstance() != null));
+         }
+
+         @Override
+         public void finish(String trackingId, String status, String message)
+         {
+         }
+      });
+      provider.thenText("A streamed answer on the provider's thread");
+      AssistantTurn row = completed(api.startTurn(conversation(), Map.of("text", "hello")));
+      assertThat(row.getStatus()).isEqualTo("COMPLETE");
+      assertThat(contexts).contains("writing:true").doesNotContain("writing:false");
+   }
+
+
+
    /*******************************************************************************
     ** A throwing post-processing collaborator cannot strand a streamed answer.
     *******************************************************************************/

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/FailSoftTurnsTest.java

@@ -0,0 +1,347 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.engine;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import com.kingsrook.qbits.assistant.AssistantQBitConfig;
+import com.kingsrook.qbits.assistant.api.AssistantApi;
+import com.kingsrook.qbits.assistant.model.AssistantConversation;
+import com.kingsrook.qbits.assistant.model.AssistantMessage;
+import com.kingsrook.qbits.assistant.model.AssistantTurn;
+import com.kingsrook.qbits.assistant.persistence.AssistantStore;
+import com.kingsrook.qbits.assistant.processes.AssistantRetentionStep;
+import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider;
+import com.kingsrook.qbits.assistant.spi.ModelEventListener;
+import com.kingsrook.qbits.assistant.spi.ModelMessage;
+import com.kingsrook.qbits.assistant.spi.ModelPart;
+import com.kingsrook.qbits.assistant.spi.ModelRequest;
+import com.kingsrook.qbits.assistant.spi.StreamHandle;
+import com.kingsrook.qbits.assistant.testing.AssistantTestFixture;
+import com.kingsrook.qbits.assistant.testing.BaseAssistantTest;
+import com.kingsrook.qbits.assistant.testing.TurnRunner;
+import com.kingsrook.qbits.assistant.uiapi.UiApiCall;
+import com.kingsrook.qbits.assistant.uiapi.UiApiGateway;
+import com.kingsrook.qbits.assistant.uiapi.UiApiResult;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.assistant.testing.AssistantTestFixture.EDITOR;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Fail soft: tool failures and timeouts become ERROR (is_error) results plus
+ ** an error notice; any exception in the loop still ends the turn (ERROR,
+ ** message.end, terminal row, unregistered); abandoned runs and stuck
+ ** messages are finalized when read or swept.
+ *******************************************************************************/
+class FailSoftTurnsTest extends BaseAssistantTest
+{
+   private static final String INTERNAL_DETAIL = "INTERNAL-DETAIL-7731";
+
+   /** what the UI API gateway does: THROW (an unexpected exception) or SLEEP (hang) */
+   private final AtomicReference<String> gatewayMode = new AtomicReference<>("THROW");
+
+   /** whether the provider's stream() throws */
+   private volatile boolean providerThrows = false;
+
+
+
+   /*******************************************************************************
+    ** A gateway that fails or hangs, a short tool limit, and a provider whose
+    ** stream() can throw outright (not a ModelProviderException).
+    *******************************************************************************/
+   @Override
+   protected void customizeConfig(AssistantQBitConfig config)
+   {
+      config.withToolCallTimeoutSeconds(1);
+      config.withUiApiGateway(new UiApiGateway()
+      {
+         @Override
+         public boolean isInvocable(String method, String pathTemplate)
+         {
+            return (true);
+         }
+
+
+
+         @Override
+         public UiApiResult invoke(UiApiCall call) throws QException
+         {
+            if("SLEEP".equals(gatewayMode.get()))
+            {
+               try
+               {
+                  Thread.sleep(30_000);
+               }
+               catch(InterruptedException e)
+               {
+                  Thread.currentThread().interrupt();
+                  throw (new QException("interrupted"));
+               }
+            }
+            throw (new IllegalStateException(INTERNAL_DETAIL));
+         }
+      });
+      provider = new ScriptedModelProvider()
+      {
+         @Override
+         public StreamHandle stream(ModelRequest request, ModelEventListener listener)
+         {
+            if(providerThrows)
+            {
+               throw (new IllegalStateException(INTERNAL_DETAIL));
+            }
+            return (super.stream(request, listener));
+         }
+      };
+      config.withModelProvider(provider);
+   }
+
+
+
+   /*******************************************************************************
+    ** The tool results the model got back in its second request.
+    *******************************************************************************/
+   private List<ModelPart> toolResultsSentBack()
+   {
+      ModelRequest second = provider.getRequests().get(1);
+      ModelMessage results = second.getMessages().get(second.getMessages().size() - 1);
+      return (results.getParts());
+   }
+
+
+
+   /*******************************************************************************
+    ** A tool that throws: an ERROR is_error result for the model, an ERROR tool
+    ** part and an error notice for the user, no internal detail in either.
+    *******************************************************************************/
+   @Test
+   void toolExceptionBecomesErrorPart() throws Exception
+   {
+      provider.thenToolUse("count_records", input("table", "blog_post", "filter", filter("AND"))).thenText("Sorry, I could not count them.");
+      TurnRunner.Outcome outcome = runner.run(EDITOR, "how many posts?");
+
+      assertThat(outcome.end()).containsEntry("status", "COMPLETE");
+      assertThat(outcome.events(TurnEvent.TOOL_RESULT)).singleElement().satisfies(result -> assertThat(result).containsEntry("outcome", "ERROR").containsEntry("summary", "Failed"));
+      assertThat(outcome.events(TurnEvent.NOTICE)).anySatisfy(notice -> assertThat(notice).containsEntry("level", "error"));
+      assertThat(toolResultsSentBack()).singleElement().satisfies(part -> assertThat(part.getIsError()).isTrue());
+
+      List<Map<String, Object>> parts = assistantParts(outcome.turnId());
+      assertThat(parts).anySatisfy(part -> assertThat(part).containsEntry("type", "tool").containsEntry("outcome", "ERROR"));
+      assertThat(parts).anySatisfy(part -> assertThat(part).containsEntry("type", "notice").containsEntry("level", "error"));
+      assertThat(String.valueOf(parts)).doesNotContain(INTERNAL_DETAIL);
+      outcome.events().forEach(event -> assertThat(event.toJson()).doesNotContain(INTERNAL_DETAIL));
+   }
+
+
+
+   /*******************************************************************************
+    ** A tool that hangs past toolCallTimeoutSeconds gets an ERROR (is_error)
+    ** result; the turn goes on and completes.
+    *******************************************************************************/
+   @Test
+   void toolTimeoutBecomesErrorResult() throws Exception
+   {
+      gatewayMode.set("SLEEP");
+      provider.thenToolUse("count_records", input("table", "blog_post", "filter", filter("AND"))).thenText("That took too long.");
+      long started = System.currentTimeMillis();
+      TurnRunner.Outcome outcome = runner.run(EDITOR, "how many posts?");
+
+      assertThat(System.currentTimeMillis() - started).isLessThan(20_000);
+      assertThat(outcome.end()).containsEntry("status", "COMPLETE");
+      assertThat(outcome.events(TurnEvent.TOOL_RESULT)).singleElement().satisfies(result -> assertThat(result).containsEntry("outcome", "ERROR").containsEntry("summary", "Timed out"));
+      assertThat(outcome.events(TurnEvent.NOTICE)).anySatisfy(notice -> assertThat(notice).containsEntry("level", "error"));
+      assertThat(toolResultsSentBack()).singleElement().satisfies(part ->
+      {
+         assertThat(part.getIsError()).isTrue();
+         assertThat(part.getText()).contains("did not finish in time");
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** The turn's own limit: status ERROR, an error event with code TIMEOUT,
+    ** then message.end.
+    *******************************************************************************/
+   @Test
+   void turnTimeoutEndsWithTimeoutError() throws Exception
+   {
+      config.setMaxTurnSeconds(1);
+      provider.withDelay(Duration.ofSeconds(3)).thenText("too late");
+      TurnRunner.Outcome outcome = runner.run(EDITOR, "slow");
+      assertThat(outcome.end()).containsEntry("status", "ERROR");
+      assertThat(outcome.events(TurnEvent.ERROR)).singleElement().satisfies(error -> assertThat(error).containsEntry("code", "TIMEOUT"));
+      List<TurnEvent> events = outcome.events();
+      assertThat(events.getLast().getType()).isEqualTo(TurnEvent.MESSAGE_END);
+      assertThat(turnRow(outcome.turnId()).getStatus()).isEqualTo("ERROR");
+   }
+
+
+
+   /*******************************************************************************
+    ** An unexpected exception anywhere in the loop: ERROR, message.end, the turn
+    ** row and the message finalized, the run unregistered, and the next
+    ** question is not locked out.
+    *******************************************************************************/
+   @Test
+   void loopExceptionStillEndsTheTurn() throws Exception
+   {
+      providerThrows = true;
+      TurnRunner.Outcome failed = runner.run(EDITOR, "hello");
+      assertThat(failed.end()).containsEntry("status", "ERROR");
+      assertThat(failed.errorCode()).isEqualTo("INTERNAL");
+      assertThat(failed.events().getLast().getType()).isEqualTo(TurnEvent.MESSAGE_END);
+      assertThat(String.valueOf(failed.events())).doesNotContain(INTERNAL_DETAIL);
+      assertThat(runner.runtime().getRunningTurn(failed.turnId())).isNull();
+
+      AssistantTurn row = turnRow(failed.turnId());
+      assertThat(row.getStatus()).isEqualTo("ERROR");
+      assertThat(row.getErrorCode()).isEqualTo("INTERNAL");
+      assertThat(row.getEndedAt()).isNotNull();
+      assertThat(message(row.getAssistantMessageId()).getStatus()).isEqualTo("ERROR");
+      assertThat(assistantParts(failed.turnId())).anySatisfy(part -> assertThat(part).containsEntry("type", "notice").containsEntry("level", "error"));
+
+      providerThrows = false;
+      provider.thenText("Fine now.");
+      assertThat(runner.run(EDITOR, failed.conversationId(), "again", null).end()).containsEntry("status", "COMPLETE");
+   }
+
+
+
+   /*******************************************************************************
+    ** Conversation detail repairs a message left STREAMING: by a run that
+    ** already ended (its status), and by an abandoned run (reaped, ERROR).
+    ** Legacy parts come back normalized.
+    *******************************************************************************/
+   @Test
+   @SuppressWarnings("unchecked")
+   void conversationDetailRepairsStuckMessages() throws Exception
+   {
+      QSession session = AssistantTestFixture.ownerSession(EDITOR, config);
+      QContext.init(qInstance, session);
+      AssistantStore        store        = config.getRuntime().getStore();
+      AssistantApi          api          = new AssistantApi(config.getRuntime(), session);
+      AssistantConversation conversation = store.createConversation(EDITOR, "stuck", null);
+
+      AssistantTurn ended = store.insert(new AssistantTurn().withConversationId(conversation.getId()).withUserId(EDITOR).withStatus("COMPLETE").withStartedAt(Instant.now()));
+      store.insert(new AssistantMessage().withConversationId(conversation.getId()).withUserId(EDITOR).withRole("ASSISTANT").withStatus("STREAMING").withSeq(2)
+         .withTurnId(ended.getId()).withPartsJson("[{\"type\":\"tool\",\"name\":\"query_records\",\"render\":{\"type\":\"records\",\"tableName\":\"t\"}}]"));
+
+      AssistantTurn abandoned = store.insert(new AssistantTurn().withConversationId(conversation.getId()).withUserId(EDITOR).withStatus("RUNNING")
+         .withStartedAt(Instant.now().minus(Duration.ofHours(1))).withHeartbeatAt(Instant.now().minus(Duration.ofHours(1))));
+      AssistantMessage stuck = store.insert(new AssistantMessage().withConversationId(conversation.getId()).withUserId(EDITOR).withRole("ASSISTANT").withStatus("STREAMING").withSeq(4)
+         .withTurnId(abandoned.getId()).withPartsJson("[{\"type\":\"text\",\"text\":\"Partial\"}]"));
+      store.updateFields(AssistantTurn.TABLE_NAME, new QRecord().withValue("id", abandoned.getId()).withValue("assistantMessageId", stuck.getId()));
+
+      AssistantMessage orphan = store.insert(new AssistantMessage().withConversationId(conversation.getId()).withUserId(EDITOR).withRole("ASSISTANT").withStatus("STREAMING").withSeq(6)
+         .withPartsJson("[]"));
+
+      List<Map<String, Object>> messages = (List<Map<String, Object>>) api.getConversation(conversation.getId(), null, null).get("messages");
+      assertThat(messages).extracting(m -> m.get("status")).containsExactly("COMPLETE", "ERROR", "ERROR");
+      Map<String, Object> render = (Map<String, Object>) ((List<Map<String, Object>>) messages.get(0).get("parts")).get(0).get("render");
+      assertThat(render).containsEntry("rows", List.of()).containsEntry("columns", List.of());
+      assertThat(String.valueOf(messages.get(1).get("parts"))).contains(AbandonedTurns.INTERRUPTED_TEXT);
+      assertThat(String.valueOf(messages.get(2).get("parts"))).contains(AbandonedTurns.UNFINISHED_TEXT);
+
+      assertThat(store.getTurn(EDITOR, abandoned.getId()).getStatus()).isEqualTo("ERROR");
+      assertThat(store.getMessage(EDITOR, orphan.getId()).getStatus()).isEqualTo("ERROR");
+      assertThat(api.activeTurns().get("turns")).isEqualTo(List.of());
+   }
+
+
+
+   /*******************************************************************************
+    ** The retention process finalizes runs nobody read again (e.g. after a pod
+    ** restart), with their messages and a terminal replay.
+    *******************************************************************************/
+   @Test
+   void retentionSweepsAbandonedRuns() throws Exception
+   {
+      QSession session = AssistantTestFixture.ownerSession(EDITOR, config);
+      QContext.init(qInstance, session);
+      AssistantStore        store        = config.getRuntime().getStore();
+      AssistantConversation conversation = store.createConversation(EDITOR, "restarted", null);
+      AssistantMessage message = store.insert(new AssistantMessage().withConversationId(conversation.getId()).withUserId(EDITOR).withRole("ASSISTANT").withStatus("STREAMING").withSeq(2)
+         .withPartsJson("[]"));
+      AssistantTurn old = store.insert(new AssistantTurn().withConversationId(conversation.getId()).withUserId(EDITOR).withStatus("RUNNING")
+         .withAssistantMessageId(message.getId()).withStartedAt(Instant.now().minus(Duration.ofHours(2))));
+      AssistantTurn fresh = store.insert(new AssistantTurn().withConversationId(conversation.getId()).withUserId(EDITOR).withStatus("RUNNING")
+         .withStartedAt(Instant.now()).withHeartbeatAt(Instant.now()));
+
+      AssistantRetentionStep.Result result = AssistantRetentionStep.purge(config, store, Instant.now());
+      assertThat(result.turnsFinalized()).isEqualTo(1);
+      assertThat(store.getTurn(EDITOR, old.getId()).getStatus()).isEqualTo("ERROR");
+      assertThat(store.getTurn(EDITOR, fresh.getId()).getStatus()).isEqualTo("RUNNING");
+      assertThat(store.getMessage(EDITOR, message.getId()).getStatus()).isEqualTo("ERROR");
+      assertThat(store.listEvents(EDITOR, old.getId(), 0, 10)).extracting(e -> e.getEventType()).containsExactly(TurnEvent.ERROR, TurnEvent.MESSAGE_END);
+   }
+
+
+
+   /*******************************************************************************
+    ** The turn row, read as its owner.
+    *******************************************************************************/
+   private AssistantTurn turnRow(Integer turnId) throws Exception
+   {
+      QContext.init(qInstance, AssistantTestFixture.ownerSession(EDITOR, config));
+      try
+      {
+         return (config.getRuntime().getStore().getTurn(EDITOR, turnId));
+      }
+      finally
+      {
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A message, read as its owner.
+    *******************************************************************************/
+   private AssistantMessage message(Integer messageId) throws Exception
+   {
+      QContext.init(qInstance, AssistantTestFixture.ownerSession(EDITOR, config));
+      try
+      {
+         return (config.getRuntime().getStore().getMessage(EDITOR, messageId));
+      }
+      finally
+      {
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The stored parts of a turn's assistant message.
+    *******************************************************************************/
+   private List<Map<String, Object>> assistantParts(Integer turnId) throws Exception
+   {
+      return (MessageParts.fromJson(message(turnRow(turnId).getAssistantMessageId()).getPartsJson()).list());
+   }
+}

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/RenderShapesTest.java

@@ -0,0 +1,208 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.assistant.engine;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Records renders keep their (empty) lists in stored parts and stream events,
+ ** and parts stored without them are completed when read.
+ *******************************************************************************/
+class RenderShapesTest
+{
+
+   /*******************************************************************************
+    ** A query that matched nothing.
+    *******************************************************************************/
+   private static Map<String, Object> emptyRecordsPart()
+   {
+      Map<String, Object> render = new LinkedHashMap<>();
+      render.put("type", "records");
+      render.put("tableName", "db_perf_finding");
+      render.put("tableLabel", "Findings");
+      render.put("columns", new ArrayList<>());
+      render.put("rows", new ArrayList<>());
+      render.put("totalCount", 0);
+      render.put("truncated", false);
+      render.put("openHref", "/app/db_perf_finding");
+      Map<String, Object> part = new LinkedHashMap<>();
+      part.put("type", "tool");
+      part.put("name", "query_records");
+      part.put("render", render);
+      return (part);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void emptyListsSurviveSerialization()
+   {
+      String json = RenderShapes.toJson(List.of(emptyRecordsPart()));
+      assertThat(json).contains("\"columns\":[]").contains("\"rows\":[]");
+      assertThat(new TurnEvent(1, "tool.result", emptyRecordsPart()).toJson()).contains("\"rows\":[]").contains("\"columns\":[]");
+   }
+
+
+
+   /*******************************************************************************
+    ** The exact shape stored on kof22 before the fix (lists omitted).
+    *******************************************************************************/
+   @Test
+   void partsStoredWithoutListsAreCompletedWhenRead()
+   {
+      String stored = "[{\"type\":\"tool\",\"name\":\"query_records\",\"outcome\":\"OK\",\"render\":{\"type\":\"records\",\"tableName\":\"db_perf_finding\",\"tableLabel\":\"Findings\",\"openHref\":\"/app/db_perf_finding\",\"totalCount\":0,\"truncated\":false}},"
+         + "{\"type\":\"tool\",\"name\":\"history\",\"render\":{\"type\":\"history\",\"entries\":[{\"message\":\"Edited\"}]}}]";
+      List<Map<String, Object>> parts = MessageParts.fromJson(stored).list();
+      @SuppressWarnings("unchecked")
+      Map<String, Object> records = (Map<String, Object>) parts.get(0).get("render");
+      assertThat(records.get("rows")).isEqualTo(List.of());
+      assertThat(records.get("columns")).isEqualTo(List.of());
+      @SuppressWarnings("unchecked")
+      Map<String, Object> history = (Map<String, Object>) parts.get(1).get("render");
+      @SuppressWarnings("unchecked")
+      Map<String, Object> entry = ((List<Map<String, Object>>) history.get("entries")).get(0);
+      assertThat(entry.get("details")).isEqualTo(List.of());
+
+      Map<String, Object> event = TurnEvent.fromJson(3, "tool.result", "{\"render\":{\"type\":\"records\",\"tableLabel\":\"Findings\"}}").getData();
+      @SuppressWarnings("unchecked")
+      Map<String, Object> eventRender = (Map<String, Object>) event.get("render");
+      assertThat(eventRender.get("rows")).isEqualTo(List.of());
+   }
+
+
+
+   /*******************************************************************************
+    ** One render through the normalizer, as a stored part would be read.
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   private static Map<String, Object> renderOf(String renderJson)
+   {
+      List<Map<String, Object>> parts = MessageParts.fromJson("[{\"type\":\"tool\",\"name\":\"t\",\"render\":" + renderJson + "}]").list();
+      assertThat(parts).hasSize(1);
+      return ((Map<String, Object>) parts.get(0).get("render"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every known render type gets its required lists and safe defaults.
+    *******************************************************************************/
+   @Test
+   @SuppressWarnings("unchecked")
+   void everyRenderTypeIsNormalized()
+   {
+      Map<String, Object> records = renderOf("{\"type\":\"records\",\"tableName\":\"t\",\"columns\":[{\"name\":\"a\"},\"junk\",{\"label\":\"no name\"}],\"rows\":[{\"id\":1},7],\"totalCount\":\"3\",\"truncated\":\"yes\"}");
+      assertThat(records).containsEntry("tableLabel", "t").containsEntry("totalCount", 3L).containsEntry("truncated", false);
+      assertThat((List<Map<String, Object>>) records.get("columns")).containsExactly(Map.of("name", "a", "label", "a"));
+      assertThat((List<Map<String, Object>>) records.get("rows")).hasSize(1).first().satisfies(row -> assertThat(row.get("values")).isEqualTo(Map.of()));
+
+      Map<String, Object> record = renderOf("{\"type\":\"record\",\"tableName\":\"t\",\"fields\":[{\"name\":\"n\",\"value\":5},{\"value\":\"orphan\"}]}");
+      assertThat((List<Map<String, Object>>) record.get("fields")).containsExactly(Map.of("name", "n", "label", "n", "value", "5"));
+
+      Map<String, Object> count = renderOf("{\"type\":\"count\",\"tableName\":\"t\",\"count\":\"many\"}");
+      assertThat(count).containsEntry("tableLabel", "t").containsEntry("count", null);
+
+      Map<String, Object> widget = renderOf("{\"type\":\"widget\",\"widgetName\":\"w\",\"data\":\"bad\"}");
+      assertThat(widget).containsEntry("label", "w").containsEntry("data", Map.of());
+
+      Map<String, Object> history = renderOf("{\"type\":\"history\",\"entries\":[{\"message\":\"x\",\"details\":\"bad\"},3]}");
+      assertThat((List<Map<String, Object>>) history.get("entries")).containsExactly(Map.of("message", "x", "details", List.of()));
+
+      Map<String, Object> pending = renderOf("{\"type\":\"pending_action\",\"actionId\":\"12\"}");
+      assertThat(pending).containsEntry("actionId", 12L).containsEntry("summary", "").containsEntry("changes", List.of());
+
+      assertThat(renderOf("{\"type\":\"bulk_load_dry_run\",\"okCount\":1}")).containsEntry("errors", List.of());
+      assertThat(renderOf("{\"type\":\"records\"}")).containsEntry("rows", List.of()).containsEntry("columns", List.of()).containsEntry("tableLabel", "");
+   }
+
+
+
+   /*******************************************************************************
+    ** Unknown render and part types pass through untouched.
+    *******************************************************************************/
+   @Test
+   void unknownTypesPassThrough()
+   {
+      assertThat(renderOf("{\"type\":\"sparkline\",\"points\":\"whatever\"}")).isEqualTo(Map.of("type", "sparkline", "points", "whatever"));
+      List<Map<String, Object>> parts = MessageParts.fromJson("[{\"type\":\"future_part\",\"x\":1}]").list();
+      assertThat(parts).containsExactly(Map.of("type", "future_part", "x", 1));
+   }
+
+
+
+   /*******************************************************************************
+    ** What cannot be repaired becomes the error notice; nothing throws.
+    *******************************************************************************/
+   @Test
+   void unrepairableRendersBecomeNotices()
+   {
+      List<Map<String, Object>> parts = MessageParts.fromJson("[{\"type\":\"tool\",\"name\":\"t\",\"outcome\":\"OK\",\"render\":{\"type\":\"pending_action\"}},"
+         + "{\"type\":\"tool\",\"name\":\"u\",\"render\":\"not an object\"},"
+         + "42, {\"no\":\"type\"}, {\"type\":\"presentation\"},"
+         + "{\"type\":\"navigate\",\"href\":\"https://evil.example\"}, {\"type\":\"notice\",\"level\":\"panic\"}]").list();
+      Map<String, Object> notice = RenderShapes.undisplayableNotice();
+      assertThat(parts).containsExactly(
+         Map.of("type", "tool", "name", "t", "outcome", "OK", "summary", ""), notice,
+         Map.of("type", "tool", "name", "u", "outcome", "ERROR", "summary", ""), notice,
+         notice, notice, notice,
+         Map.of("type", "notice", "level", "info", "text", ""));
+      assertThat(notice).containsEntry("level", "error").containsEntry("text", RenderShapes.UNDISPLAYABLE_TEXT);
+      assertThat(MessageParts.fromJson("{\"not\":\"a list\"}").list()).isEmpty();
+      assertThat(RenderShapes.parts(null)).isEmpty();
+      assertThat(RenderShapes.render(null)).isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** Presentations: blocks keep their lists; a presentation with nothing to
+    ** show is replaced (in parts and in the presentation event).
+    *******************************************************************************/
+   @Test
+   @SuppressWarnings("unchecked")
+   void presentationsAreNormalized()
+   {
+      Map<String, Object> presentation = RenderShapes.presentation(Map.of("summary", "Sales", "blocks", List.of(
+         Map.of("kind", "table", "columns", List.of(Map.of("key", "c0"), "junk"), "rows", List.of(Map.of("cells", "bad"))),
+         Map.of("kind", "chart"), Map.of("kind", "metrics"), Map.of("no", "kind"), Map.of("kind", "future"))));
+      assertThat(presentation).containsEntry("type", "presentation").containsEntry("markdown", "Sales");
+      List<Map<String, Object>> blocks = (List<Map<String, Object>>) presentation.get("blocks");
+      assertThat(blocks).hasSize(4);
+      assertThat(blocks.get(0)).containsEntry("title", "").containsEntry("columns", List.of(Map.of("key", "c0", "label", "c0")))
+         .containsEntry("rows", List.of(Map.of("cells", Map.of(), "display", Map.of())));
+      assertThat(blocks.get(1)).containsEntry("chartType", "bar").containsEntry("points", List.of());
+      assertThat(blocks.get(2)).containsEntry("items", List.of());
+      assertThat(blocks.get(3)).isEqualTo(Map.of("kind", "future"));
+
+      assertThat(RenderShapes.presentation(Map.of("blocks", List.of()))).isNull();
+      TurnEvent event = new TurnEvent(4, TurnEvent.PRESENTATION, Map.of("part", Map.of("type", "presentation")));
+      assertThat(event.getData().get("part")).isEqualTo(RenderShapes.undisplayableNotice());
+      TurnEvent toolResult = TurnEvent.fromJson(5, TurnEvent.TOOL_RESULT, "{\"name\":\"t\",\"render\":{\"type\":\"pending_action\"}}");
+      assertThat(toolResult.getData()).doesNotContainKey("render").containsEntry("name", "t");
+      assertThat(TurnEvent.fromJson(6, TurnEvent.TOOL_RESULT, "not json").getData()).containsOnlyKeys("seq");
+   }
+}

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/TurnEngineTest.java

@@ -141,6 +141,27 @@ class TurnEngineTest extends BaseAssistantTest
 
 
 
+   /*******************************************************************************
+    ** The panel reloads the conversation as soon as message.end arrives, so the
+    ** question and the answer must already be stored then, not after it.
+    *******************************************************************************/
+   @Test
+   void testMessagesAreStoredBeforeMessageEnd() throws Exception
+   {
+      provider.thenText("Four posts are published.");
+
+      TurnRunner.Outcome outcome = runner.run(EDITOR, "how many published posts?");
+      assertThat(outcome.end()).containsEntry("status", "COMPLETE");
+
+      List<AssistantMessage> stored = runner.messagesAtEnd();
+      assertThat(stored).hasSize(2);
+      assertThat(stored).extracting(AssistantMessage::getPartsJson).allSatisfy(parts -> assertThat(parts).isNotEqualTo("[]"));
+      assertThat(stored.get(0).getPartsJson()).contains("how many published posts?");
+      assertThat(stored.get(1).getPartsJson()).contains("Four posts are published.");
+   }
+
+
+
    /*******************************************************************************
     ** Parallel tool calls: all results in ONE user message, in order.
     *******************************************************************************/

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java

@@ -31,8 +31,10 @@ import com.kingsrook.qbits.assistant.AssistantQBitConfig;
 import com.kingsrook.qbits.assistant.engine.ProviderHealth;
 import com.kingsrook.qbits.assistant.engine.ProviderService;
 import com.kingsrook.qbits.assistant.engine.UsageRecorder;
+import com.kingsrook.qbits.assistant.model.AssistantBudget;
 import com.kingsrook.qbits.assistant.model.AssistantModel;
 import com.kingsrook.qbits.assistant.model.AssistantPolicy;
+import com.kingsrook.qbits.assistant.model.AssistantProfile;
 import com.kingsrook.qbits.assistant.model.AssistantProvider;
 import com.kingsrook.qbits.assistant.model.AssistantUsage;
 import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider;
@@ -303,7 +305,7 @@ class AssistantHealthChecksTest extends BaseAssistantTest
       var registry = HealthChecksMetaData.ofOrWithNew(qInstance);
       AssistantHealthChecks.register(qInstance);
       assertThat(registry.getSuppliers()).hasSize(2).allSatisfy(supplier -> assertThat(supplier.getComponent()).isEqualTo("assistant"));
-      assertThat(registry.resolveAll(qInstance)).hasSize(4).allSatisfy(check ->
+      assertThat(registry.resolveAll(qInstance)).hasSize(5).allSatisfy(check ->
       {
          assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
          assertThat(check.getCategory()).isEqualTo(HealthCategory.AI);
@@ -317,7 +319,7 @@ class AssistantHealthChecksTest extends BaseAssistantTest
       assertThat(modelCheck.getTimeoutMs()).isEqualTo(30000);
       assertThat(check("assistant.provider." + local.getId()).getTimeoutMs()).isEqualTo(10000);
       config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", model.getId()).withValue("enabled", false));
-      assertThat(registry.resolveAll(qInstance)).hasSize(3);
+      assertThat(registry.resolveAll(qInstance)).hasSize(4);
       assertThat(run(modelCheck).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
    }
 
@@ -606,12 +608,211 @@ class AssistantHealthChecksTest extends BaseAssistantTest
       assertThat(qInstance.getWidget(AssistantHealthWidget.NAME).getType()).isEqualTo(WidgetType.STATISTICS.getType());
       var renderer = new AssistantHealthWidget();
       StatisticsData data = (StatisticsData) renderer.render(new RenderWidgetInput()).getWidgetData();
-      assertThat(data.getCount()).isEqualTo("UNKNOWN");
+      assertThat(data.getCount()).isEqualTo("Not checked yet");
+      assertThat(data.getCountContext()).contains("Provider checks run every 1 to 5 minutes");
       assertThat(data.getCountURL()).startsWith("/app/").contains("health_check", "assistant");
       try(var central = cached("assistant.provider." + row.getId(), HealthCheckOutcome.up("Reachable"), Instant.now()))
       {
+         var cache = HealthRunner.class.getDeclaredMethod("cacheOutcome", String.class, HealthCheckOutcome.class);
+         cache.setAccessible(true);
+         cache.invoke(central, AssistantHealthChecks.CLIENT_ERRORS, HealthCheckOutcome.up("0 client errors reported in the last hour"));
          data = (StatisticsData) renderer.render(new RenderWidgetInput()).getWidgetData();
-         assertThat(data.getCount()).isEqualTo("UP");
+         assertThat(data.getCount()).isEqualTo("Healthy");
+
+         //////////////////////////////////////////////////////////////////
+         // many client errors in the last hour show on the card as well //
+         //////////////////////////////////////////////////////////////////
+         cache.invoke(central, AssistantHealthChecks.CLIENT_ERRORS, HealthCheckOutcome.degraded("25 client errors reported in the last hour"));
+         data = (StatisticsData) renderer.render(new RenderWidgetInput()).getWidgetData();
+         assertThat(data.getCount()).isEqualTo("Degraded");
       }
    }
+
+
+   /*******************************************************************************
+    ** A second enabled provider, not the default.
+    *******************************************************************************/
+   private AssistantProvider optionalRow(AssistantProvider defaultRow) throws Exception
+   {
+      config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("isDefault", true).withValue("name", "Claude"));
+      AssistantProvider optional = config.getRuntime().getStore().insert(new AssistantProvider().withName("lore (local GPU)")
+         .withProviderType(type.getType()).withEnabled(true).withIsDefault(false));
+      config.getRuntime().getProviderService().invalidate();
+      return (optional);
+   }
+
+
+
+   /*******************************************************************************
+    ** An unreachable optional provider is DEGRADED and names the default; the
+    ** default stays DOWN, and the breaker still refuses the optional provider.
+    *******************************************************************************/
+   @Test
+   void optionalProviderIsDegradedDefaultIsDown() throws Exception
+   {
+      AssistantProvider defaultRow = row();
+      AssistantProvider lore       = optionalRow(defaultRow);
+      healthy = false;
+
+      HealthCheckOutcome outcome = run(check("assistant.provider." + lore.getId()));
+      assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED);
+      assertThat(outcome.getMessage()).isEqualTo("Provider 'lore (local GPU)' is unreachable; it is optional (not the default and not the only route), so requests use 'Claude'");
+      assertThat(outcome.getDetails().getValues()).containsEntry("reachable", false).containsEntry("optional", true).containsEntry("providerId", lore.getId());
+      assertThat(ProviderHealth.isUnreachable(outcome)).isTrue();
+
+      HealthCheckOutcome required = run(check("assistant.provider." + defaultRow.getId()));
+      assertThat(required.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertThat(required.getDetails().getValues()).containsEntry("required", "default provider").containsEntry("reachable", false);
+
+      healthy = true;
+      assertThat(run(check("assistant.provider." + lore.getId())).getStatus()).isEqualTo(HealthStatus.UP);
+      assertThat(ProviderHealth.isUnreachable(HealthCheckOutcome.degraded("Slow"))).isFalse();
+      assertThat(ProviderHealth.isUnreachable(null)).isFalse();
+
+      ////////////////////////////////////////////////////////////////////
+      // a fresh DEGRADED-but-unreachable central result opens the breaker //
+      ////////////////////////////////////////////////////////////////////
+      var active = new ProviderService.Active(lore, type.getType(), "model", provider);
+      try(var central = cached("assistant.provider." + lore.getId(), outcome, Instant.now().minusSeconds(1)))
+      {
+         assertThatThrownBy(() -> new ProviderHealth(config.getRuntime()).beforeCall(active)).isInstanceOf(ModelProviderException.class);
+         assertThat(probes).isEqualTo(3);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Model checks follow their provider: DEGRADED when optional.
+    *******************************************************************************/
+   @Test
+   void optionalProviderModelIsDegraded() throws Exception
+   {
+      AssistantProvider defaultRow = row();
+      AssistantProvider lore       = optionalRow(defaultRow);
+      AssistantModel    model      = model(lore);
+      provider.thenError(ModelProviderException.Kind.NETWORK, SECRET);
+      HealthCheckOutcome outcome = run(check("assistant.model." + model.getId()));
+      assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED);
+      assertThat(outcome.getMessage()).startsWith("Model connection failed; provider 'lore (local GPU)' is optional").endsWith("so requests use 'Claude'");
+      assertThat(outcome.getDetails().getValues()).doesNotContainKey("reachable").containsEntry("modelId", model.getId());
+
+      AssistantModel defaultModel = model(defaultRow);
+      provider.thenError(ModelProviderException.Kind.NETWORK, SECRET);
+      assertThat(run(check("assistant.model." + defaultModel.getId())).getStatus()).isEqualTo(HealthStatus.DOWN);
+   }
+
+
+
+   /*******************************************************************************
+    ** A provider that is the only route for routing or a budget downgrade stays
+    ** DOWN; an explicit fallback on another provider makes it optional again.
+    *******************************************************************************/
+   @Test
+   void soleRouteStaysDown() throws Exception
+   {
+      AssistantProvider defaultRow = row();
+      config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("location", "CLOUD"));
+      AssistantProvider lore         = optionalRow(defaultRow);
+      AssistantModel    loreModel    = model(lore);
+      AssistantModel    defaultModel = model(defaultRow);
+      healthy = false;
+      var check = check("assistant.provider." + lore.getId());
+      assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DEGRADED);
+
+      AssistantProfile profile = config.getRuntime().getStore().insert(new AssistantProfile().withName("routing").withDisplayName("Assistant").withEnabled(true)
+         .withRoutingMode("HEURISTIC").withSimpleModelId(loreModel.getId()).withComplexModelId(defaultModel.getId()));
+      HealthCheckOutcome outcome = run(check);
+      assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertThat(outcome.getDetails().getValues()).containsEntry("required", "only route for a configured model");
+
+      config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", loreModel.getId()).withValue("fallbackModelId", defaultModel.getId()));
+      assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DEGRADED);
+
+      config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("setupState", AssistantProvider.SETUP_STATE_SETTING_UP));
+      config.getRuntime().getProviderService().invalidate();
+      assertThat(run(check).getStatus()).as("lore is the only usable provider left, hence the default").isEqualTo(HealthStatus.DOWN);
+      config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("setupState", AssistantProvider.SETUP_STATE_ACTIVE));
+      config.getRuntime().getProviderService().invalidate();
+
+      config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", loreModel.getId()).withValue("fallbackModelId", null));
+      config.getRuntime().getStore().updateFields(AssistantProfile.TABLE_NAME, new QRecord().withValue("id", profile.getId()).withValue("routingMode", "OFF"));
+      assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DEGRADED);
+
+      config.getRuntime().getStore().insert(new AssistantBudget().withName("Downgrade").withScopeType("INSTANCE").withPeriod("MONTH").withMetric("COST_USD")
+         .withLimitValue(BigDecimal.ONE).withEnabled(true).withDowngradeModelId(loreModel.getId()));
+      assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN);
+   }
+
+
+
+   /*******************************************************************************
+    ** Another local provider of the same type is the automatic fallback.
+    *******************************************************************************/
+   @Test
+   void localPoolIsAFallback() throws Exception
+   {
+      AssistantProvider defaultRow = row();
+      AssistantProvider lore       = optionalRow(defaultRow);
+      AssistantModel    loreModel  = model(lore);
+      model(defaultRow);
+      config.getRuntime().getStore().insert(new AssistantProfile().withName("routing").withDisplayName("Assistant").withEnabled(true)
+         .withRoutingMode("HEURISTIC").withSimpleModelId(loreModel.getId()).withComplexModelId(loreModel.getId()));
+      healthy = false;
+      assertThat(run(check("assistant.provider." + lore.getId())).getStatus()).isEqualTo(HealthStatus.DEGRADED);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setting up: no checks registered, no routing, and an already resolved check
+    ** reports UP without probing.
+    *******************************************************************************/
+   @Test
+   void settingUpSkipsChecksAndRouting() throws Exception
+   {
+      AssistantProvider defaultRow = row();
+      AssistantProvider lore       = optionalRow(defaultRow);
+      AssistantModel    model      = model(lore);
+      assertThat(lore.getSetupState()).isEqualTo(AssistantProvider.SETUP_STATE_ACTIVE);
+      var providerCheck = check("assistant.provider." + lore.getId());
+      var modelCheck    = check("assistant.model." + model.getId());
+
+      config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", lore.getId()).withValue("setupState", AssistantProvider.SETUP_STATE_SETTING_UP));
+      config.getRuntime().getProviderService().invalidate();
+      assertThat(HealthChecksMetaData.ofOrWithNew(qInstance).resolveAll(qInstance)).extracting(HealthCheckMetaData::getName)
+         .doesNotContain("assistant.provider." + lore.getId(), "assistant.model." + model.getId())
+         .contains("assistant.provider." + defaultRow.getId());
+      Integer before = probes;
+      assertThat(run(providerCheck).getMessage()).isEqualTo(AssistantHealthChecks.NOT_SET_UP);
+      assertThat(run(modelCheck).getStatus()).isEqualTo(HealthStatus.UP);
+      assertThat(probes).isEqualTo(before);
+      assertThat(provider.getRequests()).isEmpty();
+
+      assertThat(config.getRuntime().getProviderService().resolveModel(model.getId(), false)).isNull();
+      assertThatThrownBy(() -> new ProviderHealth(config.getRuntime()).beforeCall(new ProviderService.Active(lore, type.getType(), "model", provider)))
+         .hasMessageContaining("not set up yet");
+
+      config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", lore.getId()).withValue("isDefault", true));
+      config.getRuntime().getProviderService().invalidate();
+      assertThat(config.getRuntime().getProviderService().activeRow().getId()).as("a default that is setting up is skipped").isEqualTo(defaultRow.getId());
+
+      assertThatThrownBy(() -> config.getRuntime().getStore().insert(new AssistantProvider().withName("Bad").withProviderType(type.getType()).withEnabled(true).withSetupState("BOGUS")))
+         .hasMessageContaining("Setup state must be Active or Setting up.");
+   }
+
+
+
+   /*******************************************************************************
+    ** If routes cannot be read the failure stays DOWN; other statuses pass through.
+    *******************************************************************************/
+   @Test
+   void optionalPassThrough() throws Exception
+   {
+      assertThat(AssistantHealthChecks.optional(config.getRuntime(), null, null, false)).isNull();
+      HealthCheckOutcome up = HealthCheckOutcome.up("fine");
+      assertThat(AssistantHealthChecks.optional(config.getRuntime(), 1, up, false)).isSameAs(up);
+      assertThat(AssistantHealthChecks.optional(config.getRuntime(), null, HealthCheckOutcome.down("x"), true).getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertThat(AssistantHealthChecks.optional(null, 1, HealthCheckOutcome.down("x"), false).getStatus()).isEqualTo(HealthStatus.DOWN);
+   }
 }

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/persistence/SchemaForeignKeyPostgresTest.java

@@ -94,6 +94,9 @@ class SchemaForeignKeyPostgresTest extends BaseAssistantTest
       rule("assistant_tool_call", "pending_action_id", "assistant_pending_action", "n");
       rule("assistant_feedback", "conversation_id", "assistant_conversation", "c");
       rule("assistant_feedback", "message_id", "assistant_message", "c");
+      rule("assistant_client_error", "conversation_id", "assistant_conversation", "n");
+      rule("assistant_client_error", "message_id", "assistant_message", "n");
+      rule("assistant_client_error", "turn_id", "assistant_turn", "n");
       rule("assistant_pending_action", "conversation_id", "assistant_conversation", "c");
       rule("assistant_pending_action", "turn_id", "assistant_turn", "n");
       rule("assistant_data_audit", "conversation_id", "assistant_conversation", "n");

qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/testing/TurnRunner.java

@@ -27,6 +27,7 @@ import com.kingsrook.qbits.assistant.AssistantRuntime;
 import com.kingsrook.qbits.assistant.api.AssistantApi;
 import com.kingsrook.qbits.assistant.engine.RunningTurn;
 import com.kingsrook.qbits.assistant.engine.TurnEvent;
+import com.kingsrook.qbits.assistant.model.AssistantMessage;
 import com.kingsrook.qbits.assistant.model.AssistantToolCall;
 import com.kingsrook.qbits.assistant.persistence.AssistantStore;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -47,6 +48,8 @@ public class TurnRunner
    private final QInstance           qInstance;
    private final AssistantQBitConfig config;
 
+   private List<AssistantMessage> messagesAtEnd = List.of();
+
 
 
    /*******************************************************************************
@@ -169,6 +172,10 @@ public class TurnRunner
       {
          List<TurnEvent> batch = running.getLog().after(last, 1000);
          events.addAll(batch);
+         if(batch.stream().anyMatch(e -> TurnEvent.MESSAGE_END.equals(e.getType())))
+         {
+            messagesAtEnd = messagesOf(userId, convId);
+         }
          if(!batch.isEmpty())
          {
             last = batch.get(batch.size() - 1).getSeq();
@@ -194,6 +201,36 @@ public class TurnRunner
 
 
 
+   /*******************************************************************************
+    ** The conversation's stored messages, read the moment the run's message.end
+    ** was seen: what the panel finds when it reloads the conversation at the end
+    ** of the stream.
+    *******************************************************************************/
+   public List<AssistantMessage> messagesAtEnd()
+   {
+      return (messagesAtEnd);
+   }
+
+
+
+   /*******************************************************************************
+    ** The stored messages of a conversation (read as the user).
+    *******************************************************************************/
+   private List<AssistantMessage> messagesOf(String userId, Integer conversationId) throws Exception
+   {
+      QContext.init(qInstance, AssistantTestFixture.ownerSession(userId, config));
+      try
+      {
+         return (new AssistantStore().listMessages(userId, conversationId, null, 50));
+      }
+      finally
+      {
+         QContext.clear();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** The audited tool calls of a turn (read as the user).
     *******************************************************************************/