Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#92) from deps/align-gitea into develop

pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.15 -> 4.1.0-RC.26
pom.xml: removed Central snapshot repository
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.circleci/config.yml

@@ -1,60 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.8
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: '0.8'
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.gitea/workflows/build.yaml

@@ -0,0 +1,7 @@
+name: build
+on:
+  push:
+jobs:
+  maven:
+    uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
+    secrets: inherit

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: '0.8'
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

CHANGELOG.md

@@ -2,14 +2,26 @@
 
 All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
 
+## Release note correction (1.1.0-RC.11)
+
+1.1.0-RC.11 (Maven Central, 2026-10-15 13:05Z) was built from release/1.1 `2e2501d` and contains #61 only. The `v1.1.0-RC.11` tag mistakenly points at `6028a66`; use `2e2501d` as RC.11's source. Cluster caches (#62) first ship in 1.1.0-RC.12.
+
 ## [Unreleased]
 
-Requires QQQ 4.1.0-RC.6 or later (the spec-managed UI API extension contract).
+Requires QQQ 4.1.0-RC.15 or later (central health checks, help bundles, the spec-managed UI API extension contract, and `ClusterCaches` for multi-node deployments).
 
 ### Fixed
+- Single-node clusters: the Quick Search index no longer asks for a replica it cannot place, so a one-node OpenSearch stays green instead of yellow and `quick_search.opensearch` is no longer DEGRADED. New `indexReplicas` (default null: automatic, `min(maxAutoReplicas, data nodes - 1)`) and `maxAutoReplicas` (default 1) config; zero config gives 0 replicas on one node and 1 on larger clusters. The target is set on every new physical index and reconciled on existing ones at startup with `PUT /<index>/_settings` when it differs (idempotent, logged; a failure is logged and reported, never blocks startup; indexes with `auto_expand_replicas` and OpenSearch Serverless are left alone). The status snapshot and the `quick_search.opensearch` health details gain `unassignedReplicaShards`, `targetReplicas` and `replicaSettingsFailed`; yellow on a single node with every primary active (only replicas unassigned) is reported UP with a note. Yellow with an unassigned primary, or on a multi-node cluster, stays DEGRADED; red stays DOWN.
+- Priming config records at startup, and recording diagnostic events, build their system session with the instance, so it has its all-access keys. Startup logged "QInstance was not supplied when creating a QSystemUserSession" once per node.
+- Multi-node deployments: a Quick Search config change (an index row, its fields or the settings row, from the admin UI or a config process) now clears the cached config on every node, not only the node that made the change. The runtime's config cache (the config store snapshot and settings, and the per-table enabled flags) is registered with QQQ `ClusterCaches` as `qbit-quick-search.config`, and `QuickSearchRuntime.invalidateConfig()` publishes a cluster invalidation. The 30 s and 60 s TTLs remain as the safety net. `QuickSearchFieldRules`' warn-once table set is registered as node-local. `QuickSearchClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation.
+- Indexing bookkeeping is no longer record-audited: the `quick_search_index` runtime fields written by basepull, full reindex and real-time indexing (status, last basepull time, document count, last run, real-time error count) and the settings row's priming outcome (`lastPrimedAt`, summary) are written with `omitDmlAudit`. On a production host these wrote about 470 "Record was Edited during process: Update search index" audit rows an hour. Admin edits to an index, its fields or the settings are still audited.
 - Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write.
 
 ### Added
+- UI API extension quick-search 1.1.0 (requires QQQ 4.1.0-RC.15, UI API 1.6.0): operationIds, standard error responses with BasicErrorResponseV1, extension tags, and retained deprecated QuickSearchErrorV1 schema. Existing operationIds and success payloads are preserved.
+- Central health checks `quick_search.opensearch`, `quick_search.backlog` and the dynamic `quick_search.index.<indexName>` supplier, all SEARCH/DEPENDENCY checks every 60 seconds. Reuses cached diagnostics with deadline-bounded waits and safe count/flag details. Includes registration, state, deadline and credential-sentinel tests.
+- Host-neutral `help/qbit-quick-search.yaml` bundle, registered through QQQ help bundles, with coverage and no-help-table tests. Replaces the temporary inline help loader; hosts can override entries with a HOST bundle.
+- Every `quick_search_*` table and the Quick Search Admin app carry help that says what the table is for and how its rows get created (`help/quick-search.yaml`, QQQ help-bundle format, applied by `QuickSearchHelp` as a stand-in for core help bundles; host help is never overridden). `QuickSearchTablesGuardrailTest` fails when a table is unused by runtime code, neither primed nor listed as starting empty, or has no such help; `QuickSearchSeededDefaultsTest` proves priming populates the config tables on a fresh backend, is idempotent and keeps admin edits.
 - Session-only typeahead: `GET /qqq/v1/qbit/quick-search/suggest` (`getQuickSearchSuggestions`) and `QuickSearchSuggestService.suggest(...)`. Reuses the permission-aware search action with one bounded candidate batch per table; groups expose only record IDs, filtered labels/subtitles, `hasMore` and `truncated`. Defaults to 5 items across 8 tables (maximum 10/20); short terms and unavailable search return empty groups. The existing UI extension and generated contract now carry both status and suggestions.
 - Supported host status API (`api.QuickSearchStatusService.get()`), immutable status values, and `QuickSearchApi` home navigation constants (issue #50/F10). Permission-checked, bounded diagnostics refreshes share a short per-runtime cache and reuse saved validation counts. New spec-managed `GET /qqq/v1/qbit/quick-search/status` (`getQuickSearchStatus`) with committed OpenAPI fragment and strict HTTP/contract tests. RC.6 requires the default `/qbit/quick-search` prefix; `/quick-search` is not permitted by core. Existing routes and wire shapes are unchanged.
 - Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`.
@@ -20,6 +32,9 @@ Requires QQQ 4.1.0-RC.6 or later (the spec-managed UI API extension contract).
 - `QuickSearchResult.subtitle` and `displayValues` (from `SUBTITLE` and `DISPLAY` fields); document objects `fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`, added to existing indexes' mapping at startup.
 - Possible-value sources `quick_search_eligible_table`, `quick_search_indexable_field`, `quick_search_timestamp_field`, `quick_search_field_role`, `quick_search_field_analyzer`, `quick_search_documents_choice`, `quick_search_config_source`; the join `quick_search_index_join_quick_search_index_field` and the Searchable Fields widget on the index record. Config tables are audited at field level.
 
+### Removed
+- None for central health integration; the status API, diagnostics widget and developer panel are retained.
+
 ### Changed
 - Distinguish protected fields from fields that simply cannot be searched. Boolean, blob and hidden fields no longer trigger result rebuilding or erase record labels. Labels are suppressed only when their source fields are protected; unprotected hidden primary keys and timestamps remain usable.
 - Exclude PII/PHI-classified fields (including table defaults) and field-security-locked fields everywhere Quick Search indexes or displays values. Configuration rejects protected fields; code defaults drop them with one warning per table. Search and diagnostics omit protected values from older documents. Rebuild existing indexes after upgrading or changing classification metadata. Uses QQQ 4.1.0-RC.6; no dependency changes.
@@ -34,13 +49,14 @@ First release candidate of 1.1, published to Maven Central from `release/1.1`. I
 
 ## [1.1.0] - Unreleased
 
-Requires QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI; built against 4.1.0-RC.2. See `docs/MIGRATION-1.1.md`.
+Requires QQQ 4.1.0-RC.4 or later (the record-search provider SPI arrived in 4.1.0-RC.2; its scores, highlights, totals and paging in 4.1.0-RC.4); built against 4.1.0-RC.4. See `docs/MIGRATION-1.1.md`.
 
 ### Added
 - `QuickSearchRecordSearchProvider`: serves QQQ core record search (`RecordSearchAction`, `POST /qqq/v1/search`, the Next UI global search) from OpenSearch for the tables Quick Search indexes. One `_msearch` request with a per-table top-N search (same query, AND semantics and field weights as `QuickSearchAction`); returns record ids only, which core re-reads through `QueryAction` (permissions, record locks, labels). Audit finding F-11, decision D1.
 - `serveCoreRecordSearch` (default `true`): registers the provider and gives indexed tables core `searchFields` (visible string and integer searchable fields) when they have none. Host `searchFields` and a host-registered provider are never replaced.
 - `recordSearchTimeoutMillis` (default 2500): wall-clock limit on the OpenSearch request; on timeout or any failure core falls back to its own search, and the provider claims no tables for 30 seconds.
 - `QuickSearchOpenSearchClient.searchRecordIdsPerTable`.
+- Record search details for QQQ 4.1.0-RC.4's record-search contract: the provider passes each hit's relevance `score`, a plain-text highlight (match ranges, from OpenSearch highlighting of `fieldValues.*` with private-use markers) and each table's total hits (`totalCountByTable`, counted to 10,000), and honors the request `offset` (`from`). `sort: RECENT` keeps relevance order (the index has no record modify dates). New `QuickSearchOpenSearchClient.searchRecordsPerTable`; `searchRecordIdsPerTable` now delegates to it. This requires QQQ 4.1.0-RC.4 or later.
 - Diagnostic tools (#35), on by default (`enableDiagnosticTools`): Search Playground, Validate Search Index (schedulable with `validationCronExpression`; reports kept in the new `quickSearchValidationRun` table), Test Search Analyzer, and on every indexed table and on `quickSearchFailedEvent` rows the record actions Inspect Search Document (database vs index, and why a record does or does not match a term) and Reindex in Search. All are table or record actions, gated by the new `debugPermissionRules` (`quickSearchDebug.hasAccess`; Reindex in Search uses `adminPermissionRules`), rate-limited per user (`diagnosticRunsPerMinute`), and recorded in the new `quickSearchDiagnosticEvent` table. Settings: `enableRecordDiagnosticActions`, `validationCronExpression`, `validationCronTimeZoneId`, `validationSampleSize`.
 - Search failure diagnostics: a failed or timed-out OpenSearch request behind core record search or `QuickSearchAction` is recorded as a `SEARCH_FAILED` diagnostic event (tables, duration, error, server; never the term); the runtime counts failures and backoff fallbacks. The Search Diagnostics widget (`quickSearchDiagnostics`, on the admin app home) shows the provider's state, recent failures and the latest validation.
 - `QuickSearchDiagnosticsOperations`: read-only probe search (named clauses, field highlights, request and response JSON), explain, document reads, `_analyze`, document sampling, and raw cluster, mapping and settings reads.

README.md

@@ -8,7 +8,7 @@
 
 Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
 
-Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the 1.1 line builds against 4.1.0-RC.4 from the next release candidate on.
+Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the current build requires QQQ 4.1.0-RC.7 for help bundles.
 
 ## What it does
 
@@ -130,11 +130,11 @@ Search runs as the current `QContext` session: tables the session cannot read ar
 QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v1/search` and used by the Next UI's global search box): a case-insensitive contains-query over each table's `searchFields`. With `serveCoreRecordSearch` (default `true`) the producer registers `QuickSearchRecordSearchProvider` on the instance, and core asks it to search every table Quick Search indexes. The API and the UI do not change.
 
 - **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quick_search_index`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
-- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights); each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order. Core re-reads them by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, and records deleted since they were indexed drop out.
+- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights), skipping the request's `offset`; each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order, each with its score and a highlight of the matching `fieldValues.*` fields (plain text plus match ranges), and the table's total hits (counted to 10,000). Core re-reads the ids by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, records deleted since they were indexed drop out, and only highlights of fields the user can see (where core's own highlighting found no match) reach the response, with `tables[].totalCount` and `hasMore`. The index holds no record modify dates, so `sort: RECENT` gets relevance order for indexed tables.
 - **Failure.** The OpenSearch request is cancelled after `recordSearchTimeoutMillis` (default 2500). Any failure (timeout, cluster down, index missing) makes core search those tables with its own contains-query instead, so search degrades but never fails. After a failure, Quick Search claims no tables for 30 seconds, so a down cluster does not add a timeout to every search.
 - **Opting out.** `withServeCoreRecordSearch(false)` leaves core search and `searchFields` untouched. If the host already registered another record-search provider, the QBit keeps it and logs a warning (an instance has one provider).
 
-`QuickSearchAction` remains the richer API (scores, highlights, pagination, totals).
+`QuickSearchAction` remains the direct API (all hits in one ranked list, `searchableText` highlights, pagination across tables).
 
 ### Processes
 
@@ -227,6 +227,8 @@ The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned
 | `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
 | `fullReindexStaleMinutes` | 30 | A running full reindex that shows no sign of life for this long is treated as stopped: it no longer blocks a new one, and basepull recovers its tables. Keep it well above the time one page of `sourceBatchSize` records takes; null means the default |
 | `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
+| `indexReplicas` | null (automatic) | Index `number_of_replicas`; null means `min(maxAutoReplicas, data nodes - 1)`, so a single node gets 0 and stays green. Set on new indexes, reconciled on existing ones at startup |
+| `maxAutoReplicas` | 1 | Most replicas automatic sizing asks for |
 | `maxFieldLength` | 10000 | Truncate each indexed value |
 | `maxSearchLimit` | 100 | Largest page size |
 | `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
@@ -257,6 +259,10 @@ The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned
 - `quick_search_validation_run` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
 - `quick_search_failed_event`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
 
+### Starter defaults and empty-state help
+
+This qbit ships no DDL and no seed changelog: startup self-priming (`QuickSearchConfigPrimer`) is its seed mechanism. On a fresh database it writes a `quick_search_index` row and `quick_search_index_field` rows for every table the code declares searchable, and the single `quick_search_settings` row (id 1). `quick_search_config_tombstone` and the five log and history tables start empty by design (rows appear when an admin removes a table or field, or when the qbit runs).
+
 Real-time error counts on Quick Search Index show failures since the last successful full reindex or basepull. Each failed event increases the count; deferred changes while indexing is paused do not. Counter updates are best effort and serialized across listeners in one JVM; concurrent application nodes can still race because QQQ has no arithmetic update API.
 
 ## Extending
@@ -271,7 +277,7 @@ CI=true ./mvnw verify                       # plus Testcontainers integration te
 ./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
 ```
 
-Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.4 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
+Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports QQQ 4.1.0-RC.8 through the default-active `qqq-snapshot` profile ahead of the parent BOM (override only with a compatible released version using `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
 
 ## Upgrading from 1.0
 
@@ -354,7 +360,7 @@ String homeHref = status.homeHref();
 ```
 
 UI clients use **`GET /qqq/v1/qbit/quick-search/status`**, operationId
-`getQuickSearchStatus`, extension `quick-search` version `1.0.0`. The endpoint returns
+`getQuickSearchStatus`, extension `quick-search` version `1.1.0` (requires QQQ `4.1.0-RC.15`, UI API `1.6.0`). The endpoint returns
 that same immutable value as JSON; it requires the Quick Search app's configured
 `adminPermissionRules` (default `quick_search_admin.hasAccess`). Permission is checked
 before every read, including cache hits. No additional source-table permissions are
@@ -395,6 +401,31 @@ configured metadata prefix. `QuickSearchApi.APP_NAME` (`quickSearchAdmin`) and
 navigation. Keep detailed runs, failed events, validation and operator tools in the
 qbit's own home. [kof22 migration mapping](docs/STATUS-API-HOST-MIGRATION.md).
 
+## Health checks
+
+The QBit producer registers these checks with QQQ's central health registry. All use
+component `quick_search`, category `SEARCH`, a 60-second interval and a 5-second timeout.
+
+| Name | Kind | Interval | Reports |
+| --- | --- | --- | --- |
+| `quick_search.opensearch` | DEPENDENCY | 60s | Cluster reachability and colour: red is DOWN, yellow is DEGRADED, disabled is UNKNOWN. |
+| `quick_search.backlog` | DEPENDENCY | 60s | DEGRADED for any exhausted events, more than 100 pending events, or paused indexing. |
+| `quick_search.index.<indexName>` | DEPENDENCY | 60s | One check per persisted index row; non-active status, count mismatch, last error or real-time errors are DEGRADED. Disabled indexes are UNKNOWN. |
+
+Index names come from the row's table name, normalized to lowercase health identifiers.
+The supplier refreshes the set through core's normal discovery cycle. Checks reuse the
+five-second status cache and saved validation counts, with waits bounded by the runner's
+deadline. Incomplete measurements never become zero; indexes absent from the bounded
+100-row status snapshot report UNKNOWN. Backlog truncation reports DEGRADED.
+Details contain counts, timestamps and flags, never stored error text or credentials.
+
+The status endpoint, diagnostics widget and record developer panel remain available.
+This integration requires QQQ 4.1.0-RC.8 or later, which includes the central health SPI. The POM pins that release. For local offline verification:
+
+```powershell
+./mvnw.cmd -o verify
+```
+
 ## Classified data
 
 Quick Search excludes fields classified as **PII** or **PHI**, including fields
@@ -424,6 +455,26 @@ Diagnostic views also omit protected values and the raw response for these table
 This does not purge old OpenSearch backups or retained physical indexes; apply your
 host's retention policy to those copies.
 
-This release uses the APIs in QQQ **4.1.0-RC.6**. Support for the host's
+This release builds against QQQ **4.1.0-RC.7**. Support for the host's
 `AuditValueProtector` classification (`ClassifiedValues`, qqq#1182) and
 `FieldAccessHelper.getLockTier` (qqq#1172) is deferred until the qbit moves to RC.7.
+
+## In-app help
+
+The producer registers [help/qbit-quick-search.yaml](src/main/resources/help/qbit-quick-search.yaml)
+with QQQ's help registry (requires QQQ 4.1.0-RC.7 or later). The bundle covers
+Quick Search's own app, tables, actions, widgets and possible-value sources.
+Registration is metadata-only and works without a `helpContent` table. Hosts
+with help enabled load help content and call `HelpBundles.syncOnStartup(qInstance)`
+after producing all metadata; QQQ guards sync when the help table is absent.
+
+To customize wording or document deployment schedules, register your own
+`layer: HOST` bundle with the same entry keys and roles using
+`HelpBundles.register(qInstance, "help/my-host.yaml")`. HOST text takes precedence
+over QBIT text; edits made by users are preserved by sync. Host-specific record
+actions (Inspect Search Document and Reindex in Search on host tables) belong in
+the host bundle. Keep the bundle's literal version aligned with the POM revision.
+
+`QuickSearchTablesGuardrailTest.helpBundleIsComplete` uses `HelpBundleAssertions`
+to check owned objects, non-trivial fields, summary length and live references.
+The same test class checks registration without the help subsystem.

docs/MIGRATION-1.1.md

@@ -228,6 +228,10 @@ MySQL: use `INT AUTO_INCREMENT` for the ids, `DATETIME` for the timestamps and `
 - `quick_search_searchable_table` is now a custom possible-value source over the effective tables (it was an enum of the code-declared ones).
 - Tables added at runtime get core `searchFields` (for the global search) when they have none, but the per-table Inspect Search Document and Reindex in Search record actions appear only after the next restart.
 
+### Table help (no action needed)
+
+The qbit now sets bundle help (`help/quick-search.yaml`) on the Quick Search Admin app and each `quick_search_*` table, so an empty table says what it is for and how its rows get created. It fills only empty `help` slots; help a host already set on these tables is kept. There is no DDL change and no host include.
+
 ### Checklist for data-driven configuration
 
 1. Add the DDL above (it is additive).

docs/SESSION-STATE.md

@@ -1,122 +0,0 @@
-# Session State
-
-Updated: 2026-10-14
-
-## Real-time error counter fix (2026-10-14)
-
-- Failed-event recording now increments the table's `realTimeErrorCount` after commit, using QQQ reads and system writes. Null counts start at zero; update failures are logged and cannot fail the source write or prevent replay.
-- QQQ has no arithmetic update API. The read-modify-write fallback is serialized across listeners in this JVM; separate application nodes can still race.
-- Successful basepull and full-reindex status updates reset the count, including the single-table reindex path. Investigation found the previous zero assignments only initialized new rows. Field help, README and Unreleased changelog now describe the reset window.
-- Regression coverage includes event counts, concurrent listeners, counter-update failure, after-commit timing, basepull reset and both full-reindex reset paths.
-- Final gate: `./mvnw.cmd -o -B verify` passed against released QQQ 4.1.0-RC.6: 476 unit tests, zero failures/errors; 41 integration tests skipped because Docker is unavailable; architecture guardrails and JaCoCo coverage gates passed. No dependency changes or offline artifact gaps. Checkstyle, PMD and SpotBugs are not configured in this Maven build.
-- No commit, push or branch creation. Second-brain save is blocked by the vault being outside writable roots; this section retains the durable handoff for later vault synchronization.
-
-## Status API worktree (2026-10-13, issue #50 / F10)
-
-- Implemented `api.QuickSearchStatusService.get()`, immutable status values, default app/home constants and a spec-managed status endpoint. Admin permission is checked on every call; saved validation counts avoid source scans. Five-second per-runtime cache, two-second wait, one in-flight worker, existing diagnostics rate limiter, bounded table/backlog reads.
-- Endpoint uses core's permitted `/qqq/v1/qbit/quick-search/status` namespace (`getQuickSearchStatus`). Requested `/quick-search` is rejected by RC.6's established-prefix allowlist. Core and host clones are read-only; no edits there. No previous Quick Search HTTP operations or Next UI callers found.
-- POM targets released-coordinate `4.1.0-RC.6`; local verification uses only `-Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT`.
-- Host file/line replacement mapping: `docs/STATUS-API-HOST-MIGRATION.md`. Contract: `src/main/resources/openapi/qqq-v1/quick-search.yaml`.
-- Verification: 468 unit tests (including strict HTTP, contract, bounds, cache isolation, permissions and timeout recovery); JaCoCo passes. All 41 Docker integration tests skip because Docker is unavailable. Final command: `./mvnw.cmd -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT`.
-- Standalone repository Checkstyle is not green: HEAD has 471 main-source violations; this tree has 438, with zero newly introduced violations. New production files and both new test files pass their style checks. Existing header/import/style cleanup is outside the status API change.
-- No commit, push, merge or deployment authorized. Second-brain vault is outside writable roots; durable notes are retained here for later vault synchronization.
-
-## Context
-
-`1.0.0-RC.2` is released (2026-10-06). It fixes the five pre-GA findings from the PR #10 review, each on its own branch and PR into `develop`:
-
-| Issue | PR | Merge commit | Fix |
-|---|---|---|---|
-| #15 | #18 | `2398e62` | Transport: warn on `hostnameVerification=false`, blank `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected |
-| #17 | #21 | `e9ec54a` | `safeCount` logs at warn and keeps the previous `documentCount` |
-| #16 | #19 | `9941a70` | Search `offset` counts readable results under record locks; bounded scan; 10,000 window cap |
-| #14 | #20 | `dce0f26` | Deletes during a full reindex captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap |
-| #13 | #22 | `c5c41d4` | Listener re-read failure and missing AWS SDK (`LinkageError`) recorded as failed events |
-
-`develop` (`c5c41d4`) was merged into `release/1.0` as `b7d366c` (only difference from develop: `<revision>1.0.0-RC.1</revision>`). The orb's `publish_release_candidate` workflow succeeded on `b7d366c` (CircleCI build 82) and pushed `c11b2c2` ("Bump version to 1.0.0-RC.2 [skip ci]"). Maven Central serves the jar, pom, sources, javadoc and signatures for `com.kingsrook.qbits:qbit-quick-search:1.0.0-RC.2`.
-
-## Release mechanics (qqq-orb 0.6.8, confirmed in its source)
-
-- RCs publish artifacts only: no git tag and no GitHub release. The GitHub release comes when the release branch merges to `main` and a `vX.Y.Z` tag is pushed.
-- Never push a `vX.Y.Z-RC.n` tag: it matches the GA job's `v*` filter, maps back to `release/X.Y.Z`, and would publish a stray `RC.n+1`.
-- The next push to `release/1.0` publishes `1.0.0-RC.3`. Keep the release branch's `<revision>` on merge.
-- `develop` requires one approving review and the PR author cannot approve their own PR; James merges with `gh pr merge <n> --squash --admin`. The agent cannot run `gh pr merge`, and the auto-mode classifier also refused a background loop that polled PR merge state.
-- GitHub SSH times out from this machine; push over HTTPS with `git -c credential.helper='!gh auth git-credential' push https://github.com/QRun-IO/qbit-quick-search.git <ref>`.
-
-## Next
-
-1. qqq-all and Voyage pin `1.0.0-RC.2` and report back; their feedback decides RC.3 or GA.
-2. Remaining "Before 1.0.0 GA" items in `docs/TODO.md`: stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review), and the `qbit-build-parent` 2.1.0 re-pin.
-3. GA: merge `release/1.0` to `main` and tag `v1.0.0`.
-
-## 2026-10-14 — secured-data section 15
-
-- Implemented a shared RC.6 classification/security policy in `QuickSearchFieldRules`.
-- Applied it to config discovery, config writes/choices, document construction,
-  safe query fields, result presentation and diagnostic reads; protected primary
-  keys cannot be used as index IDs.
-- Added live-listener/basepull/stale-result PHI/PII canaries and config/rule tests.
-- Final gate: `mvnw.cmd -o -B verify` **BUILD SUCCESS**. 479 unit tests passed
-  (7 new tests); 41 Docker integration tests skipped because Docker is unavailable.
-  Architecture guardrails passed. JaCoCo: 73.84% instruction / 91.61% class coverage,
-  above the 70% / 90% gates. Checkstyle, PMD and SpotBugs are not configured in this
-  build. No missing offline artifacts. `git diff --check` passed.
-- Reviewed the diff; changes remain uncommitted. Gate log:
-  `target/secured-data-verify.log`.
-- RC.7 host audit-protector classification and lock-tier APIs remain deferred.
-
-### Changed files
-
-- Documentation: `README.md`, `CHANGELOG.md`, `docs/SESSION-STATE.md`, `docs/TODO.md`.
-- Configuration: `QuickSearchQBitProducer.java`, `config/QuickSearchFieldRules.java`,
-  `config/QuickSearchConfigPrimer.java`, `config/QuickSearchConfigStore.java`.
-- Indexing: `listeners/QuickSearchRecordChangeListener.java`,
-  `processes/IndexingUtils.java`, `processes/AbstractIndexingStep.java`,
-  `processes/BasepullIndexStep.java`.
-- Search and diagnostics: `actions/QuickSearchAction.java`,
-  `actions/QuickSearchRecordSearchProvider.java`,
-  `opensearch/QuickSearchOpenSearchClient.java`,
-  `opensearch/QuickSearchDiagnosticsOperations.java`.
-- Tests: `config/QuickSearchConfigCustomizersTest.java`,
-  `listeners/QuickSearchRecordChangeListenerTest.java`,
-  `processes/IndexingUtilsTest.java`, `publisher/SynchronousIndexEventPublisherTest.java`,
-  new `opensearch/ClassifiedDataCanaryTest.java`.
-
-Production and test paths above are relative to
-`src/{main,test}/java/com/kingsrook/qbits/quicksearch/` respectively.
-
-### Continuity
-
-Second-brain project/daily writes are unavailable in this session because the
-vault lies outside the writable roots. This local checkpoint holds the durable
-outcome for the reviewer. No commit, push, branch creation, dependency version
-change, or production filesystem IO was introduced.
-
-
-## 2026-10-14 - protection review follow-up
-
-- Added isProtected for PII/PHI (including table defaults), field locks, sensitive
-  values and masked types. Unsupported search types and hidden fields alone are
-  not protected. Searchable/snippet/display field lists still use isIndexable.
-- Labels check their configured/table source fields; unrelated protected fields
-  preserve safe labels. A copied record prevents unsafe preformatted labels from
-  returning through QQQ fallback formatting without changing the input record.
-- Primary-key guards and timestamp versions now use protection rather than
-  search eligibility. Added four regression tests and extended rule/canary tests.
-- Follow-up files: config/QuickSearchFieldRules.java, processes/IndexingUtils.java,
-  listeners/QuickSearchRecordChangeListener.java,
-  opensearch/QuickSearchOpenSearchClient.java,
-  opensearch/QuickSearchDiagnosticsOperations.java; test files
-  config/QuickSearchConfigCustomizersTest.java, processes/IndexingUtilsTest.java,
-  opensearch/ClassifiedDataCanaryTest.java; README.md, CHANGELOG.md,
-  docs/TODO.md and this file. Java paths are relative to the existing main/test
-  quicksearch packages. All earlier uncommitted work is preserved.
-- Second-brain project/daily synchronization remains unavailable because the vault
-  is outside writable roots; this checkpoint is the local handoff. Nothing committed
-  or pushed; no branches or dependency changes.
-- Final gate: `mvnw.cmd -o -B verify` BUILD SUCCESS; 483 unit tests passed
-  (four additional tests); all three PHI/PII canaries passed. All 41 Docker
-  integration tests skipped because Docker is unavailable. Architecture guards
-  and JaCoCo passed (73.88% instruction / 91.61% class coverage).
-  Checkstyle, PMD and SpotBugs are not configured in this build. No offline
-  artifacts missing. Gate log: `target/protection-review-verify.log`.

docs/STATUS-API-HOST-MIGRATION.md

@@ -34,7 +34,7 @@ group has `tableName`, `tableLabel`, `hasMore` and `items` of `recordId`, `label
 and nullable `subtitle`. See the [API details](../README.md#typeahead-suggestions)
 for normalization, bounded-batch semantics and empty responses during outages.
 
-Extension: `quick-search`, version `1.0.0`; core dependency: `4.1.0-RC.6`.
+Extension: `quick-search`, version `1.1.0`; core dependency: `4.1.0-RC.15` (UI API `1.6.0`).
 The requested `/quick-search` prefix cannot validate on the supplied RC.6 baseline:
 `MiddlewareSpecExtensionsMetaData.ESTABLISHED_PREFIXES` only admits esb, alerts,
 assistant, and sql-workbench. We use the default namespace without modifying core.

docs/TODO.md

@@ -1,5 +1,19 @@
 # TODO
 
+## UI API 1.6 migration (2026-10-15)
+
+- [x] Bump `quick-search` extension to 1.1.0, retain deprecated `QuickSearchErrorV1`, and use core `BasicErrorResponseV1` for errors; preserve operation IDs and success payloads.
+- [x] Regenerate the two-operation contract with `-Dquicksearch.openapi.update=true`, then pass the contract test without regeneration; no dangling references or separate baseline were found.
+- [x] Set the release BOM default to QQQ `4.1.0-RC.15`; validate offline against the locally installed `4.1.99-SNAPSHOT` using `./mvnw.cmd -o test -Dqqq.snapshot.version=4.1.99-SNAPSHOT -Denforcer.skipRules=requireReleaseDeps` (540 tests, no failures/errors/skips). The command-line exception is needed only because the requested local core is a SNAPSHOT; other guardrails remain enabled.
+- [x] Verify strengthened HTTP error-body assertions: 8 status/HTTP tests passed after the full suite.
+- [ ] Run Docker integration tests through `verify` in CI; this migration ran the requested `test` lifecycle.
+
+
+## In-app help bundle (2026-10-14)
+
+- [x] Pin QQQ RC.7, register the host-neutral QBIT bundle, replace the temporary loader, document HOST overrides, and add coverage/no-help-table checks.
+- [ ] Host adoption: enable QQQ help startup sync and keep deployment schedules and host-table record actions in HOST bundles.
+
 ## Real-time error counter
 
 - [x] Count failed real-time events after commit, serialize concurrent listeners within a JVM, tolerate counter-write failures, and reset after successful basepull/full reindex; add regression tests and field/documentation help.

pom.xml

@@ -18,6 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
+      <qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
       <qbit.guardrails.enforce>true</qbit.guardrails.enforce>
       <qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
       <revision>1.1.0-SNAPSHOT</revision>
@@ -32,7 +33,7 @@
    <parent>
       <groupId>com.kingsrook</groupId>
       <artifactId>qbit-build-parent</artifactId>
-      <version>2.1.0</version>
+      <version>2.2.1</version>
    </parent>
 
    <groupId>com.kingsrook.qbits</groupId>
@@ -170,6 +171,7 @@
             <directory>src/main/resources</directory>
             <filtering>true</filtering>
             <includes>
+               <include>help/*.yaml</include>
                <include>quick-search.properties</include>
             </includes>
          </resource>
@@ -177,6 +179,7 @@
             <directory>src/main/resources</directory>
             <filtering>false</filtering>
             <excludes>
+               <exclude>help/*.yaml</exclude>
                <exclude>quick-search.properties</exclude>
             </excludes>
          </resource>
@@ -281,7 +284,7 @@
             <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
-            <qqq.snapshot.version>4.1.0-RC.6</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>
@@ -294,18 +297,6 @@
                </dependency>
             </dependencies>
          </dependencyManagement>
-         <repositories>
-            <repository>
-               <id>sonatype-snapshots</id>
-               <url>https://central.sonatype.com/repository/maven-snapshots/</url>
-               <releases>
-                  <enabled>false</enabled>
-               </releases>
-               <snapshots>
-                  <enabled>true</enabled>
-               </snapshots>
-            </repository>
-         </repositories>
       </profile>
    </profiles>
 </project>

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfig.java

@@ -81,6 +81,7 @@ public class QuickSearchQBitConfig implements QBitConfig
    public static final int    DEFAULT_VALIDATION_SAMPLE_SIZE       = 50;
    public static final int    MAX_VALIDATION_SAMPLE_SIZE           = 1000;
    public static final int    DEFAULT_DIAGNOSTIC_RUNS_PER_MINUTE   = 30;
+   public static final int    DEFAULT_MAX_AUTO_REPLICAS            = 1;
 
    ///////////////////////////////////////////////////////////////////////////////
    // a live full reindex refreshes its run record after every page; a page is  //
@@ -113,6 +114,8 @@ public class QuickSearchQBitConfig implements QBitConfig
    private Integer responseTimeoutMillis = 60000;
    private Integer maxConnections = 30;
    private Integer maxBulkRequestBytes = 5 * 1024 * 1024;
+   private Integer indexReplicas;
+   private Integer maxAutoReplicas = DEFAULT_MAX_AUTO_REPLICAS;
    private QCodeReference transportCustomizer;
    private QuickSearchStartupMode startupMode = QuickSearchStartupMode.FAIL_FAST;
    private Boolean enableRealTimeIndexing = true;
@@ -443,6 +446,16 @@ public class QuickSearchQBitConfig implements QBitConfig
       {
          errors.add("basepullOverlapSeconds must not be negative for QuickSearchQBit");
       }
+
+      if(indexReplicas != null && indexReplicas < 0)
+      {
+         errors.add("indexReplicas must not be negative for QuickSearchQBit");
+      }
+
+      if(maxAutoReplicas != null && maxAutoReplicas < 0)
+      {
+         errors.add("maxAutoReplicas must not be negative for QuickSearchQBit");
+      }
    }
 
 
@@ -1495,6 +1508,73 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Getter for indexReplicas: number_of_replicas for the Quick Search index.
+    ** Null (the default) means automatic: min(maxAutoReplicas, data nodes - 1),
+    ** so a single-node cluster gets 0 replicas and stays green. Applied when a
+    ** physical index is created and reconciled on existing indexes at startup.
+    ** Ignored for OpenSearch Serverless (awsServiceName aoss).
+    *******************************************************************************/
+   public Integer getIndexReplicas()
+   {
+      return (this.indexReplicas);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for indexReplicas
+    *******************************************************************************/
+   public void setIndexReplicas(Integer indexReplicas)
+   {
+      this.indexReplicas = indexReplicas;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for indexReplicas
+    *******************************************************************************/
+   public QuickSearchQBitConfig withIndexReplicas(Integer indexReplicas)
+   {
+      this.indexReplicas = indexReplicas;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for maxAutoReplicas: the most replicas automatic mode (indexReplicas
+    ** null) asks for; it never asks for more than data nodes - 1. Default 1.
+    *******************************************************************************/
+   public Integer getMaxAutoReplicas()
+   {
+      return (this.maxAutoReplicas);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxAutoReplicas
+    *******************************************************************************/
+   public void setMaxAutoReplicas(Integer maxAutoReplicas)
+   {
+      this.maxAutoReplicas = maxAutoReplicas;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxAutoReplicas
+    *******************************************************************************/
+   public QuickSearchQBitConfig withMaxAutoReplicas(Integer maxAutoReplicas)
+   {
+      this.maxAutoReplicas = maxAutoReplicas;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for transportCustomizer: QCodeReference to an OpenSearchTransportCustomizer
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java

@@ -30,6 +30,7 @@ import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
 import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
 import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
 import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
+import com.kingsrook.qbits.quicksearch.health.QuickSearchHealthChecks;
 import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
 import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
 import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
@@ -41,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
@@ -218,6 +220,11 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
       MetaDataProducerMultiOutput output = QBitMetaDataProducer.super.produce(qInstance);
       output.addSelfToInstance(qInstance);
 
+      //////////////////////////////////////////////////////////////////////////
+      // Metadata only; core guards startup sync when helpContent is absent. //
+      //////////////////////////////////////////////////////////////////////////
+      HelpBundles.register(qInstance, "help/qbit-quick-search.yaml");
+
       return (new MetaDataProducerMultiOutput());
    }
 
@@ -257,6 +264,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
       QuickSearchRuntime runtime = new QuickSearchRuntime(qBitConfig);
       runtime.setDiscoveredTables(discoveredTables);
       qBitConfig.setRuntime(runtime);
+      QuickSearchHealthChecks.register(qInstance);
 
       if(Boolean.TRUE.equals(qBitConfig.getEnableRealTimeIndexing()))
       {

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntime.java

@@ -23,6 +23,8 @@ import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -34,6 +36,7 @@ import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
 import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -70,6 +73,17 @@ public class QuickSearchRuntime
 
    private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
 
+   /*******************************************************************************
+    ** Cluster cache name for every runtime's config cache (the config store's
+    ** snapshot and settings, and the enabled flags). A config row edited on
+    ** one node clears it on every node; the TTLs remain the safety net.
+    *******************************************************************************/
+   public static final String CONFIG_CACHE_NAME = "qbit-quick-search.config";
+
+   private static final Set<QuickSearchRuntime> RUNTIMES = ClusterCaches.shared(CONFIG_CACHE_NAME,
+      Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())),
+      QuickSearchRuntime::invalidateAllLocal, null);
+
    public static final long RECORD_SEARCH_BACKOFF_MILLIS = 30_000L;
 
    private final StatusCache statusCache = new StatusCache();
@@ -128,6 +142,7 @@ public class QuickSearchRuntime
    {
       this.config      = config;
       this.configStore = new QuickSearchConfigStore(this);
+      RUNTIMES.add(this);
    }
 
 
@@ -261,9 +276,21 @@ public class QuickSearchRuntime
 
 
    /*******************************************************************************
-    ** Drop every cached config (a config row changed).
+    ** Drop every cached config (a config row changed): on this node now, and
+    ** on the other nodes through the cluster bus.
     *******************************************************************************/
    public void invalidateConfig()
+   {
+      invalidateConfigLocal();
+      ClusterCaches.invalidate(CONFIG_CACHE_NAME);
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop this runtime's cached config on this node only.
+    *******************************************************************************/
+   void invalidateConfigLocal()
    {
       configStore.invalidate();
       enabledCache.clear();
@@ -271,6 +298,25 @@ public class QuickSearchRuntime
 
 
 
+   /*******************************************************************************
+    ** Drop the cached config of every live runtime on this node (a cluster
+    ** invalidation arrived).
+    *******************************************************************************/
+   private static void invalidateAllLocal()
+   {
+      List<QuickSearchRuntime> runtimes;
+      synchronized(RUNTIMES)
+      {
+         runtimes = new ArrayList<>(RUNTIMES);
+      }
+      for(QuickSearchRuntime runtime : runtimes)
+      {
+         runtime.invalidateConfigLocal();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Return the OpenSearch client, building it on first use. Building the
     ** client does not contact the cluster; {@link #ensureIndexReady()} does.
@@ -422,6 +468,16 @@ public class QuickSearchRuntime
 
 
 
+   /*******************************************************************************
+    ** How many enabled flags are cached (for tests).
+    *******************************************************************************/
+   int getCachedEnabledFlagCount()
+   {
+      return (enabledCache.size());
+   }
+
+
+
    /*******************************************************************************
     ** Forget cached enabled flags (after an operator toggles a table).
     *******************************************************************************/
@@ -702,7 +758,7 @@ public class QuickSearchRuntime
          {
             return;
          }
-         QContext.init(qInstance, new QSystemUserSession());
+         QContext.init(qInstance, new QSystemUserSession(qInstance));
          try
          {
             configStore.tryPrime();
@@ -741,6 +797,7 @@ public class QuickSearchRuntime
          return;
       }
       closed = true;
+      RUNTIMES.remove(this);
       statusCache.close();
 
       if(publisher != null)

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchRecordSearchProvider.java

@@ -18,6 +18,7 @@ package com.kingsrook.qbits.quicksearch.actions;
 
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -25,6 +26,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
 import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlight;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlightMatch;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
@@ -32,6 +35,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchTableHits;
 import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
 import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
@@ -51,10 +57,14 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  **
  ** A table is claimed when it is indexed by this QBit and enabled, and core
  ** record search is not backing off after a recent failure. search() sends
- ** one _msearch request with a top-N search per table and returns record ids
- ** only, best first; core re-reads them by primary key through QueryAction,
- ** so table permissions, record security locks and record labels are core's
- ** job, and ids of deleted records drop out. Each table gets
+ ** one _msearch request with a top-N search per table (after skipping the
+ ** input's offset) and returns record ids, best first, each with its score
+ ** and highlights (plain text plus match ranges, from the indexed field
+ ** values), and each table's total hits; core re-reads the ids by primary key
+ ** through QueryAction, so table permissions, record security locks and
+ ** record labels are core's job, and ids of deleted records drop out (core
+ ** also keeps only highlights of fields the session can see). The index has
+ ** no record modify dates, so a RECENT sort gets relevance order. Each table gets
  ** IDS_PER_LIMIT_MULTIPLIER x limitPerTable ids (at most
  ** RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE) so dropouts do not leave
  ** the table short.
@@ -70,6 +80,8 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
 
    public static final int IDS_PER_LIMIT_MULTIPLIER = 4;
 
+   private static final String FIELD_VALUES_PREFIX = "fieldValues.";
+
 
 
    /*******************************************************************************
@@ -136,11 +148,13 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
       int idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * IDS_PER_LIMIT_MULTIPLIER);
       int timeoutMillis = config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis();
 
-      Map<String, List<String>> idsByTable;
-      long                      start = System.currentTimeMillis();
+      int offset = input.getOffset() == null ? 0 : Math.max(0, input.getOffset());
+
+      Map<String, RecordSearchTableHits> hitsByTable;
+      long                               start = System.currentTimeMillis();
       try
       {
-         idsByTable = runtime.getClient().searchRecordIdsPerTable(term, tableNames, idsPerTable, runtime.getDiscoveredTables(), timeoutMillis);
+         hitsByTable = runtime.getClient().searchRecordsPerTable(term, tableNames, idsPerTable, offset, true, runtime.getDiscoveredTables(), timeoutMillis);
       }
       catch(QException | RuntimeException e)
       {
@@ -157,15 +171,115 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
          throw (e instanceof QException qe ? qe : new QException("Quick Search record search failed: " + e.getMessage(), e));
       }
 
+      Map<String, Integer> totalCountByTable = new HashMap<>();
       for(String tableName : tableNames)
       {
-         for(String recordId : CollectionUtils.nonNullList(idsByTable.get(tableName)))
+         RecordSearchTableHits tableHits = hitsByTable.get(tableName);
+         if(tableHits == null)
          {
-            results.add(new RecordSearchResult().withTableName(tableName).withRecordId(recordId));
+            continue;
+         }
+
+         if(tableHits.totalHits() != null)
+         {
+            totalCountByTable.put(tableName, (int) Math.min(Integer.MAX_VALUE, tableHits.totalHits()));
+         }
+
+         for(RecordSearchHit hit : CollectionUtils.nonNullList(tableHits.hits()))
+         {
+            List<RecordSearchHighlight> highlights = highlights(hit.highlights());
+            results.add(new RecordSearchResult()
+               .withTableName(tableName)
+               .withRecordId(hit.recordId())
+               .withScore(hit.score())
+               .withHighlights(highlights.isEmpty() ? null : highlights));
          }
       }
 
-      return (new RecordSearchOutput().withResults(results));
+      return (new RecordSearchOutput().withResults(results).withTotalCountByTable(totalCountByTable));
+   }
+
+
+
+   /*******************************************************************************
+    ** A hit's highlight fragments (by index field) as core record search
+    ** highlights: one per fieldValues.{fieldName} field, as plain text with
+    ** match ranges.  Other index fields, and fragments without a match, are
+    ** skipped.
+    *******************************************************************************/
+   public static List<RecordSearchHighlight> highlights(Map<String, List<String>> fragmentsByIndexField)
+   {
+      List<RecordSearchHighlight> highlights = new ArrayList<>();
+      for(Map.Entry<String, List<String>> entry : CollectionUtils.nonNullMap(fragmentsByIndexField).entrySet())
+      {
+         String indexField = entry.getKey();
+         if(indexField == null || !indexField.startsWith(FIELD_VALUES_PREFIX) || indexField.length() == FIELD_VALUES_PREFIX.length() || CollectionUtils.nullSafeIsEmpty(entry.getValue()))
+         {
+            continue;
+         }
+
+         RecordSearchHighlight highlight = highlight(indexField.substring(FIELD_VALUES_PREFIX.length()), entry.getValue().get(0));
+         if(highlight != null)
+         {
+            highlights.add(highlight);
+         }
+      }
+      return (highlights);
+   }
+
+
+
+   /*******************************************************************************
+    ** A highlight from a fragment whose matches are wrapped in the client's
+    ** pre/post tags: the fragment's text without the tags, and the ranges they
+    ** wrapped (adjacent ranges merged).  Null if the fragment is blank or has
+    ** no match.
+    *******************************************************************************/
+   public static RecordSearchHighlight highlight(String fieldName, String fragment)
+   {
+      if(fragment == null)
+      {
+         return (null);
+      }
+
+      StringBuilder                    text    = new StringBuilder();
+      List<RecordSearchHighlightMatch> matches = new ArrayList<>();
+      int                              open    = -1;
+      for(int i = 0; i < fragment.length(); i++)
+      {
+         char c = fragment.charAt(i);
+         if(c == QuickSearchOpenSearchClient.HIGHLIGHT_PRE_TAG.charAt(0))
+         {
+            open = text.length();
+         }
+         else if(c == QuickSearchOpenSearchClient.HIGHLIGHT_POST_TAG.charAt(0))
+         {
+            if(open >= 0 && text.length() > open)
+            {
+               RecordSearchHighlightMatch last = matches.isEmpty() ? null : matches.get(matches.size() - 1);
+               if(last != null && last.getEnd() == open)
+               {
+                  last.setEnd(text.length());
+               }
+               else
+               {
+                  matches.add(new RecordSearchHighlightMatch().withStart(open).withEnd(text.length()));
+               }
+            }
+            open = -1;
+         }
+         else
+         {
+            text.append(c);
+         }
+      }
+
+      if(matches.isEmpty() || text.toString().isBlank())
+      {
+         return (null);
+      }
+
+      return (new RecordSearchHighlight().withFieldName(fieldName).withText(text.toString()).withMatches(matches));
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchApi.java

@@ -48,7 +48,7 @@ public final class QuickSearchApi
     *******************************************************************************/
    public static MiddlewareSpecExtension extension()
    {
-      return (new MiddlewareSpecExtension().withName("quick-search").withSemanticVersion("1.0.0")
+      return (new MiddlewareSpecExtension().withName("quick-search").withSemanticVersion("1.1.0")
          .withDescription("Quick Search administration and suggestions").withSpec(QuickSearchStatusSpec.class).withSpec(QuickSearchSuggestSpec.class));
    }
 }

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatus.java

@@ -30,11 +30,41 @@ import com.kingsrook.qqq.openapi.model.Schema;
 /*******************************************************************************
  ** Immutable administrative snapshot. Null measurements are unknown, never zero.
  ** Counts come from persisted diagnostics; countsCheckedAt identifies their age.
+ **
+ ** unassignedReplicaShards is the unassigned shard count when every primary is
+ ** active (so all unassigned shards are replicas), else null; targetReplicas and
+ ** replicaSettingsFailed report the index replica sizing (null when unknown).
  *******************************************************************************/
 public record QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
+                                Integer unassignedReplicaShards, Integer targetReplicas, Boolean replicaSettingsFailed,
                                 Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
                                 Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref) implements ToSchema
 {
+   /*******************************************************************************
+    ** The 1.1.0-RC.14 shape, without the replica measurements (all unknown).
+    *******************************************************************************/
+   public QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
+                            Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
+                            Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref)
+   {
+      this(state, clusterReachable, clusterHealth, nodes, activeShards, unassignedShards, null, null, null, indexingPaused, indexes, pending, exhausted,
+         backlogTruncated, tablesTruncated, lastSuccessfulSync, lastError, checkedAt, homeHref);
+   }
+
+
+
+   /*******************************************************************************
+    ** Yellow only because replicas cannot be placed on a single-node cluster:
+    ** one node, every primary active and every unassigned shard a replica.
+    ** Such a cluster cannot do better, so it is healthy rather than degraded.
+    *******************************************************************************/
+   public static boolean isSingleNodeReplicaYellow(String clusterHealth, Integer nodes, Integer unassignedReplicaShards)
+   {
+      return ("yellow".equals(clusterHealth) && Integer.valueOf(1).equals(nodes) && unassignedReplicaShards != null && unassignedReplicaShards > 0);
+   }
+
+
+
    /*******************************************************************************
     ** Published wire schema for the same immutable value used by Java hosts.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatusSpec.java

@@ -27,6 +27,7 @@ import java.util.TreeMap;
 import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
 import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
 import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
+import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
 import com.kingsrook.qqq.openapi.model.HttpMethod;
 import com.kingsrook.qqq.openapi.model.Method;
 import com.kingsrook.qqq.openapi.model.Parameter;
@@ -103,9 +104,9 @@ public class QuickSearchStatusSpec extends AbstractJsonEndpointSpec
    @Override
    public List<BasicResponse> defineAdditionalBasicResponses()
    {
-      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Diagnostic rate limit exceeded", "QuickSearchErrorV1"),
-         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", "QuickSearchErrorV1"),
-         new BasicResponse(HttpStatus.FORBIDDEN, "Quick Search admin permission required", "QuickSearchErrorV1")));
+      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Diagnostic rate limit exceeded", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.FORBIDDEN, "Quick Search admin permission required", BasicErrorResponseV1.class.getSimpleName())));
    }
 
 
@@ -125,6 +126,9 @@ public class QuickSearchStatusSpec extends AbstractJsonEndpointSpec
       {
          status.put(name, field(Type.INTEGER, "Cluster health counter; null when unavailable.", true));
       }
+      status.put("unassignedReplicaShards", field(Type.INTEGER, "Unassigned shards when every primary is active (all are replicas); null when unknown or a primary is not active.", true));
+      status.put("targetReplicas", field(Type.INTEGER, "number_of_replicas Quick Search applies to its index; null when unknown or not managed (Serverless).", true));
+      status.put("replicaSettingsFailed", field(Type.BOOLEAN, "Sizing or applying the index replicas failed at startup; null when not attempted.", true));
       status.put("indexingPaused", field(Type.BOOLEAN, "Indexing is paused; search can still serve.", true));
       status.put("indexes", new Schema().withType(Type.ARRAY).withItems(new Schema().withRef("#/components/schemas/QuickSearchIndexStatusV1"))
          .withDescription("At most 100 persisted index rows, ordered by id."));
@@ -147,7 +151,8 @@ public class QuickSearchStatusSpec extends AbstractJsonEndpointSpec
       row.put("lastError", field(Type.STRING, "Persisted indexing error; null when absent.", true));
       row.put("realTimeErrorCount", field(Type.INTEGER, "Persisted real-time indexing error count.", true));
       return (Map.of("QuickSearchStatusV1", object(status), "QuickSearchIndexStatusV1", object(row),
-         "QuickSearchErrorV1", object(Map.of("error", field(Type.STRING, "Framework error message.", false)))));
+         "QuickSearchErrorV1", object(Map.of("error", field(Type.STRING, "Framework error message.", false)))
+            .withDescription("Deprecated since 1.1.0: errors use BasicErrorResponseV1.")));
    }
 
 

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchSuggestSpec.java

@@ -27,6 +27,7 @@ import java.util.TreeMap;
 import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
 import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
 import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
+import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
 import com.kingsrook.qqq.openapi.model.HttpMethod;
 import com.kingsrook.qqq.openapi.model.In;
 import com.kingsrook.qqq.openapi.model.Method;
@@ -115,9 +116,9 @@ public class QuickSearchSuggestSpec extends AbstractJsonEndpointSpec
    @Override
    public List<BasicResponse> defineAdditionalBasicResponses()
    {
-      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Missing q, normalized term longer than 100 characters, or invalid limits", "QuickSearchErrorV1"),
-         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", "QuickSearchErrorV1"),
-         new BasicResponse(HttpStatus.FORBIDDEN, "Session permission denied", "QuickSearchErrorV1")));
+      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Missing q, normalized term longer than 100 characters, or invalid limits", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.FORBIDDEN, "Session permission denied", BasicErrorResponseV1.class.getSimpleName())));
    }
 
 

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java

@@ -412,8 +412,12 @@ public class QuickSearchConfigPrimer
       }
       else
       {
+         ///////////////////////////////////////////////////////////////////////
+         // Only the priming outcome (lastPrimedAt, summary) changes here, on  //
+         // every startup: bookkeeping, not a settings change, so not audited. //
+         ///////////////////////////////////////////////////////////////////////
          values.withValue("id", rows.get(0).getValue("id"));
-         updateAll(settingsTable, List.of(values));
+         updateAll(settingsTable, List.of(values), true);
       }
    }
 
@@ -477,6 +481,17 @@ public class QuickSearchConfigPrimer
     ** Update rows, failing on any record error.
     *******************************************************************************/
    public static void updateAll(String tableName, List<QRecord> records) throws QException
+   {
+      updateAll(tableName, records, false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update rows, failing on any record error; omitAudit skips the DML audit
+    ** for runtime bookkeeping writes.
+    *******************************************************************************/
+   public static void updateAll(String tableName, List<QRecord> records, boolean omitAudit) throws QException
    {
       if(records.isEmpty())
       {
@@ -485,6 +500,7 @@ public class QuickSearchConfigPrimer
       UpdateInput updateInput = new UpdateInput();
       updateInput.setTableName(tableName);
       updateInput.setRecords(records);
+      updateInput.setOmitDmlAudit(omitAudit);
       for(QRecord record : CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords()))
       {
          if(CollectionUtils.nullSafeHasContents(record.getErrors()))

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigStore.java

@@ -60,8 +60,9 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** before 1.1.0-RC.3.
  **
  ** Reads are cached for CACHE_TTL_MILLIS and dropped at once by the config
- ** tables' customizers when a row changes on this server; other servers pick
- ** the change up when their cache expires.
+ ** tables' customizers when a row changes, on every server: the runtime
+ ** publishes a cluster cache invalidation (QuickSearchRuntime.CONFIG_CACHE_NAME).
+ ** The TTL remains the safety net when the cluster bus is down.
  *******************************************************************************/
 public class QuickSearchConfigStore
 {

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchFieldRules.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.ai.AiDataClassifications;
 import com.kingsrook.qqq.backend.core.actions.ai.DataClassification;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
@@ -59,7 +60,8 @@ import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
 public final class QuickSearchFieldRules
 {
    private static final QLogger LOG = QLogger.getLogger(QuickSearchFieldRules.class);
-   private static final WeakHashMap<QTableMetaData, Boolean> WARNED_TABLES = new WeakHashMap<>();
+   private static final WeakHashMap<QTableMetaData, Boolean> WARNED_TABLES = ClusterCaches.nodeLocal(new WeakHashMap<>(),
+      "warn-once log de-duplication for this node's table metadata objects");
 
    public static final int MAX_DEFAULT_FIELDS = 20;
    public static final int TITLE_WEIGHT       = 3;

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticEventRecorder.java

@@ -124,7 +124,7 @@ public class DiagnosticEventRecorder implements AutoCloseable
     *******************************************************************************/
    private void insert(QInstance qInstance, String tableName, QuickSearchDiagnosticEvent event)
    {
-      QSession session = new QSystemUserSession();
+      QSession session = new QSystemUserSession(qInstance);
       try
       {
          QContext.init(qInstance, session);

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java

@@ -21,12 +21,15 @@
 package com.kingsrook.qbits.quicksearch.diagnostics;
 
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.ReentrantLock;
 import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
 import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
@@ -42,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
  *******************************************************************************/
 public class StatusCache implements AutoCloseable
 {
+   private final ReentrantLock lock = new ReentrantLock();
    private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable ->
    {
       Thread thread = new Thread(runnable, "quick-search-status");
@@ -58,7 +62,47 @@ public class StatusCache implements AutoCloseable
     ** Authorization belongs to the public service, before reaching this cache.
     ** Only refreshes consume the existing per-user diagnostic rate limit.
     *******************************************************************************/
-   public synchronized QuickSearchStatus get(QuickSearchRuntime runtime) throws QException
+   public QuickSearchStatus get(QuickSearchRuntime runtime) throws QException
+   {
+      return (get(runtime, Instant.now().plusSeconds(2)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bound both lock contention and the shared refresh wait by the caller's deadline.
+    *******************************************************************************/
+   public QuickSearchStatus get(QuickSearchRuntime runtime, Instant deadline) throws QException
+   {
+      try
+      {
+         long remaining = Duration.between(Instant.now(), deadline).toNanos();
+         if(remaining <= 0 || !lock.tryLock(remaining, TimeUnit.NANOSECONDS))
+         {
+            throw (new QException("Status deadline exceeded"));
+         }
+         try
+         {
+            return (read(runtime, deadline));
+         }
+         finally
+         {
+            lock.unlock();
+         }
+      }
+      catch(InterruptedException e)
+      {
+         Thread.currentThread().interrupt();
+         throw (new QException("Status refresh interrupted", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Read or refresh while holding the deadline-aware cache lock.
+    *******************************************************************************/
+   private QuickSearchStatus read(QuickSearchRuntime runtime, Instant deadline) throws QException
    {
       if(cached != null && System.nanoTime() < expiresAt)
       {
@@ -90,7 +134,8 @@ public class StatusCache implements AutoCloseable
       }
       try
       {
-         cached = refresh.get(2, TimeUnit.SECONDS);
+         long remaining = Math.max(0, Duration.between(Instant.now(), deadline).toNanos());
+         cached = refresh.get(Math.min(TimeUnit.SECONDS.toNanos(2), remaining), TimeUnit.NANOSECONDS);
          refresh = null;
       }
       catch(TimeoutException e)

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusDiagnostics.java

@@ -135,6 +135,9 @@ public final class StatusDiagnostics
       Integer nodes = null;
       Integer active = null;
       Integer unassigned = null;
+      Integer unassignedReplicas = null;
+      Integer targetReplicas = null;
+      Boolean replicaSettingsFailed = null;
       State state = degraded ? State.DEGRADED : State.OK;
       if(!truncated && !indexes.isEmpty() && indexes.stream().noneMatch(row -> Boolean.TRUE.equals(row.enabled())))
       {
@@ -152,7 +155,15 @@ public final class StatusDiagnostics
             nodes = number(body, "number_of_nodes");
             active = number(body, "active_shards");
             unassigned = number(body, "unassigned_shards");
-            if(IndexValidator.healthLevel(health) != DiagnosticsHtml.Level.OK || body.path("timed_out").asBoolean())
+            unassignedReplicas = unassignedReplicaShards(body, unassigned);
+            var replicaStatus = runtime.getClient().getReplicaStatus();
+            if(replicaStatus != null)
+            {
+               targetReplicas = replicaStatus.targetReplicas();
+               replicaSettingsFailed = replicaStatus.failed();
+            }
+            boolean singleNodeReplicas = QuickSearchStatus.isSingleNodeReplicaYellow(health, nodes, unassignedReplicas);
+            if((IndexValidator.healthLevel(health) != DiagnosticsHtml.Level.OK && !singleNodeReplicas) || body.path("timed_out").asBoolean())
             {
                state = "red".equals(health) ? State.DOWN : State.DEGRADED;
             }
@@ -167,7 +178,8 @@ public final class StatusDiagnostics
             lastError = "OpenSearch health is unavailable; see Quick Search diagnostics.";
          }
       }
-      return (new QuickSearchStatus(state, reachable, health, nodes, active, unassigned, paused, indexes, pending, exhausted, backlog.size() > MAX_BACKLOG,
+      return (new QuickSearchStatus(state, reachable, health, nodes, active, unassigned, unassignedReplicas, targetReplicas, replicaSettingsFailed,
+         paused, indexes, pending, exhausted, backlog.size() > MAX_BACKLOG,
          truncated, lastSync, lastError, Instant.now(), homeHref));
    }
 
@@ -193,6 +205,30 @@ public final class StatusDiagnostics
 
 
 
+   /*******************************************************************************
+    ** From index-level cluster health: the unassigned shards when nothing is
+    ** initializing and every index has all its primaries active (so every
+    ** unassigned shard is a replica); null otherwise or when unknown.
+    *******************************************************************************/
+   static Integer unassignedReplicaShards(JsonNode body, Integer unassigned)
+   {
+      if(unassigned == null || !body.path("indices").isObject() || !Integer.valueOf(0).equals(number(body, "initializing_shards")))
+      {
+         return (null);
+      }
+      for(JsonNode index : body.path("indices"))
+      {
+         Integer primaries = number(index, "number_of_shards");
+         if(primaries == null || !primaries.equals(number(index, "active_primary_shards")))
+         {
+            return (null);
+         }
+      }
+      return (unassigned);
+   }
+
+
+
    /*******************************************************************************
     ** Preserve unknown counters from incomplete cluster responses.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecks.java

@@ -0,0 +1,290 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.health;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthDetails;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+
+
+/*******************************************************************************
+ ** Central health projections of the existing, shared diagnostic snapshot.
+ ** Free-form errors and remote strings never cross this boundary.
+ *******************************************************************************/
+public class QuickSearchHealthChecks implements HealthCheckSupplier
+{
+   /*******************************************************************************
+    ** Register once, even if the producer is run again on the same instance.
+    *******************************************************************************/
+   public static void register(QInstance instance)
+   {
+      HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance);
+      if(registry.getChecks().stream().noneMatch(check -> "quick_search.opensearch".equals(check.getName())))
+      {
+         registry.withCheck(definition("quick_search.opensearch", input -> run(input, QuickSearchHealthChecks::opensearch)));
+      }
+      if(registry.getChecks().stream().noneMatch(check -> "quick_search.backlog".equals(check.getName())))
+      {
+         registry.withCheck(definition("quick_search.backlog", input -> run(input, QuickSearchHealthChecks::backlog)));
+      }
+      if(registry.getSuppliers().stream().noneMatch(supplier -> "quick_search.index".equals(supplier.getNamePrefix())))
+      {
+         registry.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix("quick_search.index").withComponent("quick_search")
+            .withSupplier(new QCodeReference(QuickSearchHealthChecks.class)));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Enumerate persisted index identities, including disabled indexes. Execution
+    ** reads a fresh cached snapshot, never a row captured at supplier resolution.
+    *******************************************************************************/
+   @Override
+   public List<HealthCheckMetaData> supply(QInstance instance) throws QException
+   {
+      QuickSearchRuntime runtime = runtime(instance);
+      if(runtime == null)
+      {
+         return (List.of());
+      }
+      List<HealthCheckMetaData> checks = new ArrayList<>();
+      for(QRecord row : QuickSearchConfigPrimer.queryAll(runtime.getConfig().getQuickSearchIndexTableName()))
+      {
+         String tableName = row.getValueString("tableName");
+         String name = tableName.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]+", "_").replaceAll("[._-]{2,}", "_").replaceAll("^[._-]|[._-]$", "");
+         if(name.isEmpty())
+         {
+            throw (new QException("Index name cannot form a health check identifier"));
+         }
+         checks.add(definition("quick_search.index." + name, input -> run(input, status -> index(status, tableName))));
+      }
+      return (checks);
+   }
+
+
+
+   /*******************************************************************************
+    ** All Quick Search checks are dependencies; none gate Kubernetes readiness.
+    *******************************************************************************/
+   private static HealthCheckMetaData definition(String name, HealthCheck check)
+   {
+      return (new HealthCheckMetaData().withName(name).withLabel(name).withComponent("quick_search")
+         .withCategory(HealthCategory.SEARCH).withKind(HealthCheckKind.DEPENDENCY)
+         .withIntervalSeconds(60).withTimeoutMs(5000).withCheckInstance(check));
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve from the supplied instance, not a static or previously closed runtime.
+    *******************************************************************************/
+   private static QuickSearchRuntime runtime(QInstance instance)
+   {
+      if(instance != null && instance.getQBits() != null)
+      {
+         for(QBitMetaData qbit : instance.getQBits().values())
+         {
+            if(qbit.getConfig() instanceof QuickSearchQBitConfig config)
+            {
+               return (config.getRuntime());
+            }
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Share the diagnostic worker and respect the deadline even on a cache hit.
+    ** Expected diagnostic failures use fixed messages, never exception payloads.
+    *******************************************************************************/
+   private static HealthCheckOutcome run(HealthCheckInput input, Function<QuickSearchStatus, HealthCheckOutcome> project)
+   {
+      if(!Instant.now().isBefore(input.getDeadline()))
+      {
+         return (HealthCheckOutcome.down("Quick Search health deadline exceeded"));
+      }
+      QuickSearchRuntime runtime = runtime(input.getQInstance());
+      if(runtime == null)
+      {
+         return (HealthCheckOutcome.unknown("Quick Search is not configured"));
+      }
+      try
+      {
+         QuickSearchStatus status = runtime.getStatusCache().get(runtime, input.getDeadline());
+         if(!Instant.now().isBefore(input.getDeadline()))
+         {
+            return (HealthCheckOutcome.down("Quick Search health deadline exceeded"));
+         }
+         return (project.apply(status));
+      }
+      catch(QException e)
+      {
+         return (HealthCheckOutcome.down("Quick Search diagnostics unavailable"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health is independent of backlog and per-index degradation.
+    *******************************************************************************/
+   private static HealthCheckOutcome opensearch(QuickSearchStatus status)
+   {
+      if(status.state() == QuickSearchStatus.State.DISABLED)
+      {
+         return (HealthCheckOutcome.unknown("Quick Search is disabled"));
+      }
+      HealthCheckOutcome outcome;
+      if(status.state() == QuickSearchStatus.State.DOWN || Boolean.FALSE.equals(status.clusterReachable()) || "red".equals(status.clusterHealth()))
+      {
+         outcome = HealthCheckOutcome.down("OpenSearch is unavailable or red");
+      }
+      else if(Boolean.TRUE.equals(status.clusterReachable()) && "green".equals(status.clusterHealth()))
+      {
+         outcome = HealthCheckOutcome.up("OpenSearch is green");
+      }
+      else if(Boolean.TRUE.equals(status.clusterReachable()) && QuickSearchStatus.isSingleNodeReplicaYellow(status.clusterHealth(), status.nodes(), status.unassignedReplicaShards()))
+      {
+         outcome = HealthCheckOutcome.up("OpenSearch is yellow only because replicas cannot be assigned on a single node");
+         outcome.getDetails().withValue("note", "replicas cannot be assigned on a single-node cluster");
+      }
+      else
+      {
+         outcome = HealthCheckOutcome.degraded("OpenSearch health is yellow or incomplete");
+      }
+      detail(outcome, "reachable", status.clusterReachable());
+      detail(outcome, "nodes", status.nodes());
+      detail(outcome, "activeShards", status.activeShards());
+      detail(outcome, "unassignedShards", status.unassignedShards());
+      detail(outcome, "unassignedReplicaShards", status.unassignedReplicaShards());
+      detail(outcome, "targetReplicas", status.targetReplicas());
+      detail(outcome, "replicaSettingsFailed", status.replicaSettingsFailed());
+      return (outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts are lower bounds when truncated; do not declare an incomplete backlog healthy.
+    *******************************************************************************/
+   private static HealthCheckOutcome backlog(QuickSearchStatus status)
+   {
+      HealthCheckOutcome outcome;
+      if(status.state() == QuickSearchStatus.State.DISABLED)
+      {
+         outcome = HealthCheckOutcome.unknown("Quick Search is disabled");
+      }
+      else if(status.pending() == null || status.exhausted() == null || status.indexingPaused() == null)
+      {
+         outcome = HealthCheckOutcome.degraded("Indexing backlog measurements unavailable");
+      }
+      else if(status.indexingPaused() || status.exhausted() > 0 || status.pending() > 100 || Boolean.TRUE.equals(status.backlogTruncated()))
+      {
+         outcome = HealthCheckOutcome.degraded("Indexing is paused or backlog needs attention");
+      }
+      else
+      {
+         outcome = HealthCheckOutcome.up("Indexing backlog is within limits");
+      }
+      detail(outcome, "pending", status.pending());
+      detail(outcome, "exhausted", status.exhausted());
+      detail(outcome, "indexingPaused", status.indexingPaused());
+      detail(outcome, "backlogTruncated", status.backlogTruncated());
+      return (outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Missing rows (including those outside the bounded snapshot) stay unknown.
+    ** Persisted error text is represented only by a boolean.
+    *******************************************************************************/
+   private static HealthCheckOutcome index(QuickSearchStatus status, String tableName)
+   {
+      IndexRow row = status.indexes().stream().filter(candidate -> tableName.equals(candidate.tableName())).findFirst().orElse(null);
+      if(row == null)
+      {
+         return (HealthCheckOutcome.unknown("Index measurements unavailable"));
+      }
+      boolean hasError = row.lastError() != null && !row.lastError().isBlank();
+      boolean mismatch = row.documentCount() != null && row.sourceCount() != null && !row.documentCount().equals(row.sourceCount());
+      HealthCheckOutcome outcome;
+      if(Boolean.FALSE.equals(row.enabled()))
+      {
+         outcome = HealthCheckOutcome.unknown("Index is disabled");
+      }
+      else if(!"ACTIVE".equals(row.status()) || hasError || mismatch || (row.realTimeErrorCount() != null && row.realTimeErrorCount() > 0))
+      {
+         outcome = HealthCheckOutcome.degraded("Index needs attention");
+      }
+      else
+      {
+         outcome = HealthCheckOutcome.up("Index is active");
+      }
+      outcome.withDetails(new HealthDetails().withValue("hasError", hasError).withValue("countMismatch", mismatch));
+      detail(outcome, "enabled", row.enabled());
+      detail(outcome, "documentCount", row.documentCount());
+      detail(outcome, "sourceCount", row.sourceCount());
+      detail(outcome, "countsCheckedAt", row.countsCheckedAt());
+      detail(outcome, "lastSuccessfulSync", row.lastSuccessfulSync());
+      detail(outcome, "realTimeErrorCount", row.realTimeErrorCount());
+      return (outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Unknown measurements are omitted rather than converted to zero.
+    *******************************************************************************/
+   private static void detail(HealthCheckOutcome outcome, String name, Object value)
+   {
+      if(value != null)
+      {
+         outcome.getDetails().withValue(name, value);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java

@@ -334,6 +334,8 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
             updateInput.setTableName(indexTable);
             updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id"))
                .withValue("realTimeErrorCount", Math.addExact(previous == null ? 0 : previous, entry.getValue()))));
+            // A runtime counter, not a configuration change: not record-audited.
+            updateInput.setOmitDmlAudit(true);
             List<QRecord> updated = QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)).getRecords();
             if(CollectionUtils.nullSafeIsEmpty(updated) || CollectionUtils.nullSafeHasContents(updated.get(0).getErrors()))
             {

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java

@@ -34,6 +34,8 @@ import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
 import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
@@ -113,17 +115,40 @@ public class QuickSearchOpenSearchClient implements Closeable
    public static final String PATH_ENGLISH         = "fieldValuesEnglish";
    public static final String PATH_EXACT           = "fieldValuesExact";
 
+   ////////////////////////////////////////////////////////////////////////////
+   // record search highlight markers: private-use characters, which do not //
+   // occur in indexed text, so fragments convert to plain text and ranges  //
+   ////////////////////////////////////////////////////////////////////////////
+   public static final String HIGHLIGHT_PRE_TAG       = "\uE000"; // private use area
+   public static final String HIGHLIGHT_POST_TAG      = "\uE001"; // private use area
+   public static final int    HIGHLIGHT_FRAGMENT_SIZE = 160;
+
    private final OpenSearchClient      client;
    private final OpenSearchAsyncClient asyncClient;
    private final OpenSearchTransport   transport;
    private final String                indexName;
    private final int                   maxBulkRequestBytes;
    private final ObjectMapper          sizingMapper;
+   private final Integer               indexReplicas;
+   private final Integer               maxAutoReplicas;
+   private final Boolean               replicasManaged;
 
    private volatile Boolean mappingOutdated = false;
    private volatile Boolean indexCreated    = false;
    private volatile Instant indexCreatedAt;
    private          Boolean closed          = false;
+   private volatile ReplicaStatus replicaStatus;
+
+
+
+   /*******************************************************************************
+    ** Outcome of the latest replica sizing: the target number_of_replicas, the
+    ** data nodes it was sized for (null when explicitly configured), how many
+    ** physical indexes were changed, and whether applying it failed.
+    *******************************************************************************/
+   public record ReplicaStatus(Integer targetReplicas, Integer dataNodes, Integer updatedIndexes, Boolean failed)
+   {
+   }
 
 
 
@@ -138,6 +163,9 @@ public class QuickSearchOpenSearchClient implements Closeable
       this.indexName           = config.getOpensearchIndexName();
       this.maxBulkRequestBytes = config.getMaxBulkRequestBytes() == null ? 5 * 1024 * 1024 : config.getMaxBulkRequestBytes();
       this.sizingMapper        = new ObjectMapper().registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+      this.indexReplicas       = config.getIndexReplicas();
+      this.maxAutoReplicas     = config.getMaxAutoReplicas() == null ? QuickSearchQBitConfig.DEFAULT_MAX_AUTO_REPLICAS : config.getMaxAutoReplicas();
+      this.replicasManaged     = !(config.getEffectiveAuthMode() == QuickSearchAuthMode.AWS_SIGV4 && "aoss".equals(config.getEffectiveAwsServiceName()));
    }
 
 
@@ -164,6 +192,7 @@ public class QuickSearchOpenSearchClient implements Closeable
                LOG.info("OpenSearch index already exists", logPair("indexName", indexName));
                extendMapping();
             }
+            reconcileReplicas();
             return;
          }
 
@@ -306,13 +335,21 @@ public class QuickSearchOpenSearchClient implements Closeable
    {
       try
       {
+         Integer replicas = replicasForNewIndex();
          client.indices().create(CreateIndexRequest.of(r -> r
             .index(physicalIndex)
-            .settings(s -> s
-               .analysis(a -> a
+            .settings(s ->
+            {
+               s.analysis(a -> a
                   .analyzer(INDEX_ANALYZER, an -> an.custom(c -> c.tokenizer("standard").filter("lowercase", "edge_ngram_filter")))
                   .analyzer(SEARCH_ANALYZER, an -> an.custom(c -> c.tokenizer("standard").filter("lowercase")))
-                  .filter("edge_ngram_filter", f -> f.definition(d -> d.edgeNgram(en -> en.minGram(2).maxGram(20))))))
+                  .filter("edge_ngram_filter", f -> f.definition(d -> d.edgeNgram(en -> en.minGram(2).maxGram(20)))));
+               if(replicas != null)
+               {
+                  s.numberOfReplicas(replicas);
+               }
+               return (s);
+            })
             .mappings(buildMapping())));
       }
       catch(OpenSearchException | java.io.IOException e)
@@ -323,6 +360,127 @@ public class QuickSearchOpenSearchClient implements Closeable
 
 
 
+   /*******************************************************************************
+    ** Replicas a new physical index should ask for; null leaves the server
+    ** default (Serverless, or the cluster could not be sized; logged).
+    *******************************************************************************/
+   private Integer replicasForNewIndex()
+   {
+      if(!Boolean.TRUE.equals(replicasManaged))
+      {
+         return (null);
+      }
+      try
+      {
+         ReplicaStatus target = targetReplicas();
+         replicaStatus = new ReplicaStatus(target.targetReplicas(), target.dataNodes(), 0, false);
+         return (target.targetReplicas());
+      }
+      catch(Exception e)
+      {
+         replicaStatus = new ReplicaStatus(null, null, 0, true);
+         LOG.warn("Could not size Quick Search index replicas; the new index uses the server default", e, logPair("indexName", indexName));
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The configured indexReplicas, else min(maxAutoReplicas, data nodes - 1)
+    ** from cluster health, so a single-node cluster gets 0 and stays green.
+    *******************************************************************************/
+   ReplicaStatus targetReplicas() throws QException
+   {
+      if(indexReplicas != null)
+      {
+         return (new ReplicaStatus(indexReplicas, null, 0, false));
+      }
+      QuickSearchDiagnosticsOperations.RawResponse health = new QuickSearchDiagnosticsOperations(this).raw("GET", "/_cluster/health", null);
+      QuickSearchDiagnosticsOperations.requireOk(health, "Cluster health unavailable");
+      JsonNode dataNodes = health.body() == null ? null : health.body().path("number_of_data_nodes");
+      if(dataNodes == null || !dataNodes.isNumber())
+      {
+         throw (new QException("Cluster health did not report number_of_data_nodes"));
+      }
+      return (new ReplicaStatus(autoReplicas(dataNodes.intValue(), maxAutoReplicas), dataNodes.intValue(), 0, false));
+   }
+
+
+
+   /*******************************************************************************
+    ** Automatic replica count: a replica can only be assigned to a different
+    ** data node than its primary, so never more than data nodes - 1.
+    *******************************************************************************/
+   static int autoReplicas(int dataNodes, int maxAutoReplicas)
+   {
+      return (Math.max(0, Math.min(maxAutoReplicas, dataNodes - 1)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bring number_of_replicas of the physical index(es) behind the alias to
+    ** the target. Idempotent (only differing indexes are updated) and failure
+    ** tolerant: a failure is logged and kept in {@link #getReplicaStatus()},
+    ** never thrown, so it cannot block startup. Indexes with
+    ** auto_expand_replicas set are left to OpenSearch. Null on Serverless.
+    *******************************************************************************/
+   public ReplicaStatus reconcileReplicas()
+   {
+      if(!Boolean.TRUE.equals(replicasManaged))
+      {
+         return (null);
+      }
+      Integer target = null;
+      Integer nodes  = null;
+      try
+      {
+         ReplicaStatus sized = targetReplicas();
+         target = sized.targetReplicas();
+         nodes  = sized.dataNodes();
+         QuickSearchDiagnosticsOperations             operations = new QuickSearchDiagnosticsOperations(this);
+         QuickSearchDiagnosticsOperations.RawResponse settings   = operations.raw("GET", "/" + QuickSearchDiagnosticsOperations.encode(indexName) + "/_settings", null);
+         QuickSearchDiagnosticsOperations.requireOk(settings, "Index settings unavailable");
+         int updated = 0;
+         for(Map.Entry<String, JsonNode> entry : settings.body().properties())
+         {
+            JsonNode index      = entry.getValue().path("settings").path("index");
+            String   current    = index.path("number_of_replicas").asText(null);
+            String   autoExpand = index.path("auto_expand_replicas").asText("false");
+            if(!"false".equalsIgnoreCase(autoExpand) || String.valueOf(target).equals(current))
+            {
+               continue;
+            }
+            QuickSearchDiagnosticsOperations.RawResponse put = operations.raw("PUT", "/" + QuickSearchDiagnosticsOperations.encode(entry.getKey()) + "/_settings",
+               "{\"index\":{\"number_of_replicas\":" + target + "}}");
+            QuickSearchDiagnosticsOperations.requireOk(put, "Updating number_of_replicas failed");
+            updated++;
+            LOG.info("Updated Quick Search index replicas", logPair("physicalIndex", entry.getKey()), logPair("from", current),
+               logPair("to", target), logPair("dataNodes", nodes));
+         }
+         replicaStatus = new ReplicaStatus(target, nodes, updated, false);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not reconcile Quick Search index replicas; continuing", e, logPair("indexName", indexName), logPair("targetReplicas", target));
+         replicaStatus = new ReplicaStatus(target, nodes, 0, true);
+      }
+      return (replicaStatus);
+   }
+
+
+
+   /*******************************************************************************
+    ** The latest replica sizing outcome; null before any (or on Serverless).
+    *******************************************************************************/
+   public ReplicaStatus getReplicaStatus()
+   {
+      return (replicaStatus);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -821,36 +979,94 @@ public class QuickSearchOpenSearchClient implements Closeable
 
 
    /*******************************************************************************
-    ** Record ids of the best matches in each table, best first, for core record
-    ** search. One _msearch request with one search per table (the same query and
-    ** ranking as {@link #search}, restricted to that table), so every table gets
-    ** its own top idsPerTable hits. Only sourceTable and recordId are fetched.
+    ** One hit of a record search: the record id, its relevance score, and its
+    ** highlight fragments by index field (e.g., "fieldValues.firstName"), with
+    ** matches between {@link #HIGHLIGHT_PRE_TAG} and {@link #HIGHLIGHT_POST_TAG}.
+    *******************************************************************************/
+   public record RecordSearchHit(String recordId, Double score, Map<String, List<String>> highlights)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The hits of a record search in one table, best first, and the table's
+    ** total number of matches (counted up to {@link #MAX_RESULT_WINDOW}; null
+    ** if unknown).
+    *******************************************************************************/
+   public record RecordSearchTableHits(List<RecordSearchHit> hits, Long totalHits)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Record ids of the best matches in each table, best first - see
+    ** {@link #searchRecordsPerTable}, which also returns scores, highlights and
+    ** totals.
+    *******************************************************************************/
+   public Map<String, List<String>> searchRecordIdsPerTable(String searchTerm, List<String> tableNames, int idsPerTable, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
+   {
+      Map<String, List<String>> idsByTable = new LinkedHashMap<>();
+      for(Map.Entry<String, RecordSearchTableHits> entry : searchRecordsPerTable(searchTerm, tableNames, idsPerTable, 0, false, tableConfigs, timeoutMillis).entrySet())
+      {
+         idsByTable.put(entry.getKey(), entry.getValue().hits().stream().map(RecordSearchHit::recordId).toList());
+      }
+      return (idsByTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** The best matches in each table, best first, for core record search. One
+    ** _msearch request with one search per table (the same query and ranking as
+    ** {@link #search}, restricted to that table), so every table gets its own
+    ** top hitsPerTable hits, after skipping offset. Only sourceTable and
+    ** recordId are fetched, plus each hit's score, the table's total hits
+    ** (counted up to MAX_RESULT_WINDOW) and - when highlight is set - one
+    ** highlight fragment per matching fieldValues.* field.
     **
     ** The whole request is bounded by timeoutMillis of wall-clock time (and the
     ** same server-side search timeout): when it runs out, the request is
     ** cancelled and a QException is thrown. A failed search for any table fails
     ** the call. Returns every requested table as a key, in the given order.
     *******************************************************************************/
-   public Map<String, List<String>> searchRecordIdsPerTable(String searchTerm, List<String> tableNames, int idsPerTable, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
+   public Map<String, RecordSearchTableHits> searchRecordsPerTable(String searchTerm, List<String> tableNames, int hitsPerTable, int offset, boolean highlight, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
    {
-      Map<String, List<String>> idsByTable = new LinkedHashMap<>();
+      Map<String, RecordSearchTableHits> hitsByTable = new LinkedHashMap<>();
       if(tableNames == null || tableNames.isEmpty())
       {
-         return (idsByTable);
+         return (hitsByTable);
       }
 
+      int from = Math.max(0, Math.min(offset, MAX_RESULT_WINDOW - hitsPerTable));
+
       List<RequestItem> searches = new ArrayList<>();
       for(String tableName : tableNames)
       {
          Query query = buildSearchQuery(searchTerm, List.of(tableName), tableConfigs);
          searches.add(RequestItem.of(ri -> ri
             .header(h -> h.index(indexName))
-            .body(b -> b
-               .query(query)
-               .size(idsPerTable)
-               .trackTotalHits(t -> t.enabled(false))
-               .source(sc -> sc.filter(f -> f.includes("sourceTable", "recordId")))
-               .timeout(timeoutMillis + "ms"))));
+            .body(b ->
+            {
+               b.query(query)
+                  .from(from)
+                  .size(hitsPerTable)
+                  .trackTotalHits(t -> t.count(MAX_RESULT_WINDOW))
+                  .source(sc -> sc.filter(f -> f.includes("sourceTable", "recordId")))
+                  .timeout(timeoutMillis + "ms");
+               if(highlight)
+               {
+                  b.highlight(h -> h
+                     .preTags(HIGHLIGHT_PRE_TAG)
+                     .postTags(HIGHLIGHT_POST_TAG)
+                     .requireFieldMatch(false)
+                     .numberOfFragments(1)
+                     .fragmentSize(HIGHLIGHT_FRAGMENT_SIZE)
+                     .fields("fieldValues.*", HighlightField.of(hf -> hf)));
+               }
+               return (b);
+            })));
       }
 
       CompletableFuture<MsearchResponse<OpenSearchDocument>> future = null;
@@ -874,17 +1090,19 @@ public class QuickSearchOpenSearchClient implements Closeable
                throw (new QException("Record search failed for table [" + tableNames.get(i) + "]: " + reason));
             }
 
-            List<String> ids = new ArrayList<>();
+            List<RecordSearchHit> hits = new ArrayList<>();
             for(Hit<OpenSearchDocument> hit : item.result().hits().hits())
             {
                if(hit.source() != null && hit.source().getRecordId() != null)
                {
-                  ids.add(hit.source().getRecordId());
+                  hits.add(new RecordSearchHit(hit.source().getRecordId(), hit.score(), hit.highlight()));
                }
             }
-            idsByTable.put(tableNames.get(i), ids);
+
+            Long totalHits = item.result().hits().total() == null ? null : item.result().hits().total().value();
+            hitsByTable.put(tableNames.get(i), new RecordSearchTableHits(hits, totalHits));
          }
-         return (idsByTable);
+         return (hitsByTable);
       }
       catch(TimeoutException e)
       {

src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java

@@ -493,7 +493,10 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Update fields on a quick_search_index row.
+    ** Update runtime state on a quick_search_index row (status, last run, last
+    ** basepull time, document count). Indexing bookkeeping, not a configuration
+    ** change: it is not record-audited, or every basepull tick would write an
+    ** audit row. Admin edits to the row go through the table and stay audited.
     *******************************************************************************/
    protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
    {
@@ -506,6 +509,7 @@ public abstract class AbstractIndexingStep implements BackendStep
       UpdateInput updateInput = new UpdateInput();
       updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
       updateInput.setRecords(List.of(record));
+      updateInput.setOmitDmlAudit(true);
       QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput));
    }
 

src/main/resources/help/qbit-quick-search.yaml

@@ -0,0 +1,552 @@
+bundle: qbit-quick-search
+version: ${project.version}
+layer: QBIT
+entries:
+- key: app:quickSearchAdmin;slot:help
+  summary: Control which tables feed the site search, how their fields are weighted, and whether the search index is healthy.
+  content: |
+    Quick Search keeps a copy of chosen tables and fields in a search engine so the global search box finds records fast and ranks them by relevance. Search applies your table permissions and, when enabled by your application, record security checks.
+
+    Use **{{label:quick_search_index}}** to add or disable tables, **{{label:quick_search_index_field}}** to tune which fields count and how much, and the run, failed-event and diagnostic tables to see how indexing is going. Tools such as Search Playground and Validate Search Index are actions on the index list.
+
+    Your application controls whether indexing and validation run on a schedule. You can also run these actions manually.
+- key: table:quick_search_index;slot:help
+  summary: One row per table that the site search covers, with its schedule, status and when it was last updated.
+  content: |
+    **How rows get created:** automatically. On startup the qbit adds a row for every table declared searchable in code (from your application settings) and queues its first build. Add a table yourself with the Add Search Index action; Disable or Remove Search Index stops indexing it. A table you removed is remembered (Removed Config) and not added back. Change the interval or the fields, then run Apply Search Index Changes to rebuild.
+
+    This is the control panel for search. Each row says whether a table is searchable, how often it is checked for changes, and where indexing stands (for example up to date, building, or needing a rebuild).
+
+    Use the list actions to **{{label:quickSearchAddIndex}}**, and on selected rows **{{label:quickSearchApplyIndexChanges}}**, **{{label:quickSearchDisableIndex}}** or **{{label:quickSearchRemoveIndex}}**. Diagnostic tools such as **{{label:quickSearchPlayground}}**, **{{label:quickSearchValidateIndex}}** and **{{label:quickSearchAnalyzeText}}** also start here. You cannot create or delete rows with the standard screens.
+
+    **Searchable Fields** (on the record) lists which fields count and how heavily; **Recent Runs** lists the latest indexing runs.
+
+    Edit access needs the Quick Search permissions; diagnostics are read-only except rebuilding a record's search entry.
+- key: table:quick_search_index;section:identity
+  summary: The table, whether it is searchable, its status and how often it is checked for changes.
+- key: table:quick_search_index;section:fields
+  summary: The fields of this table that are searched, each with its weight and matching style.
+- key: table:quick_search_index;section:runs
+  summary: The most recent indexing runs for this table.
+- key: table:quick_search_index;section:activity
+  summary: When the table was last updated, reconciled and rebuilt, plus record and document counts and errors.
+- key: table:quick_search_index;section:origin
+  summary: Whether this row came from the code defaults or was added by an owner, and the fields as last built.
+- key: table:quick_search_index;field:tableName
+  summary: The table whose records are searchable.
+  content: |
+    Set when the index is created and cannot be changed afterwards. Remove the index and add it again to switch tables.
+- key: table:quick_search_index;field:enabled
+  summary: Turns search on or off for this table.
+  content: |
+    Turning it off stops indexing straight away. Turning a disabled table back on queues a fresh build. For a one-off clean-up use **{{label:quickSearchDisableIndex}}**, which also lets you choose whether to delete the search documents.
+- key: table:quick_search_index;field:basepullIntervalMinutes
+  summary: The least time, in minutes, between scheduled checks for changed records in this table.
+  content: |
+    When scheduled updates are enabled, a table is checked only after this interval has passed. Real-time indexing, when enabled, also sends new and changed records after they are saved.
+- key: table:quick_search_index;field:basepullTimestampField
+  summary: The date field used to find records changed since the last check; usually the modified date.
+  content: |
+    Leave it blank if the table has no such field. You can keep it current with reconciliation and, when enabled, real-time updates instead of incremental checks.
+- key: table:quick_search_index;field:source
+  summary: Where this row came from.
+  content: |
+    - **Code default**: seeded from the application's built-in search settings.
+    - **Added by owner**: created with **{{label:quickSearchAddIndex}}**.
+- key: table:quick_search_index;field:ownerEdited
+  summary: Yes once an owner has changed this row, so the built-in defaults never overwrite it.
+- key: table:quick_search_index;field:searchableFieldsJson
+  summary: The field list that was in force when this table's search was last built, for spotting changes since.
+  content: |
+    Read-only. If it differs from **Searchable Fields**, the table needs **{{label:quickSearchApplyIndexChanges}}**.
+- key: table:quick_search_index;field:lastBasepullTime
+  summary: When the scheduled change check last ran for this table.
+- key: table:quick_search_index;field:lastFullReindexTime
+  summary: When this table was last completely rebuilt.
+- key: table:quick_search_index;field:lastReconcileTime
+  summary: When the last reconcile finished, which re-indexes everything and removes documents whose record is gone.
+- key: table:quick_search_index_field;slot:help
+  summary: One searchable field of an index, with its weight, role and matching style.
+  content: |
+    **How rows get created:** automatically, with the index row, from the field annotations or the host's field list. Edit a weight to make matches in that field rank higher, or add a field by hand; your edits are kept and never overwritten by startup. Run Apply Search Index Changes so existing search documents reflect your changes.
+
+    Add, edit or remove fields from the **Searchable Fields** list on a **{{label:quick_search_index}}** record. A field must exist on the table, be indexable (hidden and sensitive fields never are) and appear only once per index.
+
+    Changes affect new searches and real-time updates at once, and mark the index as needing a rebuild so existing documents catch up. Run **{{label:quickSearchApplyIndexChanges}}** to preview and rebuild. Deleting a field is remembered, so the built-in defaults do not bring it back.
+- key: table:quick_search_index_field;section:identity
+  summary: Which index and field this is, and how much it counts and in what role.
+- key: table:quick_search_index_field;section:matching
+  summary: How the field is analyzed, whether it shows in snippets, and its display order.
+- key: table:quick_search_index_field;section:origin
+  summary: Whether the field came from code defaults or an owner, and when it was created and changed.
+- key: table:quick_search_index_field;field:quickSearchIndexId
+  summary: The index this field belongs to.
+- key: table:quick_search_index_field;field:fieldName
+  summary: The table field to search. Only indexable, non-sensitive fields are offered.
+  content: |
+    Required. Each field can be listed once per index.
+- key: table:quick_search_index_field;field:weight
+  summary: How much a match in this field counts toward ranking, from 1 to 1000. Higher ranks higher.
+  content: |
+    Defaults to {{default:quick_search_index_field.weight}}. Give titles and names higher weights than long body text.
+- key: table:quick_search_index_field;field:role
+  summary: What the field is used for in search results.
+  content: |
+    - **T1 display**: shown as the hit's short details.
+    - **Title**: becomes the hit's main label.
+    - **Subtitle**: shown under the title.
+    - **Body**: ordinary searchable text. Default: {{default:quick_search_index_field.role}}.
+    - **Keyword**: matched on its own and kept out of the highlighted snippet.
+    - **Exact**: matched on its own, as an exact value, and kept out of the snippet.
+- key: table:quick_search_index_field;field:analyzer
+  summary: How the field's text is split up for matching; leave blank to choose by role.
+  content: |
+    - **Prefix (as you type)**: matches the start of words, so partial words find results.
+    - **Whole words**: only complete words match.
+    - **English (stemmed)**: also matches word forms, such as plural and singular.
+    - **Exact value**: the whole value must match.
+    Changing it marks the table for a rebuild.
+- key: table:quick_search_index_field;field:includeInSnippet
+  summary: Whether matches in this field are shown in the highlighted snippet; blank follows the role.
+  content: |
+    Keyword and Exact fields stay out of snippets by default.
+- key: table:quick_search_index_field;field:includeLabel
+  summary: Adds the field's label to its text when indexed, so searching for the label also finds the record.
+  content: |
+    Default {{default:quick_search_index_field.includeLabel}}.
+- key: table:quick_search_index_field;field:sortOrder
+  summary: The order the fields are listed and combined in. Lower numbers come first.
+- key: table:quick_search_index_field;field:source
+  summary: Where this field row came from.
+  content: |
+    - **Code default**: seeded from the application's built-in search settings.
+    - **Added by owner**: added by an administrator.
+- key: table:quick_search_index_field;field:ownerEdited
+  summary: Yes once an owner has changed this field, so built-in defaults never overwrite it.
+- key: table:quick_search_index_run;slot:help
+  summary: 'A history entry for one indexing run: what kind it was, how it ended and how many records it handled.'
+  content: |
+    **How rows get created:** automatically. Each Full Reindex, Basepull (the scheduled refresh) and Reconcile writes a row, and the first build of a new index writes the first one. Nothing to set up. Cancel Index Run stops one that is still running. Old rows are purged after the run history retention period.
+
+    Runs are written by the system as indexing happens and are purged after the configured history retention period. A run can be an update (changed records), a reconcile (re-index everything and remove leftovers) or a full rebuild.
+
+    To stop a rebuild that is still running, select the run and use **{{label:quickSearchCancelRun}}**. It stops after its current batch and ends as Cancelled with its counts.
+- key: table:quick_search_index_run;section:run
+  summary: Which index, what kind of run, its status and when it started and ended.
+- key: table:quick_search_index_run;section:counts
+  summary: How many records were processed, indexed and failed, and any error text.
+- key: table:quick_search_index_run;section:cancel
+  summary: Whether a cancel was requested for this run and by whom.
+- key: table:quick_search_index_run;section:dates
+  summary: When the run was created and when it last reported progress.
+- key: table:quick_search_index_run;field:quickSearchIndexId
+  summary: The index (table) this run worked on.
+- key: table:quick_search_index_run;field:runType
+  summary: What kind of run it was.
+  content: |
+    - **Update (basepull)**: re-indexes records changed since the last check and retries failed events.
+    - **Reconcile**: re-indexes everything in place, then removes documents whose record is gone.
+    - **Full reindex / build**: builds the table from scratch, either into a fresh index or as a first build.
+- key: table:quick_search_index_run;field:status
+  summary: How the run is going or ended.
+  content: |
+    - **Running**: in progress.
+    - **Completed**: finished.
+    - **Failed**: stopped by an error (see Error).
+    - **Cancelled**: stopped by an owner using Cancel Index Run.
+- key: table:quick_search_index_run;field:startTime
+  summary: When the run started.
+- key: table:quick_search_index_run;field:endTime
+  summary: When the run finished; blank while it is still running.
+- key: table:quick_search_index_run;field:recordsProcessed
+  summary: How many source records the run looked at.
+- key: table:quick_search_index_run;field:recordsIndexed
+  summary: How many records were written to the search engine.
+- key: table:quick_search_index_run;field:errorCount
+  summary: How many records failed to index in this run.
+- key: table:quick_search_index_run;field:errorMessage
+  summary: The error text when the run failed or had record errors.
+- key: table:quick_search_index_run;field:cancelRequested
+  summary: Yes once someone asked to cancel this run; the run stops after its current batch.
+- key: table:quick_search_index_run;field:cancelRequestedBy
+  summary: The person who asked to cancel the run, filled in by Cancel Index Run.
+- key: table:quick_search_failed_event;slot:help
+  summary: A record change that could not be sent to search, kept so the system can retry it.
+  content: |
+    **How rows get created:** by the qbit. When saving a record cannot update OpenSearch, the change is recorded here and the next refresh replays it. This table is normally empty, which is the healthy state. Rows that stay here mean OpenSearch is unreachable or rejecting a document; check the home status.
+
+    When a save or delete fails to reach the search engine, it is noted here and the scheduled update retries it, up to 10 times. After that it is marked as exhausted and stops being retried.
+
+    Open a row to compare the record with its search entry using **{{label:quickSearchInspect_quick_search_failed_event}}**, or use **{{label:quickSearchReindexRecords_quick_search_failed_event}}** to rebuild the entry yourself. Successful retries remove the row, so an empty list is healthy.
+- key: table:quick_search_failed_event;section:event
+  summary: Which table and record the failed change was for, the action and its status.
+- key: table:quick_search_failed_event;section:error
+  summary: The error text and how many times it was retried.
+- key: table:quick_search_failed_event;section:dates
+  summary: When the failure was first recorded and last updated.
+- key: table:quick_search_failed_event;field:tableName
+  summary: The table the failed change belongs to.
+- key: table:quick_search_failed_event;field:recordId
+  summary: The id of the record whose search entry could not be updated.
+- key: table:quick_search_failed_event;field:action
+  summary: What was being done to the search entry.
+  content: |
+    - **Index**: add or update the record's search entry.
+    - **Delete**: remove the record's search entry.
+- key: table:quick_search_failed_event;field:errorMessage
+  summary: Why the update failed, as reported by the search engine or the system.
+- key: table:quick_search_failed_event;field:attempts
+  summary: How many times the scheduled update has retried this change. It gives up at 10.
+- key: table:quick_search_failed_event;field:status
+  summary: Where the retry stands.
+  content: |
+    - **Pending (retried by the next update)**: will be tried again.
+    - **Exhausted (gave up)**: failed 10 times; needs a manual **{{label:quickSearchReindexRecords_quick_search_failed_event}}**.
+    - **Awaiting full reindex**: a delete made while a full rebuild runs; applied after the rebuild finishes.
+    - **Deferred (indexing paused)**: held while indexing is paused; indexed after resume.
+- key: table:quick_search_diagnostic_event;slot:help
+  summary: A log of search problems and diagnostic tool runs, to help find out why search misbehaved.
+  content: |
+    **How rows get created:** by the qbit. A search that hits an OpenSearch error or timeout writes a row (core search then falls back to its own), and so does each run of a diagnostic tool such as Search Playground. Rows are read-only. An empty table means no search has failed.
+
+    Rows are written when a search request to the search engine fails or times out (the site then falls back to its own simpler search), when an administrator runs a diagnostic tool or a Reindex in Search, and when a tool run is refused for going over the rate limit.
+
+    A failed search stores only the length of the term, never the term itself. Rows are read-only.
+- key: table:quick_search_diagnostic_event;section:identity
+  summary: When it happened, what kind of event it was, where it came from and the outcome.
+- key: table:quick_search_diagnostic_event;section:request
+  summary: Who triggered it, which tables, the search term, how long it took and which server handled it.
+- key: table:quick_search_diagnostic_event;section:error
+  summary: The error class, message and any extra detail.
+- key: table:quick_search_diagnostic_event;field:eventType
+  summary: What kind of event this is.
+  content: |
+    - **SEARCH_FAILED**: a search request to the search engine failed or timed out.
+    - **TOOL_RUN**: a diagnostic tool was run.
+    - **REINDEX**: records were re-indexed with Reindex in Search.
+    - **RATE_LIMITED**: a tool run was refused because it was run too often.
+- key: table:quick_search_diagnostic_event;field:source
+  summary: 'What raised the event: the global search, the Quick Search search, or the name of the tool that was run.'
+- key: table:quick_search_diagnostic_event;field:outcome
+  summary: 'How it ended: OK, ERROR, or REFUSED when the run was blocked.'
+- key: table:quick_search_diagnostic_event;field:userName
+  summary: The person who triggered the event, when there is one.
+- key: table:quick_search_diagnostic_event;field:tableNames
+  summary: The tables involved in the search or tool run, as a list.
+- key: table:quick_search_diagnostic_event;field:searchTerm
+  summary: The term typed into a diagnostic tool, up to 100 characters. Blank for failed searches.
+- key: table:quick_search_diagnostic_event;field:durationMillis
+  summary: How long the request or tool run took, in milliseconds.
+- key: table:quick_search_diagnostic_event;field:errorClass
+  summary: The technical type of the error, useful when reporting a problem.
+- key: table:quick_search_diagnostic_event;field:errorMessage
+  summary: The error text returned by the search engine or the system.
+- key: table:quick_search_diagnostic_event;field:detail
+  summary: Extra context for the event, such as what the tool did or the term length for a failed search. Up to 4,000 characters.
+- key: table:quick_search_diagnostic_event;field:hostName
+  summary: The server that recorded the event, which helps when several servers run the site.
+- key: table:quick_search_config_tombstone;slot:help
+  summary: A reminder that an owner removed a searchable table or field, so built-in defaults do not add it back.
+  content: |
+    **How rows get created:** this table starts empty on purpose. A row appears when you Remove Search Index or remove a field that code still declares as searchable. Delete a row to let the code defaults add that table or field again on the next startup.
+
+    **{{label:quickSearchRemoveIndex}}** and deleting an index field each write one of these. At startup the application's built-in search settings skip anything listed here.
+
+    Delete the row if you want the default to be seeded again on the next start. Rows are created automatically; you normally never edit them.
+- key: table:quick_search_config_tombstone;field:tableName
+  summary: The table that was removed from search, or whose field was removed.
+- key: table:quick_search_config_tombstone;field:fieldName
+  summary: The removed field; blank means the whole table was removed.
+- key: table:quick_search_config_tombstone;field:removedBy
+  summary: The person who removed it.
+- key: table:quick_search_config_tombstone;field:reason
+  summary: An optional note on why it was removed.
+- key: table:quick_search_index;field:lastRunStatus
+  summary: The outcome of the most recent index run for this table, such as "Update Completed" or "Full reindex Failed".
+  content: |
+    You see the run kind followed by its outcome. Filled in automatically at the end of every run. A failed run is flagged as a warning in **{{label:quickSearchValidateIndex}}** and counts as needing attention on the Quick Search home.
+- key: table:quick_search_index;field:lastErrorMessage
+  summary: Why the most recent run failed for this table; blank when it went fine.
+  content: |
+    Filled in automatically at the end of each run and cut off at 4,000 characters. A successful run clears it. See **Recent Runs** on this record for the full run history.
+- key: table:quick_search_index;field:recordCount
+  summary: How many records of this table the last full rebuild or reconcile read from the database.
+  content: |
+    Updated automatically by **Rebuild Search Index** and **Reconcile Search Index**. Compare it with **Document Count** to spot gaps.
+- key: table:quick_search_index;field:documentCount
+  summary: How many search documents this table had in the index after the last rebuild or reconcile.
+  content: |
+    This is a saved number, not a live count, so it can lag behind recent edits. The Quick Search home and **{{label:quickSearchValidateIndex}}** compare live counts with the database.
+- key: table:quick_search_index;field:realTimeErrorCount
+  summary: Count of errors from instant (per-change) indexing; any value above zero marks the index as needing attention.
+  content: |
+    This count increases when real-time changes fail to reach search. A successful update or full rebuild clears it.
+- key: table:quick_search_index;field:status
+  summary: Where this table's index is in its life cycle.
+  content: |
+    Set automatically; you cannot edit it. Values:
+    - **ACTIVE**: indexed and in use.
+    - **PENDING_BUILD**: queued for its first build or a rebuild; starts with the next update.
+    - **BUILDING**: the first build is running.
+    - **REBUILDING**: a full rebuild is running; search keeps using the old documents until it finishes.
+    - **Needs Reindex**: the searchable fields changed since the last build. Run **{{label:quickSearchApplyIndexChanges}}** to rebuild.
+- key: table:quick_search_settings;slot:help
+  summary: The one-row switch that pauses or resumes all search indexing, plus a record of the last automatic self-priming.
+  content: |
+    **How rows get created:** the application creates the settings row on first startup; you do not need to add it. Use the Pause Indexing and Resume Indexing actions to stop and restart all indexing while search keeps answering from the existing index. The priming fields are written automatically and are read-only.
+
+    Open the single row to pause indexing (set **{{label:quick_search_settings.indexingPaused}}**, with an optional reason) or resume it. You can also use the **{{label:quickSearchPauseIndexing}}** and **{{label:quickSearchResumeIndexing}}** actions.
+
+    While paused, search keeps serving the current index, instant and scheduled indexing stop on every server, and a running rebuild stops after its current batch. Changes made while paused are kept and indexed after you resume. Rows cannot be added or deleted here. Needs the Quick Search admin permission.
+- key: table:quick_search_settings;section:pause
+  summary: Whether indexing is paused, why, and who paused it.
+- key: table:quick_search_settings;section:priming
+  summary: What the last automatic self-priming added to the index list.
+- key: table:quick_search_settings;section:dates
+  summary: When this settings row was created and last changed.
+- key: table:quick_search_settings;field:indexingPaused
+  summary: Turn on to stop all search indexing on every server; turn off to resume.
+  content: |
+    When turned on, **{{label:quick_search_settings.pausedAt}}** and **{{label:quick_search_settings.pausedBy}}** are filled in. When turned off, they and the reason are cleared, and changes made while paused are queued to be indexed by the next update. Search keeps working on the existing index in the meantime.
+- key: table:quick_search_settings;field:pauseReason
+  summary: An optional note on why indexing was paused, shown to other admins.
+  content: |
+    Cleared automatically when indexing resumes.
+- key: table:quick_search_settings;field:pausedAt
+  summary: When indexing was paused; filled in automatically and blank while indexing is running.
+- key: table:quick_search_settings;field:pausedBy
+  summary: The user who paused indexing; filled in automatically and blank while indexing is running.
+- key: table:quick_search_settings;field:lastPrimedAt
+  summary: When Quick Search last checked the code defaults and added missing indexes and fields.
+  content: |
+    Self-priming runs automatically at startup. It only adds what is missing; it never deletes anything and never changes a row you edited.
+- key: table:quick_search_settings;field:lastPrimingSummary
+  summary: A plain-language summary of what the last self-priming added or changed.
+  content: |
+    Filled in automatically.
+- key: table:quick_search_settings;field:lastPrimingNewTables
+  summary: How many new tables the last self-priming added to the search index list.
+  content: |
+    Filled in automatically.
+- key: table:quick_search_validation_run;slot:help
+  summary: Saved reports from Validate Search Index, each one a health check of the search index against the database.
+  content: |
+    **How rows get created:** run the Validate Search Index action from the Search Indexes table (it can also run on a schedule); each run writes one row with its status and report. It starts empty until someone validates. Rows are kept as history and purged after the run history retention period.
+
+    A report is saved every time **{{label:quickSearchValidateIndex}}** runs, either by you or on the configured schedule. Open a run to read the checks in plain language, with the raw data below.
+
+    The check covers cluster and index health, the field mapping, database rows against indexed documents, and a sample of records for missing, stale and orphaned documents. Validation only reads; it changes nothing. Reports are read-only and cannot be edited or deleted here.
+- key: table:quick_search_validation_run;section:identity
+  summary: The outcome of this validation, when it ran and how it was started.
+- key: table:quick_search_validation_run;section:scope
+  summary: Who ran it, which tables and how many records were checked, and the counts of failures and warnings.
+- key: table:quick_search_validation_run;section:report
+  summary: The full validation report, check by check.
+- key: table:quick_search_validation_run;section:data
+  summary: The same report as raw JSON, for copying or deeper inspection.
+- key: table:quick_search_validation_run;field:status
+  summary: 'The overall verdict: PASS, WARN or FAIL.'
+  content: |
+    FAIL means at least one check failed, WARN means no failures but at least one warning, and PASS means everything checked out.
+- key: table:quick_search_validation_run;field:triggerType
+  summary: Whether a person started this run (MANUAL) or the configured schedule did (SCHEDULED).
+- key: table:quick_search_validation_run;field:userName
+  summary: The user who started a manual run; blank for scheduled runs.
+- key: table:quick_search_validation_run;field:tableNames
+  summary: The tables this run checked, comma separated.
+- key: table:quick_search_validation_run;field:sampleSize
+  summary: How many records and documents were sampled per table (at most 1,000).
+- key: table:quick_search_validation_run;field:tablesChecked
+  summary: How many tables were checked.
+- key: table:quick_search_validation_run;field:failCount
+  summary: Number of checks that failed.
+- key: table:quick_search_validation_run;field:warnCount
+  summary: Number of checks that raised a warning.
+- key: table:quick_search_validation_run;field:summary
+  summary: 'One-line result, such as "WARN: 14 table(s), 0 failed check(s), 2 warning(s)."'
+- key: table:quick_search_validation_run;field:reportHtml
+  summary: 'The readable report: each check with its result and what to do about it.'
+- key: table:quick_search_validation_run;field:reportJson
+  summary: The same report as machine-readable JSON.
+- key: table:quick_search_validation_run;field:startTime
+  summary: When the validation started.
+- key: table:quick_search_validation_run;field:endTime
+  summary: When the validation finished.
+- key: process:quickSearchBasepullIndex;slot:help
+  summary: Catch the search index up with recent database changes for the tables you select.
+  content: |
+    Re-indexes records changed since the last update, using each table's timestamp field. It also retries failed instant updates and builds any index that is queued. If your application enables scheduled updates, this also runs automatically at its configured interval. Tables without a timestamp field are covered by **Reconcile Search Index** instead. Does nothing while indexing is paused. Needs the Quick Search admin permission.
+- key: process:quickSearchAddIndex;slot:help
+  summary: Start searching a table that is not in the search index yet.
+  content: |
+    Pick a table you can read that is not indexed. The review step shows which fields the index starts with and about how many records its first build will index. Sensitive and hidden fields are never indexed. The new index is queued and its first build starts with the next update. Afterwards, change which fields are searched under **Searchable Fields** on the index and run **{{label:quickSearchApplyIndexChanges}}**. Needs the Quick Search admin permission.
+- key: process:quickSearchAddIndex;field:searchTableName
+  summary: The table to make searchable; only tables you can read and that are not indexed yet are listed.
+- key: process:quickSearchAddIndex;field:basepullIntervalMinutes
+  summary: How often, in minutes, changes to this table are picked up; 0 means on every update run.
+  content: |
+    When real-time indexing is enabled, saved changes are also sent after commit. Cannot be negative. The standard interval is already filled in.
+- key: process:quickSearchAddIndex;field:basepullTimestampField
+  summary: The date-time field used to spot changed records; leave blank to let Quick Search pick one.
+  content: |
+    If the table has no suitable field, you can use **Reconcile Search Index** and, when enabled, real-time indexing to keep it current.
+- key: process:quickSearchAddIndex;field:enabled
+  summary: Turn off to add the table without indexing it yet.
+  content: |
+    If off, nothing is indexed until you enable it, and enabling it queues the build.
+- key: process:quickSearchDisableIndex;slot:help
+  summary: Stop indexing the selected tables and hide them from search, keeping the setup so you can turn them back on.
+  content: |
+    Choose whether to keep the search documents or delete them. Enabling the index again rebuilds it. Tables already disabled are skipped. Needs the Quick Search admin permission.
+- key: process:quickSearchDisableIndex;field:documents
+  summary: Whether to keep the table's search documents or delete them from the search index; the default is Keep.
+- key: process:quickSearchRemoveIndex;slot:help
+  summary: Take the selected tables out of Quick Search for good, deleting their index setup.
+  content: |
+    Indexing stops now, and the index row, its field rows and its pending failed events are deleted. The removal is remembered under **{{label:quick_search_config_tombstone}}**, so tables declared in code are not added back by the automatic self-priming. Choose whether to delete the search documents or keep them (the reconcile removes documents of removed tables). Needs the Quick Search admin permission.
+- key: process:quickSearchRemoveIndex;field:documents
+  summary: Whether to delete the table's search documents from the search index or keep them; the default is Delete.
+- key: process:quickSearchApplyIndexChanges;slot:help
+  summary: Rebuild the selected tables so field changes you made take effect in search.
+  content: |
+    The review shows what was added, changed or removed since the last build. Disabled tables must be enabled first, and tables already queued or building are skipped. Rebuilds start with the next update. Search stays available while the table is rebuilt in place; results can change during the rebuild. Needs the Quick Search admin permission.
+- key: process:quickSearchCancelRun;slot:help
+  summary: Ask a running index run to stop after its current batch.
+  content: |
+    Only runs that are still running can be cancelled; the run ends as Cancelled. Work already done stays in the index. Needs the Quick Search admin permission.
+- key: process:quickSearchPauseIndexing;slot:help
+  summary: Pause all search indexing on every server until you resume it.
+  content: |
+    Search keeps serving the current index. Instant and scheduled indexing stop, and a running rebuild stops after its current batch. Changes made meanwhile are kept and indexed after you resume. Needs the Quick Search admin permission. See also **{{label:quickSearchResumeIndexing}}**.
+- key: process:quickSearchPauseIndexing;field:pauseReason
+  summary: A short note for other admins on why you are pausing; shown on the Quick Search Settings screen.
+- key: process:quickSearchResumeIndexing;slot:help
+  summary: Start search indexing again after a pause.
+  content: |
+    Changes made while paused are queued and indexed by the next update. Needs the Quick Search admin permission.
+- key: process:quickSearchPlayground;slot:help
+  summary: Try a search term and see exactly how the global search handles it, table by table, without changing anything.
+  content: |
+    Runs the term as you, so locks and permissions apply. For each table you see the path it takes, the results you would see, and the underlying query, scores and matched fields. Leave **Tables** blank to search every searchable table. To understand one result, paste its document id (for example customer:42) into **Explain hit**. Limited in how often it can be run. Needs the Quick Search debug permission.
+- key: process:quickSearchPlayground;field:searchTerm
+  summary: The words to search for, exactly as someone would type them in the search box.
+- key: process:quickSearchPlayground;field:tableNames
+  summary: Limit the test to these tables; leave blank to try all searchable tables.
+  content: |
+    Use the table names separated by commas. If you started the action from selected indexes, they are filled in for you.
+- key: process:quickSearchPlayground;field:limitPerTable
+  summary: How many results to show from each table; the default is 5.
+  content: |
+    The search settings cap how many one table can return.
+- key: process:quickSearchPlayground;field:explainHit
+  summary: 'Optional: a document id, written as table name, colon, record id, to see why it scored as it did.'
+  content: |
+    Copy the id from a result shown above. Example: customer:42.
+- key: process:quickSearchValidateIndex;slot:help
+  summary: Check the search index for problems and save a report you can read later.
+  content: |
+    Checks the search cluster and index health, field mapping and analyzers, database rows against indexed documents, and a sample of records and documents for missing, stale and orphaned ones. It only reads. The result is saved on **{{label:quick_search_validation_run}}**. Your application can schedule validation for all tables. Select indexes first to check only those tables. Only one validation runs at a time per server. Needs the Quick Search debug permission.
+- key: process:quickSearchValidateIndex;field:sampleSize
+  summary: How many records and documents to spot-check per table; at most 1,000.
+  content: |
+    Larger samples find more problems but take longer. Your application supplies the initial sample size.
+- key: process:quickSearchAnalyzeText;slot:help
+  summary: See how the search index would break a piece of text into searchable words, and whether a search term would match it.
+  content: |
+    Type text as it would be stored in a field. You get the tokens each analyzer produces for it. Add a search term to check whether every word of the term is among the text's tokens, which is how a document's searchable text matches. Changes nothing. Needs the Quick Search debug permission.
+- key: process:quickSearchAnalyzeText;field:analyzeText
+  summary: The text to analyze, as it would appear in a record field.
+- key: process:quickSearchAnalyzeText;field:analyzeQuery
+  summary: Optional search term to test against the text above.
+- key: process:quickSearchInspect_quick_search_failed_event;slot:help
+  summary: Compare the record behind this failed event with its search document, and see why a term does or does not match.
+  content: |
+    Shows the document in the search index next to the database record. Add a search term to get a match analysis. Read-only, up to 10 records at a time. Needs the Quick Search debug permission.
+- key: process:quickSearchInspect_quick_search_failed_event;field:searchTerm
+  summary: 'Optional: a word or phrase you expected to find this record with; the result explains whether and why it matches.'
+- key: process:quickSearchReindexRecords_quick_search_failed_event;slot:help
+  summary: Rebuild the search documents for the records behind the selected events, straight from the database.
+  content: |
+    Records that no longer exist have their documents removed. The records themselves are not changed. Up to 500 records at a time; you confirm before it runs. The run is logged in **{{label:quick_search_diagnostic_event}}**. Needs the Quick Search admin permission.
+- key: process:quickSearchFullReindex;slot:help
+  summary: Rebuild the search index from the database, for every table or just one.
+  content: |
+    Leave **Table** blank to rebuild every enabled table into a fresh index and swap it in when all succeed, so search keeps serving the old documents until then. This also picks up field-mapping changes. If the rebuild fails before the swap, search continues using the previous index. With a table chosen, only that table is rebuilt in place. Only one full rebuild runs at a time. Your application may also schedule rebuilds. Does nothing while indexing is paused. Needs the Quick Search admin permission.
+- key: process:quickSearchFullReindex;field:searchTableName
+  summary: Pick one table to rebuild just it; leave blank to rebuild every table.
+- key: process:quickSearchReconcileIndex;slot:help
+  summary: Make the search index match the database exactly, re-indexing every record and removing leftover documents.
+  content: |
+    Leave **Table** blank to reconcile every enabled table and also delete documents of tables no longer configured. Documents are only deleted if every record indexed successfully. Use it as the safety net for tables without a timestamp field; your application may schedule it. Does nothing while indexing is paused. Needs the Quick Search admin permission.
+- key: process:quickSearchReconcileIndex;field:searchTableName
+  summary: Pick one table to reconcile just it; leave blank to reconcile every table.
+- key: widget:quickSearchIndexFields;slot:label
+  summary: The fields searched for this table, with their weight, role and analyzer; add or change them here.
+  content: |
+    After you change fields, the index shows Needs Reindex until you run **{{label:quickSearchApplyIndexChanges}}**.
+- key: widget:quickSearchIndexRuns;slot:label
+  summary: The 10 latest index runs for this table, with status, times and counts.
+- key: widget:quickSearchDiagnostics;slot:label
+  summary: 'Search health at a glance: this server''s search state, recent search failures and the latest validation run.'
+- key: possibleValueSource:quick_search_searchable_table;slot:help
+  summary: The tables that are currently set up for Quick Search, listed by their labels.
+- key: possibleValueSource:quick_search_eligible_table;slot:help
+  summary: 'Tables you could add to Quick Search: readable by you, not indexed yet, and not internal or secret.'
+- key: possibleValueSource:quick_search_indexable_field;slot:help
+  summary: The fields of the chosen table that can be searched; sensitive and hidden fields are not offered.
+- key: possibleValueSource:quick_search_timestamp_field;slot:help
+  summary: The date-time fields of the chosen table, which can be used to spot changed records.
+- key: possibleValueSource:quick_search_field_role;slot:help
+  summary: What a searched field is used for.
+  content: |
+    - **T1 display**: shown with each search result, labeled with the field's name.
+    - **Title**: the result's title, used in place of the record label.
+    - **Subtitle**: the line under the title.
+    - **Body**: ordinary searchable text, also used for the matched snippet.
+    - **Keyword**: matched as whole words, and kept out of the snippet.
+    - **Exact**: matched only when the search is the whole value (for codes and ids); kept out of the snippet.
+- key: possibleValueSource:quick_search_field_analyzer;slot:help
+  summary: How a field's text is broken up for matching.
+  content: |
+    - **Prefix (as you type)**: matches the start of words, so partial typing finds results. The usual default.
+    - **Whole words**: matches complete words only. The default for keyword fields.
+    - **English (stemmed)**: also matches related word forms, such as plurals.
+    - **Exact value**: matches the whole value, ignoring case and extra spaces. The default for exact fields.
+- key: possibleValueSource:quick_search_documents_choice;slot:help
+  summary: What to do with a table's search documents when you stop indexing it.
+  content: |
+    - **Delete the documents from the search index**: the table disappears from results.
+    - **Keep the documents**: they stay in the index but are no longer updated.
+- key: possibleValueSource:quick_search_run_type;slot:help
+  summary: Which kind of index run this was.
+  content: |
+    - **Update (basepull)**: the regular catch-up that picks up recent changes.
+    - **Reconcile**: a full comparison that fixes any difference with the database.
+    - **Full reindex / build**: a complete rebuild of the index.
+- key: possibleValueSource:quick_search_run_status;slot:help
+  summary: How an index run ended.
+  content: |
+    - **Running**: still in progress.
+    - **Completed**: finished.
+    - **Failed**: stopped by an error.
+    - **Cancelled**: stopped on request.
+- key: possibleValueSource:quick_search_failed_event_status;slot:help
+  summary: Where a change that could not be indexed stands.
+  content: |
+    - **Pending**: will be retried by the next update.
+    - **Exhausted**: retried the maximum number of times and given up; look at the error.
+    - **Awaiting full reindex**: recorded during a rebuild and applied when it finishes.
+    - **Deferred**: held while indexing is paused, and queued again when you resume.
+- key: possibleValueSource:quick_search_failed_event_action;slot:help
+  summary: What was being done to the search document when it failed.
+  content: |
+    - **Index**: adding or updating the document.
+    - **Delete**: removing the document.
+- key: possibleValueSource:quick_search_config_source;slot:help
+  summary: Where an index setup came from.
+  content: |
+    - **Code default**: created automatically from the application's built-in list.
+    - **Added by owner**: added by an admin with **{{label:quickSearchAddIndex}}**.
+
+- key: possibleValueSource:quick_search_index;slot:help
+  summary: Choose the search index whose fields or runs you want to view.

src/main/resources/openapi/qqq-v1/quick-search.yaml

@@ -1,6 +1,46 @@
 components:
   schemas:
+    BasicErrorResponseV1:
+      description: "The error body of every UI API operation (core and extensions)."
+      properties:
+        error:
+          description: "The error message (the same as message; kept for clients of\
+            \ UI API 1.0 to 1.5)."
+          type: "string"
+        code:
+          description: "A stable, machine-readable code. Well-known values: BAD_REQUEST\
+            \ (400), UNAUTHENTICATED (401),\nPERMISSION_DENIED (403), NOT_FOUND (404),\
+            \ CONFLICT (409), RATE_LIMITED (429), INTERNAL_ERROR (500),\nERROR (other).\
+            \ Extensions may add codes; treat an unknown code by the HTTP status.\
+            \ Since 1.6.0."
+          type: "string"
+        message:
+          description: "A message to show the user. Since 1.6.0."
+          type: "string"
+        fieldErrors:
+          description: "Errors about particular request fields, when known. Since\
+            \ 1.6.0."
+          items:
+            properties:
+              field:
+                description: "The field (or parameter) name."
+                type: "string"
+              message:
+                description: "The error about that field."
+                type: "string"
+              code:
+                description: "An optional machine-readable code, e.g. REQUIRED, TOO_LONG,\
+                  \ PATTERN."
+                type: "string"
+            type: "object"
+          type: "array"
+        traceId:
+          description: "Identifies this request in the server logs (also the X-QQQ-Trace-Id\
+            \ response header); quote it when reporting a problem. Since 1.6.0."
+          type: "string"
+      type: "object"
     QuickSearchErrorV1:
+      description: "Deprecated since 1.1.0: errors use BasicErrorResponseV1."
       properties:
         error:
           description: "Framework error message."
@@ -114,6 +154,11 @@ components:
             \ backlog rows."
           nullable: true
           type: "integer"
+        replicaSettingsFailed:
+          description: "Sizing or applying the index replicas failed at startup; null\
+            \ when not attempted."
+          nullable: true
+          type: "boolean"
         state:
           description: "Overall availability; incomplete diagnostics are DEGRADED."
           enum:
@@ -127,6 +172,16 @@ components:
           description: "More than 100 index rows exist."
           nullable: false
           type: "boolean"
+        targetReplicas:
+          description: "number_of_replicas Quick Search applies to its index; null\
+            \ when unknown or not managed (Serverless)."
+          nullable: true
+          type: "integer"
+        unassignedReplicaShards:
+          description: "Unassigned shards when every primary is active (all are replicas);\
+            \ null when unknown or a primary is not active."
+          nullable: true
+          type: "integer"
         unassignedShards:
           description: "Cluster health counter; null when unavailable."
           nullable: true
@@ -194,15 +249,48 @@ components:
       type: "object"
   securitySchemes:
     sessionUuidCookie:
+      description: "The session cookie set by POST /qqq/v1/manageSession (web clients)."
       in: "cookie"
       name: "sessionUUID"
       type: "apiKey"
+    bearerAuth:
+      bearerFormat: "JWT or session UUID"
+      description: "`Authorization: Bearer <token>`, where the token is either\n-\
+        \ an OAuth2 access token (a signed JWT) issued by the app's identity provider\
+        \ to the app's client,\n  or to its native client (`values.nativeClientId`\
+        \ in GET /qqq/v1/metaData/authentication), for\n  example from the device-authorization\
+        \ grant or auth-code + PKCE. It is verified against the\n  provider's keys\
+        \ (issuer, audience, expiry, signature) and gives the same user and permissions\n\
+        \  as signing in on the web. When it expires (401), refresh it at the provider\
+        \ and retry; or\n- a session UUID returned by POST /qqq/v1/manageSession,\
+        \ which the server keeps refreshed."
+      scheme: "bearer"
+      type: "http"
+    oauth2:
+      description: "The app's OAuth2 / OpenID Connect provider. The real endpoints\
+        \ come from the provider's discovery\ndocument: `{values.baseUrl}.well-known/openid-configuration`,\
+        \ with values from\nGET /qqq/v1/metaData/authentication. Native clients use\
+        \ `values.nativeClientId` (a public client)\nwith auth-code + PKCE (loopback\
+        \ or custom-scheme redirect URI registered at the provider) or the\ndevice-authorization\
+        \ grant (`device_authorization_endpoint`, RFC 8628), then send the access\
+        \ token\nas `Authorization: Bearer`."
+      flows:
+        authorizationCode:
+          authorizationUrl: "/authorize"
+          scopes:
+            email: "Email address"
+            offline_access: "A refresh token, to renew the access token without signing\
+              \ in again"
+            openid: "OpenID Connect sign-in"
+            profile: "Name, username and groups (permissions come from the groups)"
+          tokenUrl: "/token"
+      type: "oauth2"
 info:
   contact:
     email: "contact@kingsrook.com"
   description: "Quick Search administration and suggestions"
   title: "QQQ Middleware API extension: quick-search"
-  version: "1.0.0"
+  version: "1.1.0"
 openapi: "3.0.3"
 paths:
   /qqq/v1/qbit/quick-search/status:
@@ -217,19 +305,19 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/QuickSearchErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Diagnostic rate limit exceeded"
         401:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/QuickSearchErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Sign in required"
         403:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/QuickSearchErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Quick Search admin permission required"
         200:
           content:
@@ -237,10 +325,37 @@ paths:
               schema:
                 $ref: "#/components/schemas/QuickSearchStatusV1"
           description: "Successful Response"
+        404:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not found, or not visible to this user (code NOT_FOUND)."
+        409:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Conflict with the current state, e.g. an edit conflict or\
+            \ an invalid transition (code CONFLICT)."
+        429:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Too many requests (code RATE_LIMITED); retry later."
       security:
       - sessionUuidCookie:
         - "N/A"
+      - bearerAuth:
+        - "N/A"
+      - oauth2:
+        - "openid"
+        - "profile"
+        - "email"
       summary: "Get Quick Search status"
+      tags:
+      - "quick-search"
   /qqq/v1/qbit/quick-search/suggest:
     get:
       description: "Requires a signed-in session, not Quick Search admin permission.\
@@ -290,20 +405,20 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/QuickSearchErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Missing q, normalized term longer than 100 characters, or\
             \ invalid limits"
         401:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/QuickSearchErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Sign in required"
         403:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/QuickSearchErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Session permission denied"
         200:
           content:
@@ -311,7 +426,40 @@ paths:
               schema:
                 $ref: "#/components/schemas/QuickSearchSuggestV1"
           description: "Successful Response"
+        404:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not found, or not visible to this user (code NOT_FOUND)."
+        409:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Conflict with the current state, e.g. an edit conflict or\
+            \ an invalid transition (code CONFLICT)."
+        429:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Too many requests (code RATE_LIMITED); retry later."
       security:
       - sessionUuidCookie:
         - "N/A"
+      - bearerAuth:
+        - "N/A"
+      - oauth2:
+        - "openid"
+        - "profile"
+        - "email"
       summary: "Get Quick Search suggestions"
+      tags:
+      - "quick-search"
+servers:
+- description: "The QQQ server that serves this document (paths include /qqq/v1)."
+  url: "/"
+tags:
+- description: "Operations of the quick-search extension."
+  name: "quick-search"

src/test/java/com/kingsrook/qbits/quicksearch/OpenSearchIntegrationTest.java

@@ -48,6 +48,7 @@ import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
 import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
 import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
 import com.kingsrook.qbits.quicksearch.processes.ReconcileIndexStep;
@@ -218,6 +219,14 @@ class OpenSearchIntegrationTest
       assertThat(physicalIndexes.iterator().next()).startsWith(INDEX_NAME + "-v" + QuickSearchOpenSearchClient.MAPPING_VERSION + "-");
       assertThat(client().isMappingOutdated()).isFalse();
       assertThat(client().isHealthy()).isTrue();
+
+      ////////////////////////////////////////////////////////////////////
+      // zero config on a single node: no replica, so the index is green //
+      ////////////////////////////////////////////////////////////////////
+      String physicalIndex = physicalIndexes.iterator().next();
+      QuickSearchDiagnosticsOperations operations = new QuickSearchDiagnosticsOperations(client());
+      assertThat(operations.settings().body().path(physicalIndex).path("settings").path("index").path("number_of_replicas").asText()).isEqualTo("0");
+      assertThat(operations.clusterHealth().body().path("status").asText()).isEqualTo("green");
    }
 
 
@@ -520,6 +529,27 @@ class OpenSearchIntegrationTest
 
 
 
+   /*******************************************************************************
+    ** Test 12: an index left with a replica it cannot place on one node (yellow)
+    ** is brought back to 0 replicas (green) by the startup reconcile, once.
+    *******************************************************************************/
+   @Test
+   @Order(12)
+   void testReconcileReplicas_singleNodeDropsUnassignableReplica() throws Exception
+   {
+      String                           physicalIndex = client().getPhysicalIndexes().iterator().next();
+      QuickSearchDiagnosticsOperations operations    = new QuickSearchDiagnosticsOperations(client());
+      assertThat(operations.raw("PUT", "/" + physicalIndex + "/_settings", "{\"index\":{\"number_of_replicas\":1}}").isOk()).isTrue();
+      assertThat(operations.clusterHealth().body().path("status").asText()).isEqualTo("yellow");
+
+      assertThat(client().reconcileReplicas()).isEqualTo(new QuickSearchOpenSearchClient.ReplicaStatus(0, 1, 1, false));
+      assertThat(operations.settings().body().path(physicalIndex).path("settings").path("index").path("number_of_replicas").asText()).isEqualTo("0");
+      assertThat(operations.clusterHealth().body().path("status").asText()).isEqualTo("green");
+      assertThat(client().reconcileReplicas()).isEqualTo(new QuickSearchOpenSearchClient.ReplicaStatus(0, 1, 0, false));
+   }
+
+
+
    /*******************************************************************************
     ** Rebuild one table in place and refresh so its documents are searchable.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchApiContractTest.java

@@ -53,11 +53,14 @@ class QuickSearchApiContractTest
       assertThat(UiApiContract.check(generated, Files.readString(path), null, 1,
          "mvnw.cmd -Dtest=QuickSearchApiContractTest -Dquicksearch.openapi.update=true test")).isEmpty();
       var model = version.generateExtensionOpenAPIModel("qqq", extension);
-      assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> name.startsWith("QuickSearch"));
+      assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> name.startsWith("QuickSearch") || name.equals("BasicErrorResponseV1"));
       assertThat(model.getPaths().keySet()).containsExactlyInAnyOrder("/qqq/v1/qbit/quick-search/status", "/qqq/v1/qbit/quick-search/suggest");
       assertThat(extension.getSpecs()).hasSize(2);
       assertThat(generated).contains("getQuickSearchSuggestions", "QuickSearchSuggestV1", "QuickSearchSuggestGroupV1", "QuickSearchSuggestItemV1");
       var schemas = model.getComponents().getSchemas();
+      assertThat(schemas.get("BasicErrorResponseV1").getProperties()).containsOnlyKeys("error", "code", "message", "fieldErrors", "traceId");
+      assertThat(schemas.get("QuickSearchErrorV1").getDescription()).isEqualTo("Deprecated since 1.1.0: errors use BasicErrorResponseV1.");
+      assertThat(generated).contains("getQuickSearchStatus").doesNotContain("#/components/schemas/QuickSearchErrorV1");
       assertThat(schemas.get("QuickSearchSuggestV1").getProperties()).containsOnlyKeys("query", "groups", "truncated");
       assertThat(schemas.get("QuickSearchSuggestGroupV1").getProperties()).containsOnlyKeys("tableName", "tableLabel", "hasMore", "items");
       assertThat(schemas.get("QuickSearchSuggestItemV1").getProperties()).containsOnlyKeys("recordId", "label", "subtitle");

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchClusterCachesTest.java

@@ -0,0 +1,193 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
+import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The Quick Search caches are correct on several nodes: every static cache is
+ ** registered with ClusterCaches, and a config change on one node clears the
+ ** config cache on the others.
+ *******************************************************************************/
+class QuickSearchClusterCachesTest extends BaseQuickSearchTest
+{
+   /** the "wire": events every node's transport sees */
+   static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** A transport over the static WIRE, so a "remote" publish reaches us.
+    *******************************************************************************/
+   public static class WireBus implements ClusterBusInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void publish(ClusterEvent event)
+      {
+         WIRE.forEach(listener -> listener.accept(event));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
+      {
+         WIRE.add(listener);
+         return (() -> WIRE.remove(listener));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      ClusterBus.close(QContext.getQInstance());
+      WIRE.clear();
+      QNodeRoles.set(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Connect the current instance to the wire as node-a.
+    *******************************************************************************/
+   private static void connectAsNodeA()
+   {
+      QInstance instance = QContext.getQInstance();
+      instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class)));
+      QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
+      ClusterBus.connect(instance);
+   }
+
+
+
+   /*******************************************************************************
+    ** A runtime with one code-declared table.
+    *******************************************************************************/
+   private static QuickSearchRuntime newRuntime()
+   {
+      ///////////////////////////////////////////////////////////////////////
+      // without the field table the config comes from code, so the enabled //
+      // flag is read from the index row and cached                         //
+      ///////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTables().remove(QuickSearchIndexField.TABLE_NAME);
+      QContext.getQInstance().getTable(QuickSearchIndex.TABLE_NAME).getAssociations().clear();
+
+      QuickSearchRuntime runtime = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("localhost").withOpensearchPort(9200).withOpensearchIndexName("t"));
+      runtime.setDiscoveredTables(List.of(new QuickSearchableTableConfig().withTableName(TEST_ENTITY_TABLE).withPrimaryKeyField("id").withSearchableFields(List.of("name"))));
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Every static Memoization in the QBit is registered (shared or node-local).
+    *******************************************************************************/
+   @Test
+   void testNoUnregisteredStaticCaches() throws Exception
+   {
+      assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.quicksearch")).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Another node's config invalidation clears this node's cached config.
+    *******************************************************************************/
+   @Test
+   void testRemoteInvalidationClearsConfigCache()
+   {
+      connectAsNodeA();
+      QuickSearchRuntime runtime = newRuntime();
+      try
+      {
+         runtime.isTableEnabled(TEST_ENTITY_TABLE);
+         assertThat(runtime.getCachedEnabledFlagCount()).isEqualTo(1);
+
+         ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, QuickSearchRuntime.CONFIG_CACHE_NAME, null);
+         remote.setOriginNodeId("node-b");
+         WIRE.forEach(listener -> listener.accept(remote));
+
+         assertThat(runtime.getCachedEnabledFlagCount()).isZero();
+      }
+      finally
+      {
+         runtime.close();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A config change here clears the cache here and tells the other nodes.
+    *******************************************************************************/
+   @Test
+   void testInvalidateConfigPublishesToOtherNodes()
+   {
+      connectAsNodeA();
+      List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
+      WIRE.add(sent::add);
+
+      QuickSearchRuntime runtime = newRuntime();
+      try
+      {
+         runtime.isTableEnabled(TEST_ENTITY_TABLE);
+         runtime.invalidateConfig();
+
+         assertThat(runtime.getCachedEnabledFlagCount()).isZero();
+         assertThat(sent).anySatisfy(event ->
+         {
+            assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
+            assertThat(event.getName()).isEqualTo(QuickSearchRuntime.CONFIG_CACHE_NAME);
+            assertThat(event.getOriginNodeId()).isEqualTo("node-a");
+         });
+      }
+      finally
+      {
+         runtime.close();
+      }
+   }
+}

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfigValidationTest.java

@@ -348,7 +348,11 @@ class QuickSearchQBitConfigValidationTest
          .withBasepullOverlapSeconds(-5)
          .withRunHistoryRetentionDays(0)
          .withFullReindexStaleMinutes(0)
-         .withMaxFieldLength(0));
+         .withMaxFieldLength(0)
+         .withIndexReplicas(-1)
+         .withMaxAutoReplicas(-1));
+      assertThat(errors).anyMatch(e -> e.startsWith("indexReplicas"));
+      assertThat(errors).anyMatch(e -> e.startsWith("maxAutoReplicas"));
       assertThat(errors).anyMatch(e -> e.startsWith("connectTimeoutMillis"));
       assertThat(errors).anyMatch(e -> e.startsWith("responseTimeoutMillis"));
       assertThat(errors).anyMatch(e -> e.startsWith("maxConnections"));

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntimeTest.java

@@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
@@ -223,4 +224,31 @@ class QuickSearchRuntimeTest extends BaseQuickSearchTest
       failFast.close();
    }
 
+
+
+   /*******************************************************************************
+    ** Priming at startup without a QContext builds the system session with the
+    ** start instance (so it has the all-access keys), not from the empty context.
+    *******************************************************************************/
+   @Test
+   void primeConfigWithoutAContextGivesTheSystemSessionItsInstance()
+   {
+      QInstance          qInstance        = QContext.getQInstance();
+      QuickSearchRuntime runtime          = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("127.0.0.1").withOpensearchPort(1).withOpensearchIndexName("t"));
+      QCollectingLogger  collectingLogger = QLogger.activateCollectingLoggerForClass(QSystemUserSession.class);
+      try
+      {
+         runtime.setStartInstance(qInstance);
+         QContext.clear();
+         runtime.primeConfig();
+      }
+      finally
+      {
+         QLogger.deactivateCollectingLoggerForClass(QSystemUserSession.class);
+         QContext.init(qInstance, new QSystemUserSession(qInstance));
+         runtime.close();
+      }
+      assertThat(collectingLogger.getCollectedMessages()).noneSatisfy(m -> assertThat(m.getMessage()).contains("QInstance was not supplied"));
+   }
+
 }

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchSeededDefaultsTest.java

@@ -0,0 +1,207 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.util.List;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** This qbit ships no DDL and no changelog: QuickSearchConfigPrimer seeds its
+ ** config tables at runtime. This proves, on a fresh (memory) backend produced
+ ** by the real producer, that priming populates quick_search_index,
+ ** quick_search_index_field and the quick_search_settings row (id 1), leaves
+ ** quick_search_config_tombstone empty, is idempotent, and never overwrites an
+ ** admin's edit.
+ *******************************************************************************/
+class QuickSearchSeededDefaultsTest
+{
+   /*******************************************************************************
+    ** A searchable entity.
+    *******************************************************************************/
+   @QuickSearchable(tableName = "person", basepullIntervalMinutes = 10)
+   static class PersonEntity
+   {
+      @QuickSearchField(weight = 3)
+      private String name;
+
+      @QuickSearchField
+      private String email;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      QContext.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** A fresh install: priming fills the config tables, twice changes nothing,
+    ** and an edited row survives.
+    *******************************************************************************/
+   @Test
+   void primerPopulatesConfigTablesOnFreshBackend_idempotent_andKeepsEdits() throws QException
+   {
+      QuickSearchQBitConfig config = produce(List.of(PersonEntity.class));
+      QuickSearchRuntime    runtime = config.getRuntime();
+
+      assertThat(rows(QuickSearchIndex.TABLE_NAME)).isEmpty();
+      assertThat(rows(QuickSearchSettings.TABLE_NAME)).isEmpty();
+
+      QuickSearchConfigPrimer.PrimingReport first = new QuickSearchConfigPrimer(runtime).prime();
+
+      assertThat(first.newTables()).hasSize(1);
+      assertThat(rows(QuickSearchIndex.TABLE_NAME)).singleElement().satisfies(row ->
+      {
+         assertThat(row.getValueString("tableName")).isEqualTo("person");
+         assertThat(row.getValueString("source")).isEqualTo(QuickSearchIndex.SOURCE_CODE);
+         assertThat(row.getValueInteger("basepullIntervalMinutes")).isEqualTo(10);
+      });
+      assertThat(rows(QuickSearchIndexField.TABLE_NAME)).extracting(r -> r.getValueString("fieldName")).containsExactlyInAnyOrder("name", "email");
+      assertThat(rows(QuickSearchSettings.TABLE_NAME)).singleElement().satisfies(row ->
+      {
+         assertThat(row.getValueInteger("id")).isEqualTo(QuickSearchSettings.SETTINGS_ID);
+         assertThat(row.getValueBoolean("indexingPaused")).isFalse();
+         assertThat(row.getValueString("lastPrimingSummary")).isNotBlank();
+      });
+      assertThat(rows(QuickSearchConfigTombstone.TABLE_NAME)).as("tombstones legitimately start empty").isEmpty();
+
+      /////////////////////////////////////////////
+      // an admin edits the interval (user write) //
+      /////////////////////////////////////////////
+      QRecord indexRow = rows(QuickSearchIndex.TABLE_NAME).get(0);
+      new UpdateAction().execute(new UpdateInput(QuickSearchIndex.TABLE_NAME).withRecord(new QRecord().withValue("id", indexRow.getValue("id")).withValue("basepullIntervalMinutes", 45)));
+
+      QuickSearchConfigPrimer.PrimingReport second = new QuickSearchConfigPrimer(runtime).prime();
+
+      assertThat(second.newTables()).isEmpty();
+      assertThat(second.newFields()).isZero();
+      assertThat(rows(QuickSearchIndex.TABLE_NAME)).hasSize(1);
+      assertThat(rows(QuickSearchIndexField.TABLE_NAME)).hasSize(2);
+      assertThat(rows(QuickSearchSettings.TABLE_NAME)).hasSize(1);
+      assertThat(rows(QuickSearchIndex.TABLE_NAME).get(0).getValueInteger("basepullIntervalMinutes")).as("the admin's edit survives priming").isEqualTo(45);
+   }
+
+
+
+   /*******************************************************************************
+    ** Even with nothing searchable declared in code, priming creates the settings
+    ** row (id 1), so the pause switch and priming summary always exist.
+    *******************************************************************************/
+   @Test
+   void primerCreatesSettingsRowEvenWithNoCodeTables() throws QException
+   {
+      QuickSearchQBitConfig config = produce(List.of(PersonEntity.class));
+      config.getRuntime().setDiscoveredTables(List.of());
+
+      new QuickSearchConfigPrimer(config.getRuntime()).prime();
+
+      assertThat(rows(QuickSearchIndex.TABLE_NAME)).isEmpty();
+      assertThat(rows(QuickSearchSettings.TABLE_NAME)).singleElement().satisfies(row ->
+      {
+         assertThat(row.getValueInteger("id")).isEqualTo(QuickSearchSettings.SETTINGS_ID);
+         assertThat(row.getValueBoolean("indexingPaused")).isFalse();
+         assertThat(row.getValueString("lastPrimingSummary")).isNotBlank();
+      });
+
+      new QuickSearchConfigPrimer(config.getRuntime()).prime();
+      assertThat(rows(QuickSearchSettings.TABLE_NAME)).hasSize(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Produce the qbit on a fresh memory-backed instance.
+    *******************************************************************************/
+   private QuickSearchQBitConfig produce(List<Class<?>> entityClasses) throws QException
+   {
+      QInstance qInstance = new QInstance();
+      qInstance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
+      qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
+      qInstance.addTable(new QTableMetaData().withName("person").withBackendName("memory").withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+         .withField(new QFieldMetaData("email", QFieldType.STRING))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
+      QContext.init(qInstance, new QSession());
+
+      QuickSearchQBitConfig config = new QuickSearchQBitConfig()
+         .withStartupMode(QuickSearchStartupMode.DEGRADED)
+         .withBackendName("memory")
+         .withOpensearchHost("localhost")
+         .withOpensearchPort(9200)
+         .withOpensearchIndexName("seeded-defaults")
+         .withSearchableEntityClasses(entityClasses);
+      new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
+      return (config);
+   }
+
+
+
+   /*******************************************************************************
+    ** Every row of a table.
+    *******************************************************************************/
+   private List<QRecord> rows(String tableName) throws QException
+   {
+      return (QuickSearchConfigPrimer.queryAll(tableName));
+   }
+
+}

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchStatusTest.java

@@ -80,6 +80,7 @@ class QuickSearchStatusTest
    private QuickSearchQBitConfig config;
    private QInstance instance;
    private volatile String health = "green";
+   private volatile String extraHealth = ",\"number_of_nodes\":1,\"active_shards\":1,\"unassigned_shards\":0";
    private volatile int requests;
    private CountDownLatch blocked;
 
@@ -106,7 +107,7 @@ class QuickSearchStatusTest
                Thread.currentThread().interrupt();
             }
          }
-         byte[] body = ("{\"status\":\"" + health + "\",\"number_of_nodes\":1,\"active_shards\":1,\"unassigned_shards\":0}").getBytes(StandardCharsets.UTF_8);
+         byte[] body = ("{\"status\":\"" + health + "\"" + extraHealth + "}").getBytes(StandardCharsets.UTF_8);
          exchange.getResponseHeaders().add("Content-Type", "application/json");
          exchange.sendResponseHeaders(200, body.length);
          exchange.getResponseBody().write(body);
@@ -214,6 +215,36 @@ class QuickSearchStatusTest
 
 
 
+   /*******************************************************************************
+    ** Yellow from replicas a single node cannot hold is OK; an inactive primary
+    ** or a second node keeps it DEGRADED.
+    *******************************************************************************/
+   @Test
+   void singleNodeReplicaYellow() throws Exception
+   {
+      health = "yellow";
+      String index = ",\"indices\":{\"status-test-v2-1\":{\"number_of_shards\":1,\"active_primary_shards\":%d}}";
+      extraHealth = ",\"number_of_nodes\":1,\"active_shards\":1,\"unassigned_shards\":1,\"initializing_shards\":0" + index.formatted(1);
+      QuickSearchStatus single = StatusDiagnostics.read(config.getRuntime());
+      assertThat(single.state()).isEqualTo(QuickSearchStatus.State.OK);
+      assertThat(single.unassignedReplicaShards()).isEqualTo(1);
+
+      extraHealth = ",\"number_of_nodes\":1,\"active_shards\":0,\"unassigned_shards\":2,\"initializing_shards\":0" + index.formatted(0);
+      QuickSearchStatus primaryMissing = StatusDiagnostics.read(config.getRuntime());
+      assertThat(primaryMissing.state()).isEqualTo(QuickSearchStatus.State.DEGRADED);
+      assertThat(primaryMissing.unassignedReplicaShards()).isNull();
+
+      extraHealth = ",\"number_of_nodes\":1,\"active_shards\":0,\"unassigned_shards\":1,\"initializing_shards\":1" + index.formatted(1);
+      assertThat(StatusDiagnostics.read(config.getRuntime()).unassignedReplicaShards()).isNull();
+
+      extraHealth = ",\"number_of_nodes\":2,\"active_shards\":1,\"unassigned_shards\":1,\"initializing_shards\":0" + index.formatted(1);
+      QuickSearchStatus multi = StatusDiagnostics.read(config.getRuntime());
+      assertThat(multi.state()).isEqualTo(QuickSearchStatus.State.DEGRADED);
+      assertThat(multi.unassignedReplicaShards()).isEqualTo(1);
+   }
+
+
+
    /*******************************************************************************
     ** A blocked remote call has bounded caller latency and cannot grow a queue.
     *******************************************************************************/
@@ -362,6 +393,14 @@ class QuickSearchStatusTest
          assertThat(anonymous.statusCode()).as(anonymous.body()).isEqualTo(403);
          var denied = client.send(HttpRequest.newBuilder(URI.create(url)).header("Cookie", "sessionUUID=reader-" + suffix).GET().build(), HttpResponse.BodyHandlers.ofString());
          assertThat(denied.statusCode()).as(denied.body()).isEqualTo(403);
+         for(var forbidden : List.of(anonymous, denied))
+         {
+            var error = new ObjectMapper().readTree(forbidden.body());
+            assertThat(error.path("error").asText()).isNotBlank();
+            assertThat(error.path("message")).isEqualTo(error.path("error"));
+            assertThat(error.path("code").asText()).isEqualTo("PERMISSION_DENIED");
+            assertThat(error.path("traceId").asText()).isNotBlank();
+         }
          String suggestUrl = "http://127.0.0.1:" + port + "/qqq/v1/qbit/quick-search/suggest";
          for(String query : List.of("?q=x", "?q=%20x%20", "?q=wi&tables=unknown", "?q=wi&tables=", "?q=wi"))
          {
@@ -378,7 +417,11 @@ class QuickSearchStatusTest
             var invalid = client.send(HttpRequest.newBuilder(URI.create(suggestUrl + query))
                .header("Cookie", "sessionUUID=reader-" + suffix).GET().build(), HttpResponse.BodyHandlers.ofString());
             assertThat(invalid.statusCode()).as(invalid.body()).isEqualTo(400);
-            assertThat(new ObjectMapper().readTree(invalid.body()).path("error").isTextual()).isTrue();
+            var error = new ObjectMapper().readTree(invalid.body());
+            assertThat(error.path("error").asText()).isNotBlank();
+            assertThat(error.path("message")).isEqualTo(error.path("error"));
+            assertThat(error.path("code").asText()).isEqualTo("BAD_REQUEST");
+            assertThat(error.path("traceId").asText()).isNotBlank();
          }
          var openapi = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/qqq/v1/openapi.yaml")).GET().build(), HttpResponse.BodyHandlers.ofString());
          assertThat(openapi.body()).contains("/qqq/v1/qbit/quick-search/status", "QuickSearchStatusV1",

src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchTablesGuardrailTest.java

@@ -0,0 +1,305 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
+import com.kingsrook.qqq.backend.core.actions.help.HelpTargetCatalog;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Guardrail (owner requirement 2026-10-14: "all tables should be populated and
+ ** the qbit should be fully using them"). For every quick_search_* table:
+ ** - runtime code (not just its model or metadata) reads or writes it, so no
+ **   table exists only on paper;
+ ** - it is either populated by the startup primer (QuickSearchConfigPrimer; this
+ **   qbit ships no DDL or changelog, so the primer is its seed mechanism) or
+ **   listed in STARTS_EMPTY with why it starts empty;
+ ** - it has bundle help that says what it is for and how its rows get created.
+ ** A new table fails this test until it is wired, seeded or explained.
+ *******************************************************************************/
+class QuickSearchTablesGuardrailTest
+{
+   private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "quicksearch");
+
+   /*******************************************************************************
+    ** Tables the primer fills from code on startup (QuickSearchSeededDefaultsTest
+    ** proves it).
+    *******************************************************************************/
+   static final Map<String, String> PRIMED = Map.of(
+      "quick_search_index", "one row per table code declares searchable",
+      "quick_search_index_field", "one row per searchable field code declares",
+      "quick_search_settings", "the single settings row (id 1), with the priming summary");
+
+   /*******************************************************************************
+    ** Tables that legitimately start empty, and where their rows come from.
+    *******************************************************************************/
+   static final Map<String, String> STARTS_EMPTY = Map.of(
+      "quick_search_config_tombstone", "an admin removes a table or field that code still declares",
+      "quick_search_index_run", "runtime: every build, basepull and reconcile",
+      "quick_search_failed_event", "runtime: a real-time index update fails (empty is healthy)",
+      "quick_search_diagnostic_event", "runtime: a search fails or a diagnostic tool runs",
+      "quick_search_validation_run", "runtime: someone runs Validate Search Index");
+
+
+
+   /*******************************************************************************
+    ** A searchable entity for the produced instance.
+    *******************************************************************************/
+   @QuickSearchable(tableName = "person")
+   static class PersonEntity
+   {
+      @QuickSearchField(weight = 3)
+      private String name;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      QContext.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Every table is used by runtime code.
+    *******************************************************************************/
+   @Test
+   void everyTableIsUsedAtRuntime() throws Exception
+   {
+      Map<String, String> entityByTable = entityByTable();
+      List<String>        runtimeSource;
+      try(Stream<Path> files = Files.walk(MAIN))
+      {
+         runtimeSource = files.filter(p -> p.toString().endsWith(".java"))
+            .filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("metadata")))
+            .map(QuickSearchTablesGuardrailTest::read).toList();
+      }
+      for(String table : quickSearchTables())
+      {
+         String entity = entityByTable.get(table);
+         assertThat(entity).as("entity class for " + table).isNotNull();
+
+         //////////////////////////////////////////////////////////////////////////
+         // the entity, or the config's name accessor for its table (index and   //
+         // index run are reached by their prefixed names)                       //
+         //////////////////////////////////////////////////////////////////////////
+         Pattern use = Pattern.compile("\\b" + entity + "(\\.TABLE_NAME|\\.class|\\(|::)|\\bget" + entity + "TableName\\(");
+         assertThat(runtimeSource.stream().anyMatch(source -> use.matcher(source).find()))
+            .as(table + " (" + entity + ") must be read or written by runtime code, not only declared").isTrue();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Every table is primed or explained.
+    *******************************************************************************/
+   @Test
+   void everyTableIsPrimedOrExplained()
+   {
+      QInstance qInstance = producedInstance();
+      for(String table : quickSearchTables())
+      {
+         assertThat(PRIMED.containsKey(table) || STARTS_EMPTY.containsKey(table)).as(table + " must be primed or listed in STARTS_EMPTY").isTrue();
+         assertThat(PRIMED.containsKey(table) && STARTS_EMPTY.containsKey(table)).as(table + " cannot be both primed and listed as starting empty").isFalse();
+      }
+      assertThat(PRIMED.keySet()).allMatch(table -> qInstance.getTable(table) != null, "PRIMED names only real tables");
+      assertThat(STARTS_EMPTY.keySet()).allMatch(table -> qInstance.getTable(table) != null, "STARTS_EMPTY names only real tables");
+   }
+
+
+
+   /*******************************************************************************
+    ** Bundle coverage includes every owned object and its non-trivial fields.
+    *******************************************************************************/
+   @Test
+   void helpBundleIsComplete()
+   {
+      QInstance qInstance = producedInstance();
+      // Record actions attached to host tables belong in the host bundle.
+      // This gate owns the fixed qbit objects, including its failed-event actions.
+      HelpBundleAssertions.forInstance(qInstance)
+         .bundles("help/qbit-quick-search.yaml")
+         .owning(target -> (target.name().startsWith("quick_search_") || target.name().startsWith("quickSearch"))
+            && (target.table() == null || target.table().startsWith("quick_search_")))
+         .assertComplete();
+
+      var entries = HelpBundles.of(qInstance).getBundles().get(0).getEntries();
+      assertThat(entries).extracting(entry -> entry.getKey()).doesNotHaveDuplicates();
+      for(var target : HelpTargetCatalog.collect(qInstance))
+      {
+         if(List.of("widget", "possibleValueSource").contains(target.kind())
+            && (target.name().startsWith("quick_search_") || target.name().startsWith("quickSearch")))
+         {
+            assertThat(entries).anySatisfy(entry -> assertThat(entry.getKey()).startsWith(target.kind() + ":" + target.name() + ";slot:"));
+         }
+      }
+      assertThat(entries).filteredOn(entry -> entry.getKey().endsWith(";slot:help")
+         && (entry.getKey().startsWith("table:") || entry.getKey().startsWith("app:") || entry.getKey().startsWith("process:")))
+         .allSatisfy(entry -> assertThat(entry.getContent()).as(entry.getKey()).isNotBlank());
+      assertThat(entries).filteredOn(entry -> entry.getKey().startsWith("table:") && entry.getKey().endsWith(";slot:help"))
+         .allSatisfy(entry -> assertThat(entry.getContent()).as(entry.getKey()).contains("How rows get created"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Registration remains metadata-only on hosts without the help subsystem.
+    *******************************************************************************/
+   @Test
+   void helpRegistersWithoutHelpContentTable()
+   {
+      QInstance qInstance = producedInstance();
+      assertThat(qInstance.getTable("helpContent")).isNull();
+      assertThat(HelpBundles.of(qInstance).getBundles()).singleElement().satisfies(bundle ->
+      {
+         assertThat(bundle.getId()).isEqualTo("qbit-quick-search");
+         assertThat(bundle.getVersion()).isEqualTo(QuickSearchQBitProducer.getVersion());
+      });
+      HelpBundles.syncOnStartup(qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<String> quickSearchTables()
+   {
+      List<String> tables = producedInstance().getTables().values().stream().map(QTableMetaData::getName).filter(n -> n.startsWith("quick_search_")).sorted().toList();
+      assertThat(tables).hasSize(8);
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QInstance producedInstance()
+   {
+      try
+      {
+         QInstance qInstance = new QInstance();
+         qInstance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
+         qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
+         qInstance.addTable(new QTableMetaData().withName("person").withBackendName("memory").withPrimaryKeyField("id")
+            .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+            .withField(new QFieldMetaData("name", QFieldType.STRING))
+            .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
+         QContext.init(qInstance, new QSession());
+         new QuickSearchQBitProducer().withConfig(new QuickSearchQBitConfig()
+            .withStartupMode(QuickSearchStartupMode.DEGRADED)
+            .withBackendName("memory")
+            .withOpensearchHost("localhost")
+            .withOpensearchPort(9200)
+            .withOpensearchIndexName("guardrail")
+            .withSearchableEntityClasses(List.of(PersonEntity.class))).produce(qInstance);
+         return (qInstance);
+      }
+      catch(QException e)
+      {
+         throw (new IllegalStateException(e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** model/X.java declares TABLE_NAME = "x".
+    *******************************************************************************/
+   private static Map<String, String> entityByTable() throws Exception
+   {
+      Map<String, String> rs = new HashMap<>();
+      Pattern             p  = Pattern.compile("TABLE_NAME\\s*=\\s*\"([a-z_]+)\"");
+      try(Stream<Path> files = Files.list(MAIN.resolve("model")))
+      {
+         for(Path file : files.toList())
+         {
+            Matcher m = p.matcher(read(file));
+            if(m.find())
+            {
+               rs.put(m.group(1), file.getFileName().toString().replace(".java", ""));
+            }
+         }
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String read(Path path)
+   {
+      try
+      {
+         return (Files.readString(path, StandardCharsets.UTF_8));
+      }
+      catch(Exception e)
+      {
+         throw (new IllegalStateException(e));
+      }
+   }
+
+}

src/test/java/com/kingsrook/qbits/quicksearch/RecordSearchProviderIntegrationTest.java

@@ -20,6 +20,7 @@ package com.kingsrook.qbits.quicksearch;
 import java.io.Serializable;
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
@@ -28,9 +29,14 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlight;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlightMatch;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchProviderState;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchSource;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchTableSummary;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -46,6 +52,8 @@ import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
 import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
 import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
 import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchTableHits;
 import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -59,6 +67,7 @@ import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
 
 
 /*******************************************************************************
@@ -230,6 +239,43 @@ class RecordSearchProviderIntegrationTest
 
 
 
+   /*******************************************************************************
+    ** Scores, totals and paging come from OpenSearch; the client's highlight
+    ** fragments are marked on the fieldValues.* fields and convert to plain
+    ** text and ranges; core highlights the matching non-label field.
+    *******************************************************************************/
+   @Test
+   @Order(2)
+   void testScoresTotalsPagingAndHighlights() throws Exception
+   {
+      RecordSearchInput  input  = new RecordSearchInput().withSearchTerm("blue widget").withTableNames(List.of(PRODUCT_TABLE)).withLimitPerTable(1);
+      RecordSearchOutput page1  = new RecordSearchAction().execute(input);
+      assertThat(page1.getProviderStatus().getStatus()).isEqualTo(RecordSearchProviderState.OK);
+      RecordSearchTableSummary products = page1.getTables().get(0);
+      assertThat(products.getSource()).isEqualTo(RecordSearchSource.PROVIDER);
+      assertThat(products.getTotalCount()).isEqualTo(2);
+      assertThat(products.getHasMore()).isTrue();
+      assertThat(page1.getResults()).hasSize(1);
+      assertThat(page1.getResults().get(0).getScore()).isPositive();
+
+      RecordSearchOutput page2 = new RecordSearchAction().execute(input.withOffset(1));
+      assertThat(page2.getTables().get(0).getHasMore()).isFalse();
+      assertThat(List.of(page1.getResults().get(0).getRecordId(), page2.getResults().get(0).getRecordId())).containsExactlyInAnyOrder(1, 2);
+
+      RecordSearchResult widget = coreSearch("blue", 5).getResults().stream().filter(r -> Integer.valueOf(2).equals(r.getRecordId())).findFirst().orElseThrow();
+      assertThat(widget.getHighlights()).extracting(RecordSearchHighlight::getFieldName).containsExactly("description");
+      assertThat(widget.getHighlights().get(0).getText()).isEqualTo("comes in blue and green");
+
+      Map<String, RecordSearchTableHits> hits = config.getRuntime().getClient().searchRecordsPerTable("blue", List.of(PRODUCT_TABLE), 5, 0, true, config.getRuntime().getDiscoveredTables(), 10_000);
+      RecordSearchHit hit = hits.get(PRODUCT_TABLE).hits().stream().filter(h -> "2".equals(h.recordId())).findFirst().orElseThrow();
+      List<RecordSearchHighlight> highlights = QuickSearchRecordSearchProvider.highlights(hit.highlights());
+      assertThat(highlights).extracting(RecordSearchHighlight::getFieldName).containsExactly("description");
+      assertThat(highlights.get(0).getText()).isEqualTo("comes in blue and green");
+      assertThat(highlights.get(0).getMatches()).extracting(RecordSearchHighlightMatch::getStart, RecordSearchHighlightMatch::getEnd).containsExactly(tuple(9, 13));
+   }
+
+
+
    @Test
    @Order(3)
    void testLimitPerTable() throws Exception

src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchRecordSearchProviderTest.java

@@ -28,9 +28,14 @@ import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlight;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlightMatch;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchProviderState;
 import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchSource;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchTableSummary;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
@@ -43,12 +48,15 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchTableHits;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.assertj.core.api.Assertions.tuple;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyList;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -218,7 +226,7 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
       Map<String, List<String>> ids = new LinkedHashMap<>();
       ids.put(OTHER_TABLE, List.of("7"));
       ids.put(TEST_ENTITY_TABLE, List.of("3", "1", "2"));
-      when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenReturn(ids);
+      when(mockClient.searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt())).thenReturn(hits(ids));
 
       RecordSearchOutput output = provider.search(new RecordSearchInput()
          .withSearchTerm("  widget   blue ")
@@ -231,13 +239,13 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
          tuple(TEST_ENTITY_TABLE, "2"),
          tuple(OTHER_TABLE, "7"));
 
-      verify(mockClient).searchRecordIdsPerTable(eq("widget blue"), eq(List.of(TEST_ENTITY_TABLE, OTHER_TABLE)), eq(5 * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), anyList(), eq(1234));
+      verify(mockClient).searchRecordsPerTable(eq("widget blue"), eq(List.of(TEST_ENTITY_TABLE, OTHER_TABLE)), eq(5 * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), eq(0), eq(true), anyList(), eq(1234));
 
       provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE)).withLimitPerTable(25));
-      verify(mockClient).searchRecordIdsPerTable(eq("widget"), eq(List.of(TEST_ENTITY_TABLE)), eq(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE), anyList(), eq(1234));
+      verify(mockClient).searchRecordsPerTable(eq("widget"), eq(List.of(TEST_ENTITY_TABLE)), eq(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE), eq(0), eq(true), anyList(), eq(1234));
 
       provider.search(new RecordSearchInput().withSearchTerm("gadget").withTableNames(List.of(TEST_ENTITY_TABLE)));
-      verify(mockClient).searchRecordIdsPerTable(eq("gadget"), eq(List.of(TEST_ENTITY_TABLE)), eq(RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), anyList(), eq(1234));
+      verify(mockClient).searchRecordsPerTable(eq("gadget"), eq(List.of(TEST_ENTITY_TABLE)), eq(RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), eq(0), eq(true), anyList(), eq(1234));
    }
 
 
@@ -248,7 +256,7 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
       assertThat(provider.search(new RecordSearchInput().withSearchTerm("x").withTableNames(List.of(TEST_ENTITY_TABLE))).getResults()).isEmpty();
       assertThat(provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(UNINDEXED_TABLE))).getResults()).isEmpty();
       assertThat(provider.search(new RecordSearchInput().withSearchTerm("widget")).getResults()).isEmpty();
-      verify(mockClient, never()).searchRecordIdsPerTable(any(), any(), anyInt(), any(), anyInt());
+      verify(mockClient, never()).searchRecordsPerTable(any(), any(), anyInt(), anyInt(), anyBoolean(), any(), anyInt());
    }
 
 
@@ -256,7 +264,7 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
    @Test
    void testSearch_clientFailure_throwsAndBacksOff() throws QException
    {
-      when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenThrow(new QException("boom"));
+      when(mockClient.searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt())).thenThrow(new QException("boom"));
 
       assertThatThrownBy(() -> provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE))))
          .isInstanceOf(QException.class).hasMessageContaining("boom");
@@ -268,7 +276,7 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
    @Test
    void testSearch_uncheckedClientFailure_wrappedAsQException() throws QException
    {
-      when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenThrow(new IllegalStateException("closed"));
+      when(mockClient.searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt())).thenThrow(new IllegalStateException("closed"));
 
       assertThatThrownBy(() -> provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE))))
          .isInstanceOf(QException.class).hasMessageContaining("closed");
@@ -276,6 +284,82 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
    }
 
 
+   /*******************************************************************************
+    ** Client hits for these ids per table (no scores, highlights or totals).
+    *******************************************************************************/
+   private static Map<String, RecordSearchTableHits> hits(Map<String, List<String>> idsByTable)
+   {
+      Map<String, RecordSearchTableHits> hitsByTable = new LinkedHashMap<>();
+      idsByTable.forEach((tableName, ids) -> hitsByTable.put(tableName, new RecordSearchTableHits(ids.stream().map(id -> new RecordSearchHit(id, null, null)).toList(), null)));
+      return (hitsByTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** A fragment with these words marked as matches, as the client returns it.
+    *******************************************************************************/
+   private static String marked(String word)
+   {
+      return (QuickSearchOpenSearchClient.HIGHLIGHT_PRE_TAG + word + QuickSearchOpenSearchClient.HIGHLIGHT_POST_TAG);
+   }
+
+
+
+   /*******************************************************************************
+    ** Scores, highlights (fieldValues.* only, as plain text and ranges) and
+    ** totals reach core; the offset reaches the client.
+    *******************************************************************************/
+   @Test
+   void testSearch_returnsScoresHighlightsAndTotals_andPassesOffset() throws QException
+   {
+      Map<String, RecordSearchTableHits> hitsByTable = new LinkedHashMap<>();
+      hitsByTable.put(TEST_ENTITY_TABLE, new RecordSearchTableHits(List.of(
+         new RecordSearchHit("3", 7.5, Map.of(
+            "fieldValues.name", List.of("big " + marked("Widget") + " for " + marked("blue") + marked("s")),
+            "searchableText", List.of(marked("Widget")))),
+         new RecordSearchHit("1", 2.25, Map.of("fieldValues.name", List.of("no match here")))), 42L));
+      when(mockClient.searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt())).thenReturn(hitsByTable);
+
+      RecordSearchOutput output = provider.search(new RecordSearchInput().withSearchTerm("widget blue").withTableNames(List.of(TEST_ENTITY_TABLE)).withLimitPerTable(5).withOffset(10));
+
+      assertThat(output.getResults()).extracting(RecordSearchResult::getRecordId, RecordSearchResult::getScore).containsExactly(tuple("3", 7.5), tuple("1", 2.25));
+      assertThat(output.getTotalCountByTable()).containsEntry(TEST_ENTITY_TABLE, 42);
+
+      List<RecordSearchHighlight> highlights = output.getResults().get(0).getHighlights();
+      assertThat(highlights).hasSize(1);
+      assertThat(highlights.get(0).getFieldName()).isEqualTo("name");
+      assertThat(highlights.get(0).getText()).isEqualTo("big Widget for blues");
+      assertThat(highlights.get(0).getMatches()).extracting(RecordSearchHighlightMatch::getStart, RecordSearchHighlightMatch::getEnd).containsExactly(tuple(4, 10), tuple(15, 20));
+      assertThat(output.getResults().get(1).getHighlights()).isNull();
+
+      verify(mockClient).searchRecordsPerTable(eq("widget blue"), eq(List.of(TEST_ENTITY_TABLE)), eq(5 * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), eq(10), eq(true), anyList(), eq(1234));
+   }
+
+
+
+   /*******************************************************************************
+    ** Fragment conversion: markers removed, ranges kept; no match, blank or
+    ** unbalanced markers give no highlight; non-fieldValues fields are skipped.
+    *******************************************************************************/
+   @Test
+   void testHighlightConversion()
+   {
+      RecordSearchHighlight highlight = QuickSearchRecordSearchProvider.highlight("name", marked("ab") + "c" + marked("d"));
+      assertThat(highlight.getText()).isEqualTo("abcd");
+      assertThat(highlight.getMatches()).extracting(RecordSearchHighlightMatch::getStart, RecordSearchHighlightMatch::getEnd).containsExactly(tuple(0, 2), tuple(3, 4));
+
+      assertThat(QuickSearchRecordSearchProvider.highlight("name", "plain")).isNull();
+      assertThat(QuickSearchRecordSearchProvider.highlight("name", null)).isNull();
+      assertThat(QuickSearchRecordSearchProvider.highlight("name", marked(" "))).isNull();
+      assertThat(QuickSearchRecordSearchProvider.highlight("name", QuickSearchOpenSearchClient.HIGHLIGHT_POST_TAG + "x" + QuickSearchOpenSearchClient.HIGHLIGHT_PRE_TAG)).isNull();
+
+      assertThat(QuickSearchRecordSearchProvider.highlights(Map.of("searchableText", List.of(marked("x")), "fieldValues.", List.of(marked("x")), "fieldValues.other", List.of())))
+         .isEmpty();
+      assertThat(QuickSearchRecordSearchProvider.highlights(null)).isEmpty();
+   }
+
+
 
    /*******************************************************************************
     ** A server that accepts connections and never answers (a hung OpenSearch):
@@ -323,13 +407,16 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
       Map<String, List<String>> ids = new LinkedHashMap<>();
       ids.put(TEST_ENTITY_TABLE, List.of("5", "999", "2", "4"));
       ids.put(OTHER_TABLE, List.of());
-      when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenReturn(ids);
+      when(mockClient.searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt())).thenReturn(hits(ids));
 
       RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withLimitPerTable(2));
 
       assertThat(output.getResults()).filteredOn(r -> r.getTableName().equals(TEST_ENTITY_TABLE)).extracting(RecordSearchResult::getRecordId).containsExactly(5, 2);
       assertThat(output.getResults()).filteredOn(r -> r.getTableName().equals(UNINDEXED_TABLE)).extracting(RecordSearchResult::getRecordId).containsExactly(1);
       assertThat(output.getResults()).filteredOn(r -> r.getTableName().equals(OTHER_TABLE)).isEmpty();
+      assertThat(output.getProviderStatus().getStatus()).isEqualTo(RecordSearchProviderState.OK);
+      assertThat(output.getTables()).filteredOn(t -> t.getTableName().equals(TEST_ENTITY_TABLE)).extracting(RecordSearchTableSummary::getSource, RecordSearchTableSummary::getHasMore)
+         .containsExactly(tuple(RecordSearchSource.PROVIDER, true));
    }
 
 
@@ -341,11 +428,14 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
    void testCoreRecordSearchAction_providerFailure_fallsBackToCoreSearch() throws QException
    {
       insertEntities("Widget", 3);
-      when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenThrow(new QException("OpenSearch is down"));
+      when(mockClient.searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt())).thenThrow(new QException("OpenSearch is down"));
 
       RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE)));
 
       assertThat(output.getResults()).extracting(RecordSearchResult::getRecordId).containsExactly(1, 2, 3);
+      assertThat(output.getProviderStatus().getStatus()).isEqualTo(RecordSearchProviderState.FAILED);
+      assertThat(output.getProviderStatus().getFailureClass()).isEqualTo("QException");
+      assertThat(output.getTables()).extracting(RecordSearchTableSummary::getSource).containsExactly(RecordSearchSource.FALLBACK);
       assertThat(runtime.isRecordSearchBackingOff()).isTrue();
 
       //////////////////////////////////////////////////////////////////////
@@ -354,7 +444,7 @@ class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
       //////////////////////////////////////////////////////////////////////
       RecordSearchOutput again = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE)));
       assertThat(again.getResults()).hasSize(3);
-      verify(mockClient, times(1)).searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt());
+      verify(mockClient, times(1)).searchRecordsPerTable(anyString(), anyList(), anyInt(), anyInt(), anyBoolean(), anyList(), anyInt());
    }
 
 }

src/test/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimerTest.java

@@ -21,7 +21,9 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -77,6 +79,38 @@ class QuickSearchConfigPrimerTest extends BaseDataDrivenTest
 
 
 
+   /*******************************************************************************
+    ** Re-priming only records the priming outcome on the settings row: that is
+    ** not audited. An admin settings edit still is.
+    *******************************************************************************/
+   @Test
+   void testRepriming_settingsBookkeepingIsNotAudited() throws QException
+   {
+      new AuditsMetaDataProvider().defineAll(QContext.getQInstance(), TEST_BACKEND_NAME, null);
+      new QuickSearchConfigPrimer(runtime).prime();
+      long afterFirst = settingsEditAudits();
+      new QuickSearchConfigPrimer(runtime).prime();
+      assertThat(settingsEditAudits()).as("re-priming is not audited").isEqualTo(afterFirst);
+
+      new UpdateAction().execute(new UpdateInput().withTableName(QuickSearchSettings.TABLE_NAME)
+         .withRecord(new QRecord().withValue("id", QuickSearchSettings.SETTINGS_ID).withValue("indexingPaused", true)));
+      assertThat(settingsEditAudits()).as("an admin settings edit is audited").isEqualTo(afterFirst + 1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Audit rows recording an edit of the settings row.
+    *******************************************************************************/
+   private long settingsEditAudits() throws QException
+   {
+      return (rows(AuditsMetaDataProvider.TABLE_NAME_AUDIT).stream()
+         .filter(audit -> audit.getValueString("message") != null && audit.getValueString("message").contains("Edited"))
+         .count());
+   }
+
+
+
    /*******************************************************************************
     ** Code defaults update rows nobody edited, and never rows an owner edited.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecksTest.java

@@ -0,0 +1,442 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.health;
+
+
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
+import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode;
+import com.kingsrook.qbits.quicksearch.SearchableFieldConfig;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.State;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
+import com.kingsrook.qbits.quicksearch.diagnostics.StatusCache;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthStatus;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+
+/*******************************************************************************
+ ** Registration and projections use memory records and mocked diagnostics only.
+ *******************************************************************************/
+class QuickSearchHealthChecksTest extends BaseQuickSearchTest
+{
+   private static final String SENTINEL = "S3NT1NEL-secret";
+   private QInstance instance;
+   private QuickSearchQBitConfig config;
+   private QuickSearchRuntime runtime;
+   private QuickSearchRuntime originalRuntime;
+   private StatusCache cache;
+
+
+   /*******************************************************************************
+    ** Exercise the real producer, with credentials that must never enter outcomes.
+    *******************************************************************************/
+   @BeforeEach
+   void setUpHealth() throws Exception
+   {
+      QInstance base = QContext.getQInstance();
+      instance = new QInstance();
+      instance.addBackend(base.getBackend(TEST_BACKEND_NAME));
+      instance.addTable(base.getTable(TEST_ENTITY_TABLE));
+      instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
+      QContext.init(instance, new QSystemUserSession());
+      config = new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withStartupMode(QuickSearchStartupMode.DEGRADED)
+         .withEnableRealTimeIndexing(false).withOpensearchUrl("https://unused.invalid")
+         .withOpensearchIndexName("health-test").withSearchableTable(TEST_ENTITY_TABLE, List.of(new SearchableFieldConfig("name")))
+         .withAuthMode(QuickSearchAuthMode.BASIC).withOpensearchUsername(SENTINEL).withOpensearchPassword(SENTINEL);
+      new QuickSearchQBitProducer().withConfig(config).produce(instance);
+      QuickSearchConfigPrimer.insertOne(config.getQuickSearchIndexTableName(), new QRecord().withValue("tableName", TEST_ENTITY_TABLE)
+         .withValue("enabled", true).withValue("status", "ACTIVE").withValue("documentCount", 12));
+      originalRuntime = config.getRuntime();
+      runtime = spy(originalRuntime);
+      cache = mock(StatusCache.class);
+      doReturn(cache).when(runtime).getStatusCache();
+      config.setRuntime(runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Release the producer's real worker even when tests replace the cache.
+    *******************************************************************************/
+   @AfterEach
+   void closeHealth()
+   {
+      if(originalRuntime != null)
+      {
+         originalRuntime.close();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Static checks and dynamically resolved indexes have the specified metadata.
+    *******************************************************************************/
+   @Test
+   void registration() throws Exception
+   {
+      QuickSearchHealthChecks.register(instance);
+      HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance);
+      assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName)
+         .containsExactly("quick_search.opensearch", "quick_search.backlog");
+      assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
+      {
+         assertThat(supplier.getNamePrefix()).isEqualTo("quick_search.index");
+         assertThat(supplier.getComponent()).isEqualTo("quick_search");
+      });
+      assertThat(registry.resolveAll(instance)).hasSize(3).allSatisfy(check ->
+      {
+         assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*");
+         assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
+         assertThat(check.getCategory()).isEqualTo(HealthCategory.SEARCH);
+         assertThat(check.getComponent()).isEqualTo("quick_search");
+         assertThat(check.getIntervalSeconds()).isEqualTo(60);
+         assertThat(check.getTimeoutMs()).isEqualTo(5000);
+      });
+      assertThat(registry.resolveAll(instance)).extracting(HealthCheckMetaData::getName).contains("quick_search.index.testentity");
+      verifyNoInteractions(cache);
+   }
+
+
+
+   /*******************************************************************************
+    ** Overall degradation from indexing must not downgrade a healthy cluster.
+    *******************************************************************************/
+   @ParameterizedTest
+   @CsvSource({"OK,true,green,UP", "DEGRADED,true,green,UP", "DEGRADED,true,yellow,DEGRADED", "DOWN,true,red,DOWN",
+      "DOWN,false,,DOWN", "DISABLED,,,UNKNOWN", "DEGRADED,,,DEGRADED", "OK,true,unexpected,DEGRADED"})
+   void clusterStates(State state, Boolean reachable, String color, HealthStatus expected) throws Exception
+   {
+      snapshot(state, reachable, color, false, 0, 0, false, List.of(row(true, "ACTIVE", 12L, 12L, null, 0)));
+      assertThat(run("opensearch").getStatus()).isEqualTo(expected);
+   }
+
+
+
+   /*******************************************************************************
+    ** Yellow is healthy only on one node with every unassigned shard a replica.
+    *******************************************************************************/
+   @ParameterizedTest
+   @CsvSource({"yellow,1,1,UP", "yellow,1,0,DEGRADED", "yellow,1,,DEGRADED", "yellow,2,1,DEGRADED", "yellow,,1,DEGRADED",
+      "green,1,0,UP", "red,1,1,DOWN"})
+   void singleNodeReplicas(String color, Integer nodes, Integer unassignedReplicas, HealthStatus expected) throws Exception
+   {
+      when(cache.get(eq(runtime), any(Instant.class))).thenReturn(new QuickSearchStatus(State.OK, true, color, nodes, 1, 1, unassignedReplicas, 0, false,
+         false, List.of(row(true, "ACTIVE", 12L, 12L, null, 0)), 0, 0, false, false, null, SENTINEL, Instant.now(), "/unused"));
+      HealthCheckOutcome outcome = run("opensearch");
+      assertThat(outcome.getStatus()).isEqualTo(expected);
+      assertThat(outcome.getDetails().toJson()).contains("\"targetReplicas\":0", "\"replicaSettingsFailed\":false");
+      if(expected == HealthStatus.UP && "yellow".equals(color))
+      {
+         assertThat(outcome.getDetails().toJson()).contains("\"unassignedReplicaShards\":1", "replicas cannot be assigned on a single-node cluster");
+      }
+      assertClean(outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Boundary values, exhausted events, pause, truncation and missing measurements.
+    *******************************************************************************/
+   @ParameterizedTest
+   @CsvSource({"OK,false,0,0,false,UP", "DEGRADED,false,100,0,false,UP", "DEGRADED,false,101,0,false,DEGRADED",
+      "DEGRADED,false,0,1,false,DEGRADED", "DEGRADED,true,0,0,false,DEGRADED", "OK,false,0,0,true,DEGRADED",
+      "DEGRADED,false,,0,false,DEGRADED", "DEGRADED,false,0,,false,DEGRADED", "DEGRADED,,0,0,false,DEGRADED",
+      "DISABLED,false,0,0,false,UNKNOWN", "DOWN,false,0,0,false,UP"})
+   void backlogStates(State state, Boolean paused, Integer pending, Integer exhausted, Boolean truncated, HealthStatus expected) throws Exception
+   {
+      snapshot(state, true, "green", paused, pending, exhausted, truncated, List.of());
+      assertThat(run("backlog").getStatus()).isEqualTo(expected);
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts, index status, last errors and real-time errors each affect the check.
+    *******************************************************************************/
+   @ParameterizedTest
+   @CsvSource({"true,ACTIVE,12,12,,0,UP", "false,ACTIVE,12,12,,0,UNKNOWN", "true,REBUILDING,12,12,,0,DEGRADED",
+      "true,ACTIVE,12,13,,0,DEGRADED", "true,ACTIVE,12,12,error,0,DEGRADED", "true,ACTIVE,12,12,,2,DEGRADED",
+      "true,ACTIVE,12,,,0,UP", "true,ACTIVE,,12,,,UP", "true,,12,12,,0,DEGRADED"})
+   void indexStates(Boolean enabled, String state, Long documents, Long source, String error, Integer realTimeErrors, HealthStatus expected) throws Exception
+   {
+      snapshot(State.OK, true, "green", false, 0, 0, false, List.of(row(enabled, state, documents, source, error, realTimeErrors)));
+      HealthCheckOutcome outcome = run("index.testentity");
+      assertThat(outcome.getStatus()).isEqualTo(expected);
+      if(realTimeErrors != null)
+      {
+         assertThat(outcome.getDetails().getValues()).containsEntry("realTimeErrorCount", realTimeErrors);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Supplier identities are refreshed; checks never retain old measurements.
+    *******************************************************************************/
+   @Test
+   void dynamicRowsAndMissingSnapshot() throws Exception
+   {
+      HealthCheckMetaData check = check("index.testentity");
+      snapshot(State.OK, true, "green", false, 0, 0, false, List.of(row(true, "ACTIVE", 12L, 12L, null, 0)));
+      assertThat(check.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP);
+      snapshot(State.OK, true, "green", false, 0, 0, false, List.of());
+      assertThat(check.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      QuickSearchConfigPrimer.insertOne(config.getQuickSearchIndexTableName(), new QRecord().withValue("tableName", "New.Index-Name")
+         .withValue("enabled", false).withValue("status", "ACTIVE"));
+      assertThat(HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance)).extracting(HealthCheckMetaData::getName)
+         .contains("quick_search.index.new.index-name");
+   }
+
+
+
+   /*******************************************************************************
+    ** Raw outcomes omit poisoned diagnostics and exceptions; sanitize is package-private in core.
+    *******************************************************************************/
+   @Test
+   void noSecrets() throws Exception
+   {
+      List<HealthCheckMetaData> checks = HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance);
+      snapshot(State.DEGRADED, true, SENTINEL, true, 101, 1, false, List.of(row(true, SENTINEL, 12L, 13L, SENTINEL, 9)));
+      for(HealthCheckMetaData check : checks)
+      {
+         assertClean(check.getCheckInstance().run(input()));
+      }
+      when(cache.get(eq(runtime), any(Instant.class))).thenThrow(new QException("Failure " + SENTINEL));
+      for(HealthCheckMetaData check : checks)
+      {
+         HealthCheckOutcome outcome = check.getCheckInstance().run(input());
+         assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN);
+         assertClean(outcome);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An expired deadline does no diagnostic work; missing runtime is UNKNOWN.
+    *******************************************************************************/
+   @Test
+   void expiredAndUnconfigured() throws Exception
+   {
+      for(HealthCheckMetaData check : HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance))
+      {
+         assertThat(check.getCheckInstance().run(input().withDeadline(Instant.now().minusSeconds(1))).getStatus()).isEqualTo(HealthStatus.DOWN);
+         assertThat(check.getCheckInstance().run(input().withQInstance(new QInstance())).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      }
+      assertThat(new QuickSearchHealthChecks().supply(new QInstance())).isEmpty();
+      verifyNoInteractions(cache);
+   }
+
+
+
+   /*******************************************************************************
+    ** The real cache also removes exception text from a failed diagnostic refresh.
+    *******************************************************************************/
+   @Test
+   void realCacheFailureIsClean() throws Exception
+   {
+      doReturn(originalRuntime.getStatusCache()).when(runtime).getStatusCache();
+      doThrow(new QException("Failed " + SENTINEL)).when(runtime).getClient();
+      assertThat(run("opensearch").getStatus()).isEqualTo(HealthStatus.DOWN);
+      for(HealthCheckMetaData check : HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance))
+      {
+         assertClean(check.getCheckInstance().run(input()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A stuck refresh cannot hold another caller beyond its shorter deadline.
+    *******************************************************************************/
+   @Test
+   void cacheWaitAndLockRespectDeadline() throws Exception
+   {
+      CountDownLatch entered = new CountDownLatch(1);
+      CountDownLatch release = new CountDownLatch(1);
+      doReturn(originalRuntime.getStatusCache()).when(runtime).getStatusCache();
+      org.mockito.Mockito.doAnswer(invocation ->
+      {
+         entered.countDown();
+         release.await(5, TimeUnit.SECONDS);
+         throw (new QException("Failed " + SENTINEL));
+      }).when(runtime).getClient();
+      try(var executor = Executors.newSingleThreadExecutor())
+      {
+         var first = executor.submit(() ->
+         {
+            QContext.init(instance, new QSystemUserSession());
+            try
+            {
+               return (runtime.getStatusCache().get(runtime, Instant.now().plusSeconds(2)));
+            }
+            finally
+            {
+               QContext.clear();
+            }
+         });
+         try
+         {
+            assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue();
+            Instant started = Instant.now();
+            assertThatThrownBy(() -> runtime.getStatusCache().get(runtime, started.plusMillis(100))).isInstanceOf(QException.class);
+            assertThat(Instant.now()).isBefore(started.plusSeconds(1));
+         }
+         finally
+         {
+            release.countDown();
+         }
+         first.get(3, TimeUnit.SECONDS);
+      }
+      assertThatThrownBy(() -> runtime.getStatusCache().get(runtime, Instant.now().minusSeconds(1))).isInstanceOf(QException.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** A short health deadline bounds the refresh wait as well as lock contention.
+    *******************************************************************************/
+   @Test
+   void refreshWaitRespectsDeadline() throws Exception
+   {
+      CountDownLatch release = new CountDownLatch(1);
+      doReturn(originalRuntime.getStatusCache()).when(runtime).getStatusCache();
+      org.mockito.Mockito.doAnswer(invocation ->
+      {
+         release.await(5, TimeUnit.SECONDS);
+         throw (new QException("Failed " + SENTINEL));
+      }).when(runtime).getClient();
+      try
+      {
+         HealthCheckMetaData check = check("opensearch");
+         Instant started = Instant.now();
+         HealthCheckOutcome outcome = check.getCheckInstance().run(input().withDeadline(started.plusMillis(100)));
+         assertThat(Instant.now()).isBefore(started.plusSeconds(1));
+         assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN);
+         assertClean(outcome);
+      }
+      finally
+      {
+         release.countDown();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Supply a deterministic snapshot without creating a network transport.
+    *******************************************************************************/
+   private void snapshot(State state, Boolean reachable, String color, Boolean paused, Integer pending, Integer exhausted, Boolean truncated, List<IndexRow> rows) throws Exception
+   {
+      when(cache.get(eq(runtime), any(Instant.class))).thenReturn(new QuickSearchStatus(state, reachable, color, 1, 2, 0, paused, rows,
+         pending, exhausted, truncated, false, null, SENTINEL, Instant.now(), "/unused"));
+   }
+
+
+
+   /*******************************************************************************
+    ** A row whose identity is deliberately mixed case.
+    *******************************************************************************/
+   private IndexRow row(Boolean enabled, String state, Long documents, Long source, String error, Integer realTimeErrors)
+   {
+      return (new IndexRow(TEST_ENTITY_TABLE, enabled, state, documents, source, Instant.now(), Instant.now(), error, realTimeErrors));
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve through the actual core registry, including supplier validation.
+    *******************************************************************************/
+   private HealthCheckMetaData check(String suffix) throws Exception
+   {
+      return (HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).stream()
+         .filter(check -> check.getName().equals("quick_search." + suffix)).findFirst().orElseThrow());
+   }
+
+
+
+   /*******************************************************************************
+    ** Run one registered check with a bounded input.
+    *******************************************************************************/
+   private HealthCheckOutcome run(String suffix) throws Exception
+   {
+      return (check(suffix).getCheckInstance().run(input()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The runner always supplies an instance and absolute deadline.
+    *******************************************************************************/
+   private HealthCheckInput input()
+   {
+      return (new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Assert the raw public boundary, without relying on core's later redaction.
+    *******************************************************************************/
+   private void assertClean(HealthCheckOutcome outcome)
+   {
+      assertThat(outcome.getMessage()).doesNotContain(SENTINEL);
+      assertThat(outcome.getDetails().toJson()).doesNotContain(SENTINEL);
+   }
+}

src/test/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchReplicasTest.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.opensearch;
+
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.ReplicaStatus;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Replica sizing and reconcile against a stub OpenSearch HTTP server.
+ *******************************************************************************/
+class QuickSearchReplicasTest
+{
+   private HttpServer   server;
+   private List<String> requests;
+   private List<String> bodies;
+   private Integer      dataNodes;
+   private String       settings;
+   private Integer      settingsStatus;
+   private Integer      putStatus;
+
+
+
+   /*******************************************************************************
+    ** Stub: _cluster/health reports dataNodes, GET _settings returns settings,
+    ** PUT _settings returns putStatus; every request is recorded.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws IOException
+   {
+      requests       = new ArrayList<>();
+      bodies         = new ArrayList<>();
+      dataNodes      = 1;
+      settingsStatus = 200;
+      putStatus      = 200;
+      settings       = settingsBody("1", null);
+      server         = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+      server.createContext("/", this::handle);
+      server.start();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      server.stop(0);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void handle(HttpExchange exchange) throws IOException
+   {
+      String request = exchange.getRequestMethod() + " " + exchange.getRequestURI().getPath();
+      requests.add(request);
+      bodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
+      int    status = 200;
+      String body   = "{}";
+      if(request.equals("GET /_cluster/health"))
+      {
+         body = dataNodes == null ? "{\"status\":\"green\"}" : "{\"status\":\"yellow\",\"number_of_data_nodes\":" + dataNodes + "}";
+      }
+      else if(request.startsWith("GET ") && request.endsWith("/_settings"))
+      {
+         status = settingsStatus;
+         body   = status == 200 ? settings : "{\"error\":{\"reason\":\"nope\"}}";
+      }
+      else if(request.startsWith("PUT ") && request.endsWith("/_settings"))
+      {
+         status = putStatus;
+         body   = status == 200 ? "{\"acknowledged\":true}" : "{\"error\":{\"reason\":\"blocked\"}}";
+      }
+      byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+      exchange.getResponseHeaders().add("Content-Type", "application/json");
+      exchange.sendResponseHeaders(status, bytes.length);
+      exchange.getResponseBody().write(bytes);
+      exchange.close();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String settingsBody(String replicas, String autoExpand)
+   {
+      return ("{\"alias-v2-1\":{\"settings\":{\"index\":{\"number_of_shards\":\"1\",\"number_of_replicas\":\"" + replicas + "\""
+         + (autoExpand == null ? "" : ",\"auto_expand_replicas\":\"" + autoExpand + "\"") + "}}}}");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QuickSearchQBitConfig config()
+   {
+      return (new QuickSearchQBitConfig().withOpensearchUrl("http://127.0.0.1:" + server.getAddress().getPort()).withOpensearchIndexName("alias"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Automatic sizing never exceeds data nodes - 1 nor maxAutoReplicas.
+    *******************************************************************************/
+   @ParameterizedTest
+   @CsvSource({"1,1,0", "2,1,1", "3,1,1", "3,2,2", "5,2,2", "0,1,0", "4,0,0"})
+   void autoReplicas(int nodes, int max, int expected)
+   {
+      assertThat(QuickSearchOpenSearchClient.autoReplicas(nodes, max)).isEqualTo(expected);
+   }
+
+
+
+   /*******************************************************************************
+    ** Zero config on one node: the replica is dropped, once.
+    *******************************************************************************/
+   @Test
+   void singleNodeDropsReplicaIdempotently() throws Exception
+   {
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config()))
+      {
+         ReplicaStatus status = client.reconcileReplicas();
+         assertThat(status).isEqualTo(new ReplicaStatus(0, 1, 1, false));
+         assertThat(client.getReplicaStatus()).isEqualTo(status);
+         assertThat(requests).contains("PUT /alias-v2-1/_settings");
+         assertThat(bodies.get(requests.indexOf("PUT /alias-v2-1/_settings"))).isEqualTo("{\"index\":{\"number_of_replicas\":0}}");
+
+         settings = settingsBody("0", null);
+         requests.clear();
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(0, 1, 0, false));
+         assertThat(requests).noneMatch(request -> request.startsWith("PUT"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Multi-node clusters keep (or get) one replica by default.
+    *******************************************************************************/
+   @Test
+   void multiNodeKeepsOneReplica() throws Exception
+   {
+      dataNodes = 3;
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config()))
+      {
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(1, 3, 0, false));
+         assertThat(requests).noneMatch(request -> request.startsWith("PUT"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An explicit indexReplicas wins and does not need cluster health.
+    *******************************************************************************/
+   @Test
+   void explicitReplicasOverrideAuto() throws Exception
+   {
+      settings = settingsBody("0", null);
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config().withIndexReplicas(2)))
+      {
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(2, null, 1, false));
+         assertThat(requests).doesNotContain("GET /_cluster/health");
+         assertThat(bodies.get(requests.indexOf("PUT /alias-v2-1/_settings"))).isEqualTo("{\"index\":{\"number_of_replicas\":2}}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An operator's auto_expand_replicas is left alone.
+    *******************************************************************************/
+   @Test
+   void autoExpandIsRespected() throws Exception
+   {
+      settings = settingsBody("1", "0-1");
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config()))
+      {
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(0, 1, 0, false));
+         assertThat(requests).noneMatch(request -> request.startsWith("PUT"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Failures are recorded, never thrown.
+    *******************************************************************************/
+   @Test
+   void failuresAreToleratedAndRecorded() throws Exception
+   {
+      putStatus = 403;
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config()))
+      {
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(0, 1, 0, true));
+
+         settingsStatus = 500;
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(0, 1, 0, true));
+
+         dataNodes = null;
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(null, null, 0, true));
+         assertThat(client.getReplicaStatus().failed()).isTrue();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An unreachable cluster is a recorded failure too.
+    *******************************************************************************/
+   @Test
+   void unreachableIsRecorded() throws Exception
+   {
+      QuickSearchQBitConfig config = config();
+      server.stop(0);
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config.withConnectTimeoutMillis(500)))
+      {
+         assertThat(client.reconcileReplicas()).isEqualTo(new ReplicaStatus(null, null, 0, true));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** OpenSearch Serverless manages replicas itself: nothing is sent.
+    *******************************************************************************/
+   @Test
+   void serverlessIsNotManaged() throws Exception
+   {
+      QuickSearchQBitConfig config = config().withAuthMode(QuickSearchAuthMode.AWS_SIGV4).withAwsServiceName("aoss").withAwsRegion("us-east-1");
+      try(QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config))
+      {
+         assertThat(client.reconcileReplicas()).isNull();
+         assertThat(client.getReplicaStatus()).isNull();
+         assertThat(requests).isEmpty();
+      }
+   }
+}

src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -324,6 +325,45 @@ class BasepullIndexStepTest extends BaseQuickSearchTest
 
 
 
+   /*******************************************************************************
+    ** Basepull bookkeeping on the index row (status, lastBasepullTime,
+    ** documentCount, last run) writes no audit; an admin edit of the same row
+    ** is still audited.
+    *******************************************************************************/
+   @Test
+   void testBasepullBookkeeping_isNotAudited_adminEditIs() throws QException
+   {
+      if(QContext.getQInstance().getTable(AuditsMetaDataProvider.TABLE_NAME_AUDIT) == null)
+      {
+         new AuditsMetaDataProvider().defineAll(QContext.getQInstance(), TEST_BACKEND_NAME, null);
+      }
+      insertTestEntities(3);
+      QRecord row = insertIndexRow(TEST_ENTITY_TABLE, true, null, 5);
+      when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenReturn(3L);
+
+      new BasepullIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
+      assertThat(queryAll(QuickSearchIndex.TABLE_NAME).get(0).getValueInteger("documentCount")).isEqualTo(3);
+      assertThat(indexRowEditAudits()).as("basepull bookkeeping is not audited").isZero();
+
+      new UpdateAction().execute(new UpdateInput().withTableName(QuickSearchIndex.TABLE_NAME)
+         .withRecord(new QRecord().withValue("id", row.getValue("id")).withValue("basepullIntervalMinutes", 15)));
+      assertThat(indexRowEditAudits()).as("an admin configuration edit is audited").isEqualTo(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Audit rows recording an edit of a quick_search_index row.
+    *******************************************************************************/
+   private long indexRowEditAudits() throws QException
+   {
+      return (queryAll(AuditsMetaDataProvider.TABLE_NAME_AUDIT).stream()
+         .filter(audit -> audit.getValueString("message") != null && audit.getValueString("message").contains("Edited"))
+         .count());
+   }
+
+
+
    /*******************************************************************************
     ** Test: no discovered tables, no quickSearchIndex rows - step completes
     ** without calling the OpenSearch client.