fix(status): count real-time indexing failures

realTimeErrorCount was only ever written as 0, so the status screen,
diagnostics and the status API always said there were no real-time
errors. When the change listener records failed events it now adds
their number to the table's count (after commit, serialized in the JVM,
never failing the user's write). A successful basepull or full reindex
resets it; the field's help text says so.

Fixes #53

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.circleci/config.yml

@@ -22,7 +22,7 @@ workflows:
               context: [qqq-maven-registry-credentials]
               filters:
                  branches:
-                    ignore: /(develop|release\/.*|hotfix\/.*)/
+                    ignore: /(develop|release\/.*)/
 
    publish_snapshot:
       jobs:
@@ -57,15 +57,4 @@ workflows:
                  branches:
                     ignore: /.*/
                  tags:
-                    only: [/v.*/]
-
-   publish_hotfix_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: hotfix
-              filters:
-                 branches:
-                    only: [/hotfix\/.*/]
+                    only: [/^v\d+\.\d+\.\d+$/]

.editorconfig

@@ -0,0 +1,20 @@
+# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 3
+tab_width = 3
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{cmd,bat}]
+end_of_line = crlf

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto eol=lf
+*.cmd text eol=crlf
+mvnw text eol=lf

.github/dependabot.yml

@@ -0,0 +1,25 @@
+# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
+# major updates stay separate so each gets its own review. At most three version-update PRs per
+# ecosystem are open at once. Security updates are enabled in the repository settings and are
+# not limited by this schedule.
+version: 2
+updates:
+  - package-ecosystem: maven
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      maven-minor-and-patch:
+        update-types: [minor, patch]
+
+  - package-ecosystem: github-actions
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      github-actions:
+        patterns: ['*']

.github/workflows/release-rc.yml

@@ -0,0 +1,25 @@
+# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
+# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
+# the release line and the release/* push runs CircleCI publish_release_candidate.
+name: Release candidate
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Print the plan without merging or creating branches
+        type: boolean
+        default: false
+  pull_request:
+    types: [closed]
+    branches: [develop]
+
+permissions:
+  contents: write
+
+jobs:
+  release-rc:
+    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
+    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    with:
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}

AGENTS.md

@@ -15,7 +15,7 @@ This file provides guidance to Codex when working with code in this repository.
 ./mvnw package          # Build JAR
 CI=true ./mvnw verify   # Full verify: integration tests (Docker) and the JaCoCo gate
 ./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0   # other server line
-./mvnw verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT   # other qqq 4.1 build (profile is active by default)
+./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1   # other qqq 4.1 build (profile is active by default)
 ```
 
 Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
@@ -31,12 +31,13 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
 - `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
 - `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
 - `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
-- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
+- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quick_search_failed_event`.
 - `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
 - `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
+- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
 - `metadata/*` - Component producers and the table/process helpers.
 
-**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
+**Tables Produced:** `quick_search_index`, `quick_search_index_run`, `quick_search_failed_event` (plus a PVS over `quick_search_index`).
 
 **Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
 
@@ -50,7 +51,7 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
 ## Dependencies
 
 - Java 21
-- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom:4.1.0-RC.1` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007)
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI
 - OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
 - Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
 - JUnit 5 + AssertJ + Mockito for testing

CHANGELOG.md

@@ -2,6 +2,52 @@
 
 All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
 
+## [Unreleased]
+
+Requires QQQ 4.1.0-RC.6 or later (the spec-managed UI API extension contract).
+
+### Fixed
+- Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write.
+
+### Added
+- Supported host status API (`api.QuickSearchStatusService.get()`), immutable status values, and `QuickSearchApi` home navigation constants (issue #50/F10). Permission-checked, bounded diagnostics refreshes share a short per-runtime cache and reuse saved validation counts. New spec-managed `GET /qqq/v1/qbit/quick-search/status` (`getQuickSearchStatus`) with committed OpenAPI fragment and strict HTTP/contract tests. RC.6 requires the default `/qbit/quick-search` prefix; `/quick-search` is not permitted by core. Existing routes and wire shapes are unchanged.
+- Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`.
+
+- **Data-driven configuration.** The searchable tables and fields are records: `quick_search_index` (the table registry, new columns `source` and `ownerEdited`) and the new `quick_search_index_field` (weight, role `DISPLAY`/`TITLE`/`SUBTITLE`/`BODY`/`KEYWORD`/`EXACT`, analyzer `PREFIX`/`WORD`/`ENGLISH`/`EXACT`, snippet and label flags). Indexing, search, the record-search provider and the diagnostics read the effective config from them (cached 30 s, dropped on change), so changes need no redeploy. Code (`@QuickSearchable`, `searchableTables`) only seeds them; rows an admin edited are never overwritten, and removals are remembered in the new `quick_search_config_tombstone`. Precedence and DDL: `docs/MIGRATION-1.1.md`, "Data-driven configuration".
+- **Self-priming.** New code-declared tables and fields get their rows at startup and are built by the next basepull (`PENDING_BUILD`, `BUILDING`); a missing index is recreated and rebuilt, an empty one rebuilt; existing rows are migrated without a rebuild. The Search Diagnostics widget reports it; `QuickSearchStatus` gives hosts the same banner.
+- Admin processes with review and result steps (record or table actions, never in the nav): Add Search Index (the valid create: eligible-table picker, estimated record count, default fields), Apply Search Index Changes (previews field changes, then rebuilds), Disable Search Index and Remove Search Index (delete or keep documents), Cancel Index Run (cooperative cancel between batches; runs end `CANCELLED` with counts), Pause Indexing and Resume Indexing (global pause in the new `quick_search_settings`; search keeps serving, changes are deferred and replayed).
+- `QuickSearchResult.subtitle` and `displayValues` (from `SUBTITLE` and `DISPLAY` fields); document objects `fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`, added to existing indexes' mapping at startup.
+- Possible-value sources `quick_search_eligible_table`, `quick_search_indexable_field`, `quick_search_timestamp_field`, `quick_search_field_role`, `quick_search_field_analyzer`, `quick_search_documents_choice`, `quick_search_config_source`; the join `quick_search_index_join_quick_search_index_field` and the Searchable Fields widget on the index record. Config tables are audited at field level.
+
+### Changed
+- `quick_search_index`: the generic Create and Delete are off (`TABLE_INSERT`, `TABLE_DELETE`); use Add Search Index and Remove Search Index. `quick_search_searchable_table` is now a custom source over the effective tables. `quick_search_index_run` gains `cancelRequested` and `cancelRequestedBy`.
+- Hidden and `isSensitive` fields are never indexed (previously hidden and password only).
+- Record screens (record-screen audit): `quick_search_index_run` and `quick_search_failed_event` are written by the QBit only, so Create and Edit are off (Delete stays, for clearing); both get named field sections, record labels and pickers (`quick_search_run_type`, `quick_search_run_status`, `quick_search_failed_event_status`, `quick_search_failed_event_action`, and the table picker for `tableName`). The index record lists its Recent Runs (join `quick_search_index_join_quick_search_index_run`). `quick_search_validation_run.reportJson` uses a JSON code editor. The Basepull, Full Reindex and Reconcile processes run from Quick Search Index's Actions and are no longer app (nav) children; their table input is `searchTableName` (`tableName` still works for API callers, and a Quick Search table name is ignored), so running them from the table no longer limits them to `quick_search_index`. Process input steps now carry `HELP_TEXT` and `EDIT_FORM` components, so the Next UI renders their fields.
+- **Snake_case names (owner naming rule, 2026-10-12), for 1.1.0-RC.3.** The tables are now `quick_search_index`, `quick_search_index_run`, `quick_search_failed_event`, `quick_search_diagnostic_event` and `quick_search_validation_run` (they were camelCase). The possible-value sources are `quick_search_index` and `quick_search_searchable_table`. The permission bases are `quick_search_admin` and `quick_search_debug`, and table permissions follow the table names. A table-name prefix is now converted to snake_case and joined with `_`. The old names remain as `@Deprecated` `LEGACY_*` constants. This is metadata only: the QBit ships no DDL. Upgrade steps, with SQL for camelCase databases and for stored references (audit table names, saved views, permission grants), are in `docs/MIGRATION-1.1.md` under "Snake_case names (1.1.0-RC.3)".
+
+## [1.1.0-RC.1] - 2026-10-11
+
+First release candidate of 1.1, published to Maven Central from `release/1.1`. It contains every change listed under 1.1.0 and 1.0.0 below (1.0.0 has not been released as GA). Requires QQQ 4.1.0-RC.2 or later. Upgrade notes: `docs/MIGRATION-1.1.md`.
+
+## [1.1.0] - Unreleased
+
+Requires QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI; built against 4.1.0-RC.2. See `docs/MIGRATION-1.1.md`.
+
+### Added
+- `QuickSearchRecordSearchProvider`: serves QQQ core record search (`RecordSearchAction`, `POST /qqq/v1/search`, the Next UI global search) from OpenSearch for the tables Quick Search indexes. One `_msearch` request with a per-table top-N search (same query, AND semantics and field weights as `QuickSearchAction`); returns record ids only, which core re-reads through `QueryAction` (permissions, record locks, labels). Audit finding F-11, decision D1.
+- `serveCoreRecordSearch` (default `true`): registers the provider and gives indexed tables core `searchFields` (visible string and integer searchable fields) when they have none. Host `searchFields` and a host-registered provider are never replaced.
+- `recordSearchTimeoutMillis` (default 2500): wall-clock limit on the OpenSearch request; on timeout or any failure core falls back to its own search, and the provider claims no tables for 30 seconds.
+- `QuickSearchOpenSearchClient.searchRecordIdsPerTable`.
+- Diagnostic tools (#35), on by default (`enableDiagnosticTools`): Search Playground, Validate Search Index (schedulable with `validationCronExpression`; reports kept in the new `quickSearchValidationRun` table), Test Search Analyzer, and on every indexed table and on `quickSearchFailedEvent` rows the record actions Inspect Search Document (database vs index, and why a record does or does not match a term) and Reindex in Search. All are table or record actions, gated by the new `debugPermissionRules` (`quickSearchDebug.hasAccess`; Reindex in Search uses `adminPermissionRules`), rate-limited per user (`diagnosticRunsPerMinute`), and recorded in the new `quickSearchDiagnosticEvent` table. Settings: `enableRecordDiagnosticActions`, `validationCronExpression`, `validationCronTimeZoneId`, `validationSampleSize`.
+- Search failure diagnostics: a failed or timed-out OpenSearch request behind core record search or `QuickSearchAction` is recorded as a `SEARCH_FAILED` diagnostic event (tables, duration, error, server; never the term); the runtime counts failures and backoff fallbacks. The Search Diagnostics widget (`quickSearchDiagnostics`, on the admin app home) shows the provider's state, recent failures and the latest validation.
+- `QuickSearchDiagnosticsOperations`: read-only probe search (named clauses, field highlights, request and response JSON), explain, document reads, `_analyze`, document sampling, and raw cluster, mapping and settings reads.
+
+### Changed
+- Run history purging also purges `quickSearchDiagnosticEvent` and `quickSearchValidationRun` rows older than `runHistoryRetentionDays`.
+
+### Security
+- Fields of type `PASSWORD` are no longer indexed (as hidden fields already were). Reconcile or fully reindex once to remove values a 1.0 index may hold.
+
 ## [1.0.0-RC.2] - 2026-10-06
 
 Second release candidate, published to Maven Central from `release/1.0`. Fixes the five pre-GA findings from the PR #10 review (#13 to #17); the changes are listed under 1.0.0 below. Requires QQQ 4.1. No schema change, but two new status values: `REBUILDING` on `quickSearchIndex.status` and `AWAITING_REINDEX` on `quickSearchFailedEvent.status`; a host that sized its own `quickSearchFailedEvent.status` column needs at least 16 characters. Known gaps before GA are tracked in #12.
@@ -55,6 +101,7 @@ Requires QQQ 4.1. Closes every blocker of the 2026-10-04 production-readiness au
 - An update was dropped without a `quickSearchFailedEvent` row when the listener's re-read failed or the optional AWS SDK was missing (`LinkageError`); both are now recorded for basepull, and a `DEGRADED` start without the AWS SDK no longer aborts host startup (#13).
 - Search pages overlapped under record locks, per-table mode ignored `offset`, and a large `maxSearchLimit` could exceed the 10,000 result window (#16).
 - A failing document count was swallowed and cleared `documentCount`; it is now logged at warn and the previous count is kept (#17).
+- A real-time insert was indexed with raw values (a foreign key as its id, the record label as "2 - email") until the next basepull. Inserts are now re-read in the write's transaction with display values and possible-value labels, as updates are; a failed re-read is recorded in `quickSearchFailedEvent` for basepull and never fails the insert (#28).
 
 ## [0.2.0] - 2026-03-30
 - Config-driven table indexing (`SearchableTableConfig`).

CLAUDE.md

@@ -15,7 +15,7 @@ This file provides guidance to Claude Code when working with code in this reposi
 ./mvnw package          # Build JAR
 CI=true ./mvnw verify   # Full verify: integration tests (Docker) and the JaCoCo gate
 ./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0   # other server line
-./mvnw verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT   # other qqq 4.1 build (profile is active by default)
+./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1   # other qqq 4.1 build (profile is active by default)
 ```
 
 Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
@@ -31,12 +31,15 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
 - `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
 - `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
 - `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
-- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
+- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quick_search_failed_event`.
 - `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
 - `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
+- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
 - `metadata/*` - Component producers and the table/process helpers.
 
-**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
+**Tables Produced:** `quick_search_index`, `quick_search_index_field`, `quick_search_settings`, `quick_search_config_tombstone`, `quick_search_index_run`, `quick_search_failed_event`, `quick_search_diagnostic_event`, `quick_search_validation_run` (plus PVSs).
+
+**Data-driven config (`config/`):** the records are the source of truth. `QuickSearchConfigStore` builds the effective `QuickSearchableTableConfig`s from `quick_search_index` + `quick_search_index_field` (30 s cache, dropped by the tables' customizers); `QuickSearchRuntime.getDiscoveredTables()/getTableConfig()` return them, `getCodeTables()` the code defaults. `QuickSearchConfigPrimer` seeds records from code (never over `ownerEdited` rows or tombstones). QBit writes to config tables go through `QuickSearchConfigWrites.asSystem` so customizers do not treat them as owner edits. Admin processes live in `processes/config/` (streamed ETL with review steps).
 
 **Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
 
@@ -50,7 +53,7 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
 ## Dependencies
 
 - Java 21
-- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom:4.1.0-RC.1` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007)
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.4`: the record-search provider SPI arrived in RC.2, the developer panel contract (`DeveloperPanelProviderInterface`) in RC.4
 - OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
 - Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
 - JUnit 5 + AssertJ + Mockito for testing

README.md

@@ -1,6 +1,6 @@
 # QBit: Quick Search
 
-[![Version](https://img.shields.io/badge/version-1.0.0--RC.1-blue.svg)](https://github.com/QRun-IO/qbit-quick-search)
+[![Version](https://img.shields.io/badge/version-1.1.0--RC.1-blue.svg)](https://github.com/QRun-IO/qbit-quick-search)
 [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://www.apache.org/licenses/LICENSE-2.0)
 [![Java](https://img.shields.io/badge/java-21+-blue.svg)](https://adoptium.net/)
 
@@ -8,19 +8,20 @@
 
 Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
 
-Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services).
+Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the 1.1 line builds against 4.1.0-RC.4 from the next release candidate on.
 
 ## What it does
 
 | Capability | How |
 |---|---|
-| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quickSearchFailedEvent` and replayed by the scheduled basepull. |
+| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quick_search_failed_event` and replayed by the scheduled basepull. |
 | Basepull catch-up | Scheduled process (`schedulerName` plus `basepullRepeatSeconds`) that re-indexes rows changed since the previous run's start, with a configurable overlap, paging by primary key. |
 | Reconcile | Scheduled or on-demand process that re-indexes everything in place, then removes documents whose source row is gone, including documents of tables no longer configured. No search blackout. |
 | Full rebuild | Indexes every table into a fresh physical index and atomically swaps the alias; the old index keeps serving until the new one is complete. Also upgrades an index created by an older release. |
+| Core search | `QuickSearchRecordSearchProvider` serves QQQ's `RecordSearchAction` (`POST /qqq/v1/search`, the Next UI global search box) from OpenSearch for the indexed tables, with a timeout and an automatic fallback to core's own search. |
 | Search | `QuickSearchAction`: AND semantics, edge-ngram typeahead with a plain search analyzer, per-field weights, highlights, `tableNames` and `limitPerTable`, bounded paging, `tableLabel` on results. Only tables the session may read are searched; hits are re-read through `QueryAction` so record security locks apply. |
 | Connections | HTTP or HTTPS, basic auth, AWS IAM SigV4, custom CA or trust store, mTLS, timeouts, pool size, a transport customizer SPI for anything else. Secrets by `${env.X}` reference. |
-| Operations | `quickSearchIndex` (per-table status, counts, last run, last error), `quickSearchIndexRun` (history, purged after `runHistoryRetentionDays`), `quickSearchFailedEvent`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
+| Operations | `quick_search_index` (per-table status, counts, last run, last error), `quick_search_index_run` (history, purged after `runHistoryRetentionDays`), `quick_search_failed_event`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
 
 ## Supported platforms
 
@@ -48,11 +49,11 @@ For `AWS_SIGV4` add these optional dependencies to your host (the QBit declares
 <dependency>
     <groupId>com.kingsrook.qbits</groupId>
     <artifactId>qbit-quick-search</artifactId>
-    <version>1.0.0-RC.1</version>
+    <version>1.1.0-RC.1</version>
 </dependency>
 ```
 
-`1.0.0-RC.1` is the current release candidate; `1.0.0` follows once host applications have adopted it (#12).
+`1.1.0-RC.1` is the current release candidate (the 1.1 line, published from `release/1.1`; it needs QQQ 4.1.0-RC.2 or later, see [Upgrading from 1.0](#upgrading-from-10)). Hosts staying on the 1.0 line use `1.0.0-RC.2`; `1.0.0` follows once host applications have adopted it (#12).
 
 Annotate entity classes:
 
@@ -99,7 +100,7 @@ config.withSearchableTable(new SearchableTableConfig("crmContact", List.of(
    .withRecordLabelFormat("%s %s", "firstName", "lastName"));
 ```
 
-Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull.
+Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden and password fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull, and the QBit logs a warning for it at startup. Set `basepullTimestampField` to null on a config-driven table to opt out without the warning.
 
 ### Searching
 
@@ -124,16 +125,85 @@ output.getTotalHits(); output.getTotalHitsIsLowerBound(); output.getHasMore();
 
 Search runs as the current `QContext` session: tables the session cannot read are skipped, and with `applyRecordSecurityLocks` (default) hits are re-read through `QueryAction` so record security locks apply.
 
+### Core search (`POST /qqq/v1/search`)
+
+QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v1/search` and used by the Next UI's global search box): a case-insensitive contains-query over each table's `searchFields`. With `serveCoreRecordSearch` (default `true`) the producer registers `QuickSearchRecordSearchProvider` on the instance, and core asks it to search every table Quick Search indexes. The API and the UI do not change.
+
+- **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quick_search_index`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
+- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights); each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order. Core re-reads them by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, and records deleted since they were indexed drop out.
+- **Failure.** The OpenSearch request is cancelled after `recordSearchTimeoutMillis` (default 2500). Any failure (timeout, cluster down, index missing) makes core search those tables with its own contains-query instead, so search degrades but never fails. After a failure, Quick Search claims no tables for 30 seconds, so a down cluster does not add a timeout to every search.
+- **Opting out.** `withServeCoreRecordSearch(false)` leaves core search and `searchFields` untouched. If the host already registered another record-search provider, the QBit keeps it and logs a warning (an instance has one provider).
+
+`QuickSearchAction` remains the richer API (scores, highlights, pagination, totals).
+
 ### Processes
 
 | Process | Trigger | What it does |
 |---|---|---|
-| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
+| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Recovers a full reindex that stopped without finishing, replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
 | Quick Search Reconcile Index | Cron via `reconcileCronExpression` when `schedulerName` is set; also on demand (optional `tableName` input) | Re-indexes everything in place, removes documents without a source row and documents of unconfigured tables |
-| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place |
+| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place. Fails to start while another full reindex is running |
 
 Without `schedulerName` the processes exist but nothing runs them; the QBit logs a warning at startup.
 
+### Managing searchable tables (data-driven, 1.1.0-RC.3)
+
+Which tables and fields are indexed is data: `quick_search_index` (one row per table) and `quick_search_index_field` (its fields: weight, role, analyzer, snippet and label flags), edited in the Quick Search admin app with the standard screens and these record actions (each with a review step; none is in the nav):
+
+- **Add Search Index** (Quick Search Index, Actions): pick a readable, eligible table, review the estimated record count and the fields it starts with, save; the first build is queued.
+- **Searchable Fields** on the index record: add, edit or remove fields; then **Apply Search Index Changes** previews the changes and rebuilds.
+- **Disable Search Index** and **Remove Search Index**: stop indexing at once and delete or keep the documents. Removals are remembered, so code defaults do not bring them back.
+- **Cancel Index Run** (Quick Search Index Run): stops a running reindex after its current batch (`CANCELLED`, with counts).
+- **Pause Indexing** and **Resume Indexing** (Quick Search Settings): pause all indexing while search keeps serving; changes are indexed after resume.
+
+`@QuickSearchable` entities and `searchableTables` only seed these records (self-priming at startup: new tables and fields are added and built automatically; rows an admin edited are never overwritten). The precedence rules, the DDL for the new tables and the upgrade steps are in `docs/MIGRATION-1.1.md`, "Data-driven configuration".
+
+### Scheduling
+
+The schedules run only when the host starts through QQQ's `QApplicationLauncher` (`com.kingsrook.qqq.middleware.javalin.QApplicationLauncher`, in `qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` (because the QBit's processes are scheduled), then the instance's runtime services. A host that starts only a server, or builds the `QInstance` itself, gets the processes but nothing runs them on a schedule, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
+
+Basepull's fixed interval runs on any QQQ scheduler. `reconcileCronExpression` needs a scheduler that supports cron, which in QQQ 4.1 means a `QuartzSchedulerMetaData`; a `SimpleSchedulerMetaData` runs fixed intervals only. The expression uses Quartz syntax, seconds first (`0 0 3 * * ?` is 03:00 every day), and needs `reconcileCronTimeZoneId`, a Java time zone ID such as `UTC` or `America/Chicago`; QQQ has no default zone. `produce()` rejects a cron without a zone, an invalid expression or zone, and a cron on a scheduler in the instance that does not support cron.
+
+```java
+qInstance.addScheduler(new QuartzSchedulerMetaData().withName("quartz").withProperties(quartzProperties));
+
+config.withSchedulerName("quartz")
+   .withReconcileCronExpression("0 0 3 * * ?")
+   .withReconcileCronTimeZoneId("America/Chicago");
+```
+
+### Diagnostic tools
+
+With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate the search and debug it from the admin UI. They are table and record actions (never left-nav entries), gated by permissions, recorded in `quick_search_diagnostic_event`, limited to `diagnosticRunsPerMinute` runs per user and tool, and read-only except Reindex in Search.
+
+| Tool | Where | What it does |
+|---|---|---|
+| Search Playground | Quick Search Index, Actions (select rows to pre-fill their tables) | Runs a term the way the global search does, as you: the path each table takes (OpenSearch provider, core database search, or not searched) and why; the provider's own request, timed; the global search results you would see; per indexed table, the production query with scores, highlights, matched fields, which hits you can read and which the global search shows, the query DSL and the raw response (hits you cannot read are redacted); the term's tokens; and `explain` for a hit (`table:id`). The form stays open, so you can change the term and run again. |
+| Validate Search Index | Quick Search Index, Actions; also scheduled by `validationCronExpression` | Cluster and index health, the alias, leftover physical indexes from interrupted rebuilds, the live mapping (version, field types, analyzers, the `fieldValues` template) and analysis settings, then per table: database rows vs documents, a sample of records checked present and current in the index, a random sample of documents checked to still have a record, the index row's status and last run, and waiting failed events. Each finding is OK, INFO, WARN or FAIL with what to do. The report is saved as a `quick_search_validation_run` row (HTML for the record view, JSON for tools). Database reads run as the system user so counts compare like with like; the report holds counts, ids and field names, never record values. One validation runs at a time per server. |
+| Test Search Analyzer | Quick Search Index, Actions | `_analyze` on the live index: the tokens of a text as indexed (edge n-grams of 2 to 20 characters) and as a query, and, given a term, whether every term token is among the text's indexed tokens. |
+| Inspect Search Document | Every indexed table's records (Actions), and Quick Search Failed Event rows | The record's document in the index next to the document indexing would write from the database now, field by field (current, stale or missing), with versions and the raw document. Enter a term to see why the record does or does not match it: term length, table enabled, read permission, document present and current, each term token found or not (and why: one character, longer than 20, not in the text, or only in the database's newer values), `explain`, and the record's rank against the number of ids the global search takes. |
+| Reindex in Search | Every indexed table's records, and Quick Search Failed Event rows | Rebuild the selected records' documents from the database now (versioned writes, so a newer document is never overwritten); a record that no longer exists has its document removed; a record you cannot read is skipped. Asks to confirm. Needs the admin permission. |
+| Search Diagnostics widget | The Quick Search Admin app home (hosts with their own home add `quickSearchDiagnostics`) | Whether the global search is served by OpenSearch on this server or backing off (and why), failure and fallback counts, the latest search failures from every server, and the latest validation run. |
+
+Search failures are recorded too: when the provider's OpenSearch request fails or times out (and core falls back to its own search), or `QuickSearchAction`'s search fails, a `SEARCH_FAILED` event keeps the tables, duration, error class and message, and the server. The user's term is never stored, only its length. At most 120 events are written per server per minute.
+
+Permissions: the read-only tools and the widget use `debugPermissionRules` (default `quick_search_debug.hasAccess`); Reindex in Search uses `adminPermissionRules` (`quick_search_admin.hasAccess`); the two history tables use `tablePermissionRules` (grant `.read` only: rows are written by the QBit). Impersonation ("search as user X") is deliberately not offered: a fabricated session would not carry the real user's security key values, so its results would mislead.
+
+### Record developer panel
+
+With QQQ 4.1.0-RC.4 or later the QBit registers a record developer panel provider (`QuickSearchDeveloperPanelProvider`, id `quickSearch`, in QQQ's `DeveloperPanelsMetaData`). Frontends that render developer panels (the Next UI, `/app/{table}/{id}/dev`) show a "Quick Search" panel on every record of an indexed table, to users who may read Quick Search Index. Every part is permission-checked:
+
+| Part | Shown when |
+|---|---|
+| Index: enabled, status, record count, last basepull, last full reindex, searchable fields, link to the index row | always (read on Quick Search Index) |
+| Latest index run: type, status, start and end, processed, indexed, errors | the user may read Quick Search Index Run |
+| Search document: verdict (current, stale, missing) and a Field / Database / Index / Match table | `enableDiagnosticTools` and the debug permission (`debugPermissionRules`); otherwise a note says why it is missing |
+| Failed index events of the record | the user may read Quick Search Failed Event and the record has any |
+| Actions: Inspect search document, Reindex this record | the record actions exist and the user may run them (debug and admin permission) |
+| Link to Quick Search Admin | the user may open the app |
+
+The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned off in Quick Search Index), or OpenSearch unavailable. When OpenSearch cannot be reached the panel still shows the database-side state, with a warning in place of the document comparison. Hosts without a developer-panel frontend are unaffected.
+
 ## Configuration reference
 
 | Field | Default | Description |
@@ -151,15 +221,24 @@ Without `schedulerName` the processes exist but nothing runs them; the QBit logs
 | `startupMode` | `FAIL_FAST` | `FAIL_FAST` fails `produce()` when the cluster is unreachable; `DEGRADED` boots and retries on first use |
 | `enableRealTimeIndexing` | `true` | Register the record-change listener |
 | `enableBasepullProcess`, `enableMaintenanceProcesses` | `true` | Register basepull; register full reindex and reconcile. `enableScheduledProcesses` is a deprecated alias for both |
-| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none | Scheduling |
+| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none, none | Scheduling; a cron needs the time zone ID and a Quartz scheduler, and nothing runs without `QApplicationLauncher` (see [Scheduling](#scheduling)) |
 | `basepullOverlapSeconds` | 300 | Re-read window before the previous run start |
 | `defaultBasepullIntervalMinutes` | 5 | Per-table due interval for config-driven tables |
 | `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
+| `fullReindexStaleMinutes` | 30 | A running full reindex that shows no sign of life for this long is treated as stopped: it no longer blocks a new one, and basepull recovers its tables. Keep it well above the time one page of `sourceBatchSize` records takes; null means the default |
 | `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
 | `maxFieldLength` | 10000 | Truncate each indexed value |
 | `maxSearchLimit` | 100 | Largest page size |
 | `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
-| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchAdmin` | Rules for the app and processes |
+| `serveCoreRecordSearch` | `true` | Register `QuickSearchRecordSearchProvider` so core `POST /qqq/v1/search` uses OpenSearch for indexed tables, and give those tables `searchFields` when they have none |
+| `recordSearchTimeoutMillis` | 2500 | Wall-clock limit on the OpenSearch request behind a core search; past it, core falls back to its own search |
+| `enableDiagnosticTools` | `true` | Add the diagnostic tools, the `quick_search_diagnostic_event` and `quick_search_validation_run` tables and the diagnostics widget |
+| `enableRecordDiagnosticActions` | `true` | With the tools on, add Inspect Search Document and Reindex in Search to every indexed table |
+| `validationCronExpression`, `validationCronTimeZoneId` | none | Schedule Validate Search Index; same rules as the reconcile schedule (a cron-capable scheduler such as Quartz, a valid Quartz expression, a time zone ID) |
+| `validationSampleSize` | 50 | Records and documents sampled per table by validation (at most 1000) |
+| `diagnosticRunsPerMinute` | 30 | Per-user, per-tool limit on diagnostic runs |
+| `debugPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quick_search_debug` | Rules for the read-only diagnostic tools and the widget |
+| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quick_search_admin` | Rules for the app, the maintenance processes and Reindex in Search |
 | `tablePermissionRules` | `READ_WRITE_PERMISSIONS` | Rules for the operational tables |
 | `tableMetaDataCustomizer` | | Applied to every produced table (backend details, for example) |
 | `searchableEntityClasses`, `searchableTables` | at least one | What to index |
@@ -169,9 +248,16 @@ Without `schedulerName` the processes exist but nothing runs them; the QBit logs
 
 ## Operational tables
 
-- `quickSearchIndex`: one row per table with `enabled` (honoured by indexing and search, cached for one minute), `basepullIntervalMinutes`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`, `NEEDS_REINDEX` when the configured fields changed since the row was created, or `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
-- `quickSearchIndexRun`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status`, counts, `errorMessage`.
-- `quickSearchFailedEvent`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. If a full reindex is killed mid-run, its tables stay `REBUILDING` and deletes keep collecting as `AWAITING_REINDEX`; no data is lost, and the recovery is to run a full reindex of all tables again, which applies the collected deletes and returns the tables to `ACTIVE`.
+- `quick_search_index`: one row per table with `source` (`CODE` or `OWNER`), `ownerEdited`, `enabled` (honoured by indexing and search; a change applies at once), `basepullIntervalMinutes`, `basepullTimestampField`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`; `PENDING_BUILD` when a build is queued and `BUILDING` while the basepull builds it; `NEEDS_REINDEX` when the fields changed since the last build; `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
+- `quick_search_index_field`: the searchable fields of each index row: `fieldName`, `weight`, `role` (`DISPLAY`, `TITLE`, `SUBTITLE`, `BODY`, `KEYWORD`, `EXACT`), `analyzer` (`PREFIX`, `WORD`, `ENGLISH`, `EXACT`; blank: by role), `includeInSnippet`, `includeLabel`, `sortOrder`, `source`, `ownerEdited`. Unique on (`quickSearchIndexId`, `fieldName`).
+- `quick_search_settings`: one row: `indexingPaused` (with `pausedAt`, `pausedBy`, `pauseReason`) and the last self-priming report (`lastPrimedAt`, `lastPrimingSummary`).
+- `quick_search_config_tombstone`: tables (blank `fieldName`) and fields an admin removed; priming never re-adds them.
+- `quick_search_index_run`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status` (`RUNNING`, `COMPLETED`, `FAILED`, `CANCELLED`), counts, `errorMessage`, `cancelRequested`, `cancelRequestedBy`.
+- `quick_search_diagnostic_event` (with `enableDiagnosticTools`): `eventType` (`SEARCH_FAILED`, `TOOL_RUN`, `REINDEX`, `RATE_LIMITED`), `source`, `outcome`, `userName`, `tableNames`, `searchTerm` (tool runs only), `durationMillis`, `errorClass`, `errorMessage` (text), `detail` (text), `hostName`, `createDate`. Purged after `runHistoryRetentionDays`.
+- `quick_search_validation_run` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
+- `quick_search_failed_event`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
+
+Real-time error counts on Quick Search Index show failures since the last successful full reindex or basepull. Each failed event increases the count; deferred changes while indexing is paused do not. Counter updates are best effort and serialized across listeners in one JVM; concurrent application nodes can still race because QQQ has no arithmetic update API.
 
 ## Extending
 
@@ -185,12 +271,78 @@ CI=true ./mvnw verify                       # plus Testcontainers integration te
 ./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
 ```
 
-Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.1 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
+Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.4 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
+
+## Upgrading from 1.0
+
+See [docs/MIGRATION-1.1.md](docs/MIGRATION-1.1.md). In short: 1.1 needs the QQQ release with the record-search provider SPI; core search starts using OpenSearch for indexed tables by default, and indexed tables gain `searchFields` (set `serveCoreRecordSearch(false)` to keep 1.0 behaviour).
 
 ## Upgrading from 0.x
 
-See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, and run a full reindex once so the index gets the new mapping.
+See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, grant `quick_search_admin.hasAccess` and the table permissions, add the new operational table columns and `quick_search_failed_event`, and run a full reindex once so the index gets the new mapping.
+
+## Adopting in another app
+
+Follow [Getting started](#getting-started), the [Configuration reference](#configuration-reference) and [Operational tables](#operational-tables) (the host application creates the three tables; this QBit ships no DDL), then run a full reindex. Hosts on 1.0 or 0.x should read the upgrade sections above.
+
+See also the [QQQ adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) for the full per-feature list (QQQ upgrade, Next dashboard, QBits, migrations, permissions).
 
 ## License
 
 Apache License, Version 2.0. See [LICENSE](LICENSE).
+
+## Status API for hosts
+
+Requires QQQ **4.1.0-RC.6**. Hosts should use the supported API rather than reach into
+`QuickSearchRuntime`, `QuickSearchOpenSearchClient`, diagnostic renderers, or qbit tables:
+
+```java
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatusService;
+
+// Runs in the host request's QContext, with the caller's QSession.
+QuickSearchStatus status = QuickSearchStatusService.get();
+String homeHref = status.homeHref();
+```
+
+UI clients use **`GET /qqq/v1/qbit/quick-search/status`**, operationId
+`getQuickSearchStatus`, extension `quick-search` version `1.0.0`. The endpoint returns
+that same immutable value as JSON; it requires the Quick Search app's configured
+`adminPermissionRules` (default `quick_search_admin.hasAccess`). Permission is checked
+before every read, including cache hits. No additional source-table permissions are
+required: like index validation, this is an administrator's aggregate view of the index.
+The committed contract is [quick-search.yaml](src/main/resources/openapi/qqq-v1/quick-search.yaml).
+
+The default core namespace is intentional: RC.6 permits `/qbit/quick-search` but rejects
+`/quick-search`, which is not an established legacy prefix. There were no existing
+Quick Search UI API routes to preserve. Core source is unchanged.
+
+- `state`: `OK`, `DEGRADED` (including paused indexing, backlog, incomplete diagnostics
+  or a refresh timeout), `DOWN` (unavailable health or red health), or `DISABLED`
+  (qbit absent or every persisted index disabled). Pausing indexing does not disable search.
+- `clusterReachable`, `clusterHealth`, `nodes`, `activeShards`, `unassignedShards` come
+  from the existing diagnostics health operation. Null means unknown/not probed.
+- `indexes` contains up to 100 persisted index rows. `documentCount` and `sourceCount`
+  come from the latest saved `IndexValidator` report, with `countsCheckedAt`. Without
+  matching validation, only the last indexing run's document count is available and
+  its observation time is unknown. These are **saved counts, not live counts**. A
+  missing source count must not be displayed as zero or “in sync.”
+- `pending` includes `PENDING` and `AWAITING_REINDEX` failed events; `exhausted` counts
+  failed events that gave up. At most 1000 backlog rows are read; `backlogTruncated`
+  makes both counts lower bounds. `tablesTruncated` indicates incomplete index rows.
+  These counters do not measure an external publisher's queue.
+- `lastSuccessfulSync` is the newest successful indexing watermark among the included
+  tables (basepull is start-anchored), not proof that all tables are current.
+  `lastError` is the newest available indexing/runtime search error or snapshot failure.
+- `checkedAt` is snapshot completion time. A per-runtime five-second cache bounds
+  polling cost; each refresh uses `DiagnosticsRateLimiter`'s per-user `STATUS` key.
+  Rate-limit refusal uses the existing diagnostics error (HTTP 400). A refresh waits
+  at most two seconds and retains at most one in-flight worker, even if a backend is
+  stuck. Runtime shutdown stops the worker. No source counts, record samples, full
+  validation, or per-table OpenSearch calls run on this polling path.
+
+Link to `status.homeHref()` instead of copying the Quick Search home. It includes the
+configured metadata prefix. `QuickSearchApi.APP_NAME` (`quickSearchAdmin`) and
+`QuickSearchApi.HOME_HREF` (`/app/quickSearchAdmin`) are available for default-name
+navigation. Keep detailed runs, failed events, validation and operator tools in the
+qbit's own home. [kof22 migration mapping](docs/STATUS-API-HOST-MIGRATION.md).

docs/MIGRATION-1.0.md

@@ -12,20 +12,84 @@
 1. **Remove any manual `QBitMetaData` registration.** The producer now registers `com.kingsrook.qbits:quick-search` itself; a second `addQBit` with that name throws at startup. (qqq-all's `DemoQBits` did this in 0.x.)
 2. **Decide the startup mode.** `startupMode` defaults to `FAIL_FAST`: `produce()` fails when OpenSearch is unreachable. Demo profiles that may run without a cluster should set `withStartupMode(QuickSearchStartupMode.DEGRADED)`.
 3. **Run a full reindex once after upgrading.** The index mapping changed (version 2: search analyzer, text-typed `fieldValues`). The QBit logs a warning when it finds an index with the old mapping; "Quick Search Full Reindex" with no table name builds a new physical index and swaps the configured name to an alias. Search keeps working during the rebuild. The identity the QBit uses needs index create, delete and alias privileges for this (see the README role).
-4. **Set `schedulerName`** (and optionally `basepullRepeatSeconds`, `reconcileCronExpression`) so basepull and reconcile actually run. In 0.x they were never scheduled.
-5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `allowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
-6. **Permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant them to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
+4. **Set `schedulerName` and start the host through `QApplicationLauncher`** (and optionally `basepullRepeatSeconds`, or `reconcileCronExpression` with `reconcileCronTimeZoneId` on a Quartz scheduler) so basepull and reconcile actually run. In 0.x they were never scheduled. See [Scheduling](#scheduling).
+5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `withAllowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
+6. **Grant the permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant the names listed under [Permissions](#permissions) to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
+7. **Update the operational tables before starting 1.0.** `quickSearchIndex` gains five columns, `searchableFieldsJson` changes from `STRING` to `TEXT`, and `tableName` becomes required with a unique key. `quickSearchIndexRun.errorMessage` changes from `STRING` to `TEXT`. There is a new `quickSearchFailedEvent` table. A host that checks its columns at startup rejects the 0.x tables. See [Schema changes](#schema-changes).
+
+## Permissions
+
+QQQ names a permission `{baseName}.{type}`. The admin rules use the base name `quickSearchAdmin`; the table rules have no base name, so each table's name is its base name. With the default rules, grant these to operators:
+
+| Permission | Covers |
+|---|---|
+| `quickSearchAdmin.hasAccess` | The `quickSearchAdmin` app and the three processes (`quickSearchBasepullIndex`, `quickSearchReconcileIndex`, `quickSearchFullReindex`) |
+| `quickSearchIndex.read`, `quickSearchIndex.write` | The `quickSearchIndex` table |
+| `quickSearchIndexRun.read`, `quickSearchIndexRun.write` | The `quickSearchIndexRun` table |
+| `quickSearchFailedEvent.read`, `quickSearchFailedEvent.write` | The `quickSearchFailedEvent` table |
+
+The tables use `READ_WRITE_PERMISSIONS`: `.read` covers query, get and count, and `.write` covers insert, edit and delete. If you set `tablePermissionRules` to `READ_INSERT_EDIT_DELETE_PERMISSIONS`, grant `.read`, `.insert`, `.edit` and `.delete` on each table instead. With `tableNamePrefix` set, the table permissions carry the prefix (`{prefix}quickSearchIndex.read`); the admin permission stays `quickSearchAdmin.hasAccess`.
+
+Users who only search need none of these. Search checks the read permission of each searched table as that table's own rules name it (for example `customer.read`). The QBit's own reads and writes of its tables go through QQQ's table actions, which do not check permissions, so real-time indexing does not depend on these grants.
+
+For a worked example, qqq-all's RC.2 adoption (QRun-IO/qqq-all#30) adds `infra/postgres/06-quick-search-permissions.sql`, which grants the seven names above to its admin role.
+
+## Schema changes
+
+The QBit declares its tables in metadata and ships no DDL. A backend that creates tables from metadata gets the 1.0 shape on a new database. A host that manages its own schema, or checks columns at startup, must change the 0.x tables before starting 1.0. Names below are QQQ field names and types; your backend's naming decides the physical table and column names.
+
+`quickSearchIndex`:
+
+| Field | Type | Change from 0.x |
+|---|---|---|
+| `lastReconcileTime` | `DATE_TIME` | New |
+| `lastRunStatus` | `STRING` | New. Run type and outcome, for example `FULL_REINDEX COMPLETED` (at most 22 characters) |
+| `lastErrorMessage` | `TEXT` | New. Up to 4,000 characters |
+| `documentCount` | `INTEGER` | New |
+| `realTimeErrorCount` | `INTEGER` | New |
+| `searchableFieldsJson` | `TEXT` | Was `STRING` |
+| `tableName` | `STRING` | Now required, with a unique key |
+
+`status` holds `ACTIVE`, `NEEDS_REINDEX` (new in 1.0) or `REBUILDING` (new in RC.2), so it needs at least 13 characters. QQQ checks the `tableName` unique key on insert; a unique index in the database also stops two concurrent first runs from both inserting. 0.x could leave duplicate rows, so keep one row per table before adding the index.
+
+`quickSearchIndexRun` has no new columns. `errorMessage` changed from `STRING` to `TEXT` and holds up to 4,000 characters.
+
+`quickSearchFailedEvent` is new:
+
+| Field | Type | Notes |
+|---|---|---|
+| `id` | `INTEGER` | Primary key, generated by the backend like the other two tables |
+| `tableName` | `STRING` | Source table |
+| `recordId` | `STRING` | Source record's primary key as text |
+| `action` | `STRING` | `INDEX` or `DELETE` |
+| `errorMessage` | `TEXT` | Up to 4,000 characters |
+| `attempts` | `INTEGER` | Replay attempts |
+| `status` | `STRING` | `PENDING`, `EXHAUSTED` or `AWAITING_REINDEX` (new in RC.2); size it for at least 16 characters |
+| `createDate`, `modifyDate` | `DATE_TIME` | |
+
+The QBit sets no maximum length on its `STRING` fields. Map `STRING` to a character column sized for the values above, `TEXT` to a type that holds at least 4,000 characters, and `DATE_TIME` to a timestamp, the same way your other QQQ tables do.
+
+If your backend creates tables from metadata and you do not need the 0.x run history, you can instead drop the 0.x `quickSearchIndex` and `quickSearchIndexRun` tables and let startup recreate them, as qqq-all#30 documents. The next basepull, reconcile or full reindex recreates each `quickSearchIndex` row, with `enabled` and `basepullIntervalMinutes` back at their configured defaults.
 
 ## Behaviour changes to be aware of
 
 - Real-time indexing happens after commit. Code that searched for a record inside the same transaction that inserted it will not find it until commit.
 - Real-time failures no longer vanish: they land in `quickSearchFailedEvent` and are replayed by the next basepull. Review that table after an OpenSearch outage.
-- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile.
+- Inserted and updated records are re-read inside the write's transaction, with display values and possible-value labels, before they are indexed. That is one extra query per write batch on an indexed table, and it runs the host's possible-value providers. If the re-read fails, the write still succeeds and the records are replayed by basepull.
+- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile. The QBit logs a warning for each such table at startup, for annotated and config-driven tables alike; a config-driven table with `basepullTimestampField` set to null has opted out and gets no warning.
 - `limit` is capped (default 100), `offset + limit` at 10,000, and search terms must be 2 to 100 characters.
 - Query semantics changed from OR over n-grams to AND over whole terms with prefix matching: "blue widget" now requires both words.
 - `enableScheduledProcesses` still works and sets both `enableBasepullProcess` and `enableMaintenanceProcesses`.
 - `QuickSearchQBitContext` is deprecated and removed in 1.1; use `QuickSearchRuntime.get()`.
 
+## Scheduling
+
+Setting `schedulerName` puts schedules on the basepull and reconcile processes; QQQ runs them only once something starts its `QScheduleManager`.
+
+- **Start through `QApplicationLauncher`.** In QQQ 4.1 that is `com.kingsrook.qqq.middleware.javalin.QApplicationLauncher` (`qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` when any process is scheduled, then the instance's runtime services, including this QBit's. A host that starts only a `QApplicationJavalinServer`, or builds its `QInstance` without the launcher, gets the processes but no scheduled runs, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
+- **A reconcile cron needs a Quartz scheduler.** Basepull's fixed interval runs on any QQQ scheduler, but only `QuartzSchedulerMetaData` supports cron; QQQ's `SimpleSchedulerMetaData` runs fixed intervals only, and a cron schedule on it fails when the schedule manager starts.
+- **A reconcile cron needs a time zone.** Set `reconcileCronTimeZoneId` to a Java time zone ID (`UTC`, `America/Chicago`). QQQ has no default zone: its instance validation rejects a cron schedule without one at startup. `produce()` now catches this first, along with an invalid Quartz expression (seconds first, for example `0 0 3 * * ?`), an unknown zone ID, and a cron on a scheduler in the instance that does not support cron.
+
 ## Removed
 
 - `QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`. Hosts that referenced them (for example to compose with their own customizers) should remove those references; the record-change listener coexists with host customizers.

docs/MIGRATION-1.1.md

@@ -0,0 +1,236 @@
+# Migrating from 1.0 to 1.1
+
+1.1 lets QQQ core's record search (`POST /qqq/v1/search`, the Next UI global search box) use the Quick Search index. Hosts that do nothing get it by default.
+
+## Prerequisites
+
+- The QQQ release that carries the record-search provider SPI (`RecordSearchProviderInterface`, `QInstance.withRecordSearchProvider`), first released in QQQ `4.1.0-RC.2`. 1.1 builds against `4.1.0-RC.2` through the `qqq-snapshot` profile.
+
+## What changes at startup
+
+1. **The QBit registers a record-search provider.** `QuickSearchRecordSearchProvider` becomes the instance's `recordSearchProvider`. If the host already registered a different provider, the QBit keeps it and logs a warning; core search then does not use OpenSearch.
+2. **Indexed tables get `searchFields`.** Core only searches tables with `searchFields`. Each indexed table that has none gets its visible string and integer searchable fields. Tables that already have `searchFields` keep them. A table added to the instance after the QBit is produced gets none, so add tables first (as for field validation) or set `searchFields` yourself.
+3. **Password fields are no longer indexed.** Fields of type `PASSWORD` are now dropped from the searchable fields like hidden fields. If a 1.0 index holds such values, run "Quick Search Reconcile Index" (or a full reindex) to rewrite the documents without them.
+
+## Behaviour to be aware of
+
+- Next UI search results for indexed tables now come from OpenSearch: all words must match (in any searchable field), with prefix matching and field weights, instead of a contains-match on one field. Tables not indexed by Quick Search keep core's search.
+- Results still pass core's security: core re-reads every hit through `QueryAction`, so table permissions and record security locks apply, and records deleted since they were indexed do not appear.
+- Records are only as fresh as the index: a record updated in a way that is not yet indexed (for example while real-time indexing is off) is found by its indexed text.
+- When OpenSearch is slow or down, core waits at most `recordSearchTimeoutMillis` (default 2500) and then searches with its own contains-query; for the next 30 seconds it skips OpenSearch entirely.
+
+## Opting out
+
+```java
+new QuickSearchQBitConfig()
+   // ...
+   .withServeCoreRecordSearch(false);
+```
+
+restores 1.0 behaviour: no provider, no `searchFields` changes.
+
+## Diagnostic tools (1.1.0-RC.2)
+
+The QBit now adds admin tools to validate and debug search (see the README, "Diagnostic tools"). They are on by default.
+
+- **Two new operational tables**, on `backendName` like the others: `quick_search_diagnostic_event` and `quick_search_validation_run` (columns in the README, "Operational tables"). Create them in the host's schema, with the naming the host applies to the other operational tables (`tableMetaDataCustomizer` applies to them too). Until they exist, writing events and saving reports fails quietly (logged at warn) and the tools still run.
+- **Permissions.** Grant `quick_search_debug.hasAccess` to the people who may use the read-only tools and see the widget, `quick_search_admin.hasAccess` for Reindex in Search, and `.read` on the two new tables (the QBit writes them; users never do).
+- **New processes** have a `tableName` (Quick Search Index, Quick Search Failed Event, and each indexed table), so they appear as table and record actions. They are not added to the admin app's children; a host that strips processes from its nav needs no change.
+- **Opting out.** `withEnableDiagnosticTools(false)` adds none of it; `withEnableRecordDiagnosticActions(false)` keeps the tools on Quick Search Index but adds nothing to the indexed tables.
+
+## Snake_case names (1.1.0-RC.3)
+
+Owner naming rule (2026-10-12): table names are lowercase snake_case, and a QBit's tables are prefixed with its full name. RC.3 renames the QQQ metadata names. The QBit ships no DDL, so nothing is dropped or rewritten; what you change depends on how your database tables are named today.
+
+| Kind | Before (RC.1, RC.2) | From RC.3 |
+|---|---|---|
+| Table | `quickSearchIndex` | `quick_search_index` |
+| Table | `quickSearchIndexRun` | `quick_search_index_run` |
+| Table | `quickSearchFailedEvent` | `quick_search_failed_event` |
+| Table | `quickSearchDiagnosticEvent` | `quick_search_diagnostic_event` |
+| Table | `quickSearchValidationRun` | `quick_search_validation_run` |
+| Possible-value source | `quickSearchIndex`, `quickSearchSearchableTable` | `quick_search_index`, `quick_search_searchable_table` |
+| Table permissions | `quickSearchIndex.read`, `quickSearchIndexRun.read`, ... | `quick_search_index.read`, `quick_search_index_run.read`, ... |
+| Admin permission | `quickSearchAdmin.hasAccess` | `quick_search_admin.hasAccess` |
+| Debug permission | `quickSearchDebug.hasAccess` | `quick_search_debug.hasAccess` |
+| Record actions on failed events | `quickSearchInspect_quickSearchFailedEvent`, `quickSearchReindexRecords_quickSearchFailedEvent` | `quickSearchInspect_quick_search_failed_event`, `quickSearchReindexRecords_quick_search_failed_event` |
+
+Unchanged: Java class names, field names (`quickSearchIndexId`, `tableName`), process, widget and app names (`quickSearchFullReindex`, `quickSearchDiagnostics`, `quickSearchAdmin` app), the OpenSearch index, and the developer panel id.
+
+**With a table-name prefix** (`withTableNamePrefix` or the producer namespace), table and possible-value source names are now the prefix in snake_case joined with `_`: prefix `myApp_` or `myApp` gives `my_app_quick_search_index`, where RC.2 gave `myApp_quickSearchIndex`. Process, widget and app names keep the prefix as given.
+
+**The old names stay available** as `@Deprecated` constants for upgrade scripts: `QuickSearchIndex.LEGACY_TABLE_NAME` (and the same on the other four entities), `QuickSearchSearchableTablePossibleValueSourceProducer.LEGACY_NAME`, and `QuickSearchQBitConfig.LEGACY_ADMIN_PERMISSION_BASE_NAME` / `LEGACY_DEBUG_PERMISSION_BASE_NAME`. They will be removed in 2.0.
+
+### Database tables
+
+- **Your tables are already `quick_search_*`** (for example you set the RDBMS backend name with `inferBackendName` or a table customizer): no DDL. You can drop the customizer, since the QQQ name now equals the database name.
+- **Your tables are camelCase** (`"quickSearchIndex"`): rename them, in a maintenance window, before starting RC.3. PostgreSQL example:
+
+```sql
+ALTER TABLE "quickSearchIndex"           RENAME TO quick_search_index;
+ALTER TABLE "quickSearchIndexRun"        RENAME TO quick_search_index_run;
+ALTER TABLE "quickSearchFailedEvent"     RENAME TO quick_search_failed_event;
+ALTER TABLE "quickSearchDiagnosticEvent" RENAME TO quick_search_diagnostic_event;
+ALTER TABLE "quickSearchValidationRun"   RENAME TO quick_search_validation_run;
+```
+
+  Rename the matching sequences, indexes and constraints the same way if your tooling depends on their names. Data is untouched. If you cannot rename yet, point each table at its old name with an `RDBMSTableBackendDetails` table name in a table customizer.
+
+### Stored references
+
+QQQ stores table, process and permission names in your own data. Update them in the same window (PostgreSQL examples; adjust schema and column names to your instance):
+
+```sql
+-- audit history: audit_table.name holds the QQQ table name
+UPDATE audit_table SET name = 'quick_search_index'            WHERE name = 'quickSearchIndex';
+UPDATE audit_table SET name = 'quick_search_index_run'        WHERE name = 'quickSearchIndexRun';
+UPDATE audit_table SET name = 'quick_search_failed_event'     WHERE name = 'quickSearchFailedEvent';
+UPDATE audit_table SET name = 'quick_search_diagnostic_event' WHERE name = 'quickSearchDiagnosticEvent';
+UPDATE audit_table SET name = 'quick_search_validation_run'   WHERE name = 'quickSearchValidationRun';
+
+-- saved views and saved reports
+UPDATE saved_view   SET table_name = 'quick_search_index' WHERE table_name = 'quickSearchIndex';
+UPDATE saved_report SET table_name = 'quick_search_index' WHERE table_name = 'quickSearchIndex';
+-- ... and the same for the other four tables
+```
+
+- **Permission grants.** Wherever your roles live (QQQ role tables, an identity provider's group attributes, code), replace `quickSearchAdmin.hasAccess` with `quick_search_admin.hasAccess`, `quickSearchDebug.hasAccess` with `quick_search_debug.hasAccess`, and `quickSearchIndex.read` / `.edit` (and the other tables' permissions) with the `quick_search_*` names. Users without the new grants lose the admin app and tools until they are granted.
+- **Bookmarks and links** to `/app/quickSearchIndex/...` move to `/app/quick_search_index/...`.
+- **Index rows and failed events** store *your* table names (`quick_search_index.table_name`, `quick_search_failed_event.table_name`), not the QBit's, so they need no change for this rename. The OpenSearch documents are keyed by your tables too, so **no reindex is needed**. If you rename your own tables at the same time, update those two columns and run a full reindex.
+
+### Checklist
+
+1. Back up the database.
+2. Rename the tables (only if they are camelCase) and update the stored references above.
+3. Update permission grants.
+4. Upgrade to RC.3 and start; `startupMode` `FAIL_FAST` reports any table the backend cannot find.
+5. Open Admin › Quick Search and run Validate Search Index.
+
+## Data-driven configuration (1.1.0-RC.3)
+
+From RC.3, which tables and fields Quick Search indexes is **data**: standard QQQ tables an admin edits, secured by permissions and audited. Code (`@QuickSearchable` / `@QuickSearchField` entities and `searchableTables`) still works, but it only **seeds** the records. Changes take effect without a redeploy.
+
+### New tables and columns
+
+| Table | One row per | Notes |
+|---|---|---|
+| `quick_search_index` (existing) | searchable table | New columns `source` (`CODE` or `OWNER`) and `owner_edited`. `basepull_timestamp_field` is now editable. The generic Create and Delete screens are off: use Add Search Index and Remove Search Index. |
+| `quick_search_index_field` | searchable field of an index | `field_name`, `weight` (1-1000), `role` (`DISPLAY` T1 display, `TITLE`, `SUBTITLE`, `BODY`, `KEYWORD`, `EXACT`), `analyzer` (blank, `PREFIX`, `WORD`, `ENGLISH`, `EXACT`), `include_in_snippet` (blank: by role), `include_label`, `sort_order`, `source`, `owner_edited`. Unique on (`quick_search_index_id`, `field_name`). Edited from the index record's Searchable Fields list. |
+| `quick_search_settings` | one row (id 1) | The global pause (`indexing_paused`, `paused_at`, `paused_by`, `pause_reason`) and the last self-priming report. Created by the QBit. |
+| `quick_search_config_tombstone` | table or field an admin removed | Priming never adds back what a tombstone names. Delete the tombstone to let code seed it again. |
+| `quick_search_index_run` (existing) | run | New columns `cancel_requested` and `cancel_requested_by` (Cancel Index Run). |
+
+The config tables are audited at field level (QQQ's `audit` tables; without them the audit is skipped with a log line) and use `tablePermissionRules` (by default `quick_search_index_field.read`, `.edit`, and so on). The new processes use `adminPermissionRules` (`quick_search_admin.hasAccess`).
+
+### Precedence: code versus data
+
+1. **The records are the source of truth** once the config tables exist and priming has run. Indexing (real-time, basepull, reconcile, full reindex), search, the core record-search provider and the diagnostic tools read the effective config from them. It is cached for 30 seconds and dropped at once on the server where a config row changes, so an edit takes effect immediately there and within 30 seconds on other servers.
+2. **Code seeds; it never overrides an owner.** At startup (and on each basepull until it succeeds), priming:
+   - adds a row (source `CODE`, `enabled` per `enabledByDefault`) and its field rows for every code-declared table that has none, and queues its first build;
+   - updates the interval and timestamp field of rows nobody edited (`owner_edited` false); `enabled` is set only when a row is created;
+   - adds code-declared fields that have no row, and updates the weight and label flag of field rows nobody edited; either change queues a rebuild of that table;
+   - never touches a row or field an admin added or edited (`owner_edited` true), and never re-adds anything a tombstone names;
+   - never deletes anything: removing an annotation leaves the rows for the admin to remove.
+3. **Removals are remembered.** Remove Search Index, and deleting a field row, write a tombstone, so code does not bring them back.
+4. **Without the config tables** (before you add the DDL below), or until priming succeeds, Quick Search uses the code-declared tables exactly as before RC.3. So the upgrade can be staged.
+
+### Self-priming
+
+- A new code-declared table gets its rows on startup and is built by the next basepull (status `PENDING_BUILD`, then `BUILDING`, then `ACTIVE`; the run is recorded as `FULL_REINDEX`). No manual step is needed.
+- A new install, or an index that was deleted, is recreated by the basepull, and the tables that had documents in it are rebuilt. An index that is empty while a table last counted documents in it is rebuilt too.
+- The Search Diagnostics widget on the Quick Search home reports it ("New: 2 tables added and building (...)"), and `QuickSearchStatus.read(runtime).bannerHtml()` gives the same lines for a host's own home widget.
+- Existing rows from RC.2 or earlier get their field rows on the first start, with **no rebuild** when the index was built with exactly the code's fields.
+
+### Admin actions
+
+All are standard streamed-ETL processes with a review step and a result step, run from the table's Actions (never from the nav):
+
+| Action | On | What it does |
+|---|---|---|
+| Add Search Index | Quick Search Index (no selection) | Pick a table you can read that is not indexed and is not internal or secret; set the interval, timestamp field and enabled flag. The review shows the estimated record count, the fields it starts with (record-label fields as titles, then text fields) and what is never indexed (hidden and `isSensitive` fields). Saving queues the first build (or runs it at once when no scheduler runs the basepull). |
+| Searchable Fields (child list on the index record) | Quick Search Index Field | Add, edit or remove fields with the standard screens; a field must exist, be indexable and not sensitive, and not be listed twice. The change applies to search and new writes at once and marks the index `NEEDS_REINDEX`. |
+| Apply Search Index Changes | Quick Search Index (selected) | The review previews the field changes since the last build and the records the rebuild reads; executing rebuilds the table. |
+| Disable Search Index, Remove Search Index | Quick Search Index (selected) | Stop indexing at once. The input step's "Search documents" choice (delete now, or keep) is restated on the review step. Remove also deletes the field rows and pending failed events and leaves a tombstone. Enabling a disabled index again (edit Enabled) queues a build. |
+| Cancel Index Run | Quick Search Index Run (selected `RUNNING` runs) | Sets a cooperative cancel flag that the run checks between batches; it ends `CANCELLED` with its counts. A cancelled all-tables rebuild keeps the previous index; a cancelled queued build waits for Apply Search Index Changes. |
+| Pause Indexing, Resume Indexing | Quick Search Settings | Pauses all indexing (real-time and scheduled; a running rebuild stops after its batch) while search keeps serving the current index. Changes made while paused are kept (`DEFERRED` failed events) and indexed by the basepull after Resume. The home shows "Paused since ... by ...". |
+
+Roles and analyzers: `TITLE` fields make the hit's label, `SUBTITLE` its subtitle (`QuickSearchResult.subtitle`), `DISPLAY` fields its short details (`QuickSearchResult.displayValues`); `KEYWORD` and `EXACT` fields stay out of the highlighted snippet and match on their own. The `WORD`, `ENGLISH` and `EXACT` analyzers use new document objects (`fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`); their mapping is added to an existing index at startup (no mapping-version change), and documents get them as they are rebuilt. Tables of plain fields produce the same documents and queries as before.
+
+### DDL
+
+The QBit ships no DDL. PostgreSQL, with the snake_case column names QQQ infers (`QInstanceEnricher.setInferredFieldBackendNames`); use the field names as written if your backend maps fields to camelCase columns. Liquibase formatted SQL:
+
+```sql
+-- liquibase formatted sql
+
+-- changeset quick-search:1.1.0-RC.3-1
+ALTER TABLE quick_search_index ADD COLUMN IF NOT EXISTS source VARCHAR(20);
+ALTER TABLE quick_search_index ADD COLUMN IF NOT EXISTS owner_edited BOOLEAN;
+ALTER TABLE quick_search_index_run ADD COLUMN IF NOT EXISTS cancel_requested BOOLEAN;
+ALTER TABLE quick_search_index_run ADD COLUMN IF NOT EXISTS cancel_requested_by VARCHAR(250);
+
+-- changeset quick-search:1.1.0-RC.3-2
+CREATE TABLE IF NOT EXISTS quick_search_index_field (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   quick_search_index_id INTEGER NOT NULL,
+   field_name VARCHAR(255) NOT NULL,
+   weight INTEGER,
+   role VARCHAR(20),
+   analyzer VARCHAR(20),
+   include_in_snippet BOOLEAN,
+   include_label BOOLEAN,
+   sort_order INTEGER,
+   source VARCHAR(20),
+   owner_edited BOOLEAN,
+   create_date TIMESTAMP WITHOUT TIME ZONE,
+   modify_date TIMESTAMP WITHOUT TIME ZONE,
+   CONSTRAINT pk_quick_search_index_field PRIMARY KEY (id),
+   CONSTRAINT uq_quick_search_index_field UNIQUE (quick_search_index_id, field_name)
+);
+
+-- changeset quick-search:1.1.0-RC.3-3
+CREATE TABLE IF NOT EXISTS quick_search_settings (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   indexing_paused BOOLEAN,
+   paused_at TIMESTAMP WITHOUT TIME ZONE,
+   paused_by VARCHAR(250),
+   pause_reason VARCHAR(500),
+   last_primed_at TIMESTAMP WITHOUT TIME ZONE,
+   last_priming_summary TEXT,
+   last_priming_new_tables INTEGER,
+   create_date TIMESTAMP WITHOUT TIME ZONE,
+   modify_date TIMESTAMP WITHOUT TIME ZONE,
+   CONSTRAINT pk_quick_search_settings PRIMARY KEY (id)
+);
+
+-- changeset quick-search:1.1.0-RC.3-4
+CREATE TABLE IF NOT EXISTS quick_search_config_tombstone (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   table_name VARCHAR(255) NOT NULL,
+   field_name VARCHAR(255),
+   removed_by VARCHAR(250),
+   reason VARCHAR(500),
+   create_date TIMESTAMP WITHOUT TIME ZONE,
+   CONSTRAINT pk_quick_search_config_tombstone PRIMARY KEY (id)
+);
+CREATE INDEX IF NOT EXISTS idx_quick_search_config_tombstone_table ON quick_search_config_tombstone (table_name);
+```
+
+MySQL: use `INT AUTO_INCREMENT` for the ids, `DATETIME` for the timestamps and `TINYINT(1)` for the booleans, and drop `IF NOT EXISTS` from `ADD COLUMN`.
+
+### Behaviour changes to be aware of
+
+- `quick_search_index` no longer offers Create or Delete (`TABLE_INSERT` and `TABLE_DELETE` are off), so the API cannot insert or delete rows either. Rows come from priming and Add Search Index, and go through Remove Search Index.
+- Basepull builds queued tables (`PENDING_BUILD`) itself and recreates a missing index. `NEEDS_REINDEX` still waits for an admin (Apply Search Index Changes, or Rebuild).
+- With the config tables, a table that is not in the instance is not part of the effective config, and `ensureIndexRowExists` no longer creates rows (priming does, honouring tombstones).
+- Hidden and `isSensitive` fields are never indexed, whether declared in code or added as rows.
+- `quick_search_index_run` and `quick_search_failed_event` offer no Create or Edit (Delete stays). The Basepull, Full Reindex and Reconcile processes are table actions on Quick Search Index rather than app children; their table input is now `searchTableName` (API callers may keep sending `tableName`).
+- `quick_search_searchable_table` is now a custom possible-value source over the effective tables (it was an enum of the code-declared ones).
+- Tables added at runtime get core `searchFields` (for the global search) when they have none, but the per-table Inspect Search Document and Reindex in Search record actions appear only after the next restart.
+
+### Checklist for data-driven configuration
+
+1. Add the DDL above (it is additive).
+2. Grant `quick_search_index_field`, `quick_search_settings` and `quick_search_config_tombstone` permissions (read, and edit where admins should change them) next to the existing Quick Search grants.
+3. Upgrade and start. The Search Diagnostics widget should not say the searchable tables come from code, and shows the first priming's summary.
+4. Optionally move code-declared search config into the records and delete the annotations: the rows stay and are now the owner's.

docs/SESSION-STATE.md

@@ -1,6 +1,25 @@
 # Session State
 
-Updated: 2026-10-06
+Updated: 2026-10-14
+
+## Real-time error counter fix (2026-10-14)
+
+- Failed-event recording now increments the table's `realTimeErrorCount` after commit, using QQQ reads and system writes. Null counts start at zero; update failures are logged and cannot fail the source write or prevent replay.
+- QQQ has no arithmetic update API. The read-modify-write fallback is serialized across listeners in this JVM; separate application nodes can still race.
+- Successful basepull and full-reindex status updates reset the count, including the single-table reindex path. Investigation found the previous zero assignments only initialized new rows. Field help, README and Unreleased changelog now describe the reset window.
+- Regression coverage includes event counts, concurrent listeners, counter-update failure, after-commit timing, basepull reset and both full-reindex reset paths.
+- Final gate: `./mvnw.cmd -o -B verify` passed against released QQQ 4.1.0-RC.6: 476 unit tests, zero failures/errors; 41 integration tests skipped because Docker is unavailable; architecture guardrails and JaCoCo coverage gates passed. No dependency changes or offline artifact gaps. Checkstyle, PMD and SpotBugs are not configured in this Maven build.
+- No commit, push or branch creation. Second-brain save is blocked by the vault being outside writable roots; this section retains the durable handoff for later vault synchronization.
+
+## Status API worktree (2026-10-13, issue #50 / F10)
+
+- Implemented `api.QuickSearchStatusService.get()`, immutable status values, default app/home constants and a spec-managed status endpoint. Admin permission is checked on every call; saved validation counts avoid source scans. Five-second per-runtime cache, two-second wait, one in-flight worker, existing diagnostics rate limiter, bounded table/backlog reads.
+- Endpoint uses core's permitted `/qqq/v1/qbit/quick-search/status` namespace (`getQuickSearchStatus`). Requested `/quick-search` is rejected by RC.6's established-prefix allowlist. Core and host clones are read-only; no edits there. No previous Quick Search HTTP operations or Next UI callers found.
+- POM targets released-coordinate `4.1.0-RC.6`; local verification uses only `-Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT`.
+- Host file/line replacement mapping: `docs/STATUS-API-HOST-MIGRATION.md`. Contract: `src/main/resources/openapi/qqq-v1/quick-search.yaml`.
+- Verification: 468 unit tests (including strict HTTP, contract, bounds, cache isolation, permissions and timeout recovery); JaCoCo passes. All 41 Docker integration tests skip because Docker is unavailable. Final command: `./mvnw.cmd -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT`.
+- Standalone repository Checkstyle is not green: HEAD has 471 main-source violations; this tree has 438, with zero newly introduced violations. New production files and both new test files pass their style checks. Existing header/import/style cleanup is outside the status API change.
+- No commit, push, merge or deployment authorized. Second-brain vault is outside writable roots; durable notes are retained here for later vault synchronization.
 
 ## Context
 

docs/STATUS-API-HOST-MIGRATION.md

@@ -0,0 +1,56 @@
+# Quick Search host migration (kof22 #164)
+
+Issue #50 / architecture finding F10. Paths below are under
+`Website-Backend/src/main/java/com/kof22/website/metadata/adminhome/`;
+line numbers refer to the local clone inspected on 2026-10-13. No host files were changed.
+
+| Existing host code | Replacement |
+| --- | --- |
+| `LiveAdminHomeProbes.java:340` `searchConfigured()` | Use `QuickSearchApi.APP_NAME` for the default-name app lookup; for an authorized snapshot use `QuickSearchStatusService.get()` and its state/homeHref. `DISABLED` also includes all indexes disabled, so it is not exactly “qbit absent.” |
+| `LiveAdminHomeProbes.java:351` `searchCluster()` | One `QuickSearchStatusService.get()`; map `clusterReachable`, `clusterHealth`, `nodes`, `activeShards`, `unassignedShards`. Remove direct cluster HTTP/auth/CA handling. |
+| `LiveAdminHomeProbes.java:386` `searchDocumentCounts(...)` | Reuse the same snapshot's `indexes()` and each row's `tableName()` / `documentCount()` / `sourceCount()` / `countsCheckedAt()`. Remove `QuickSearchOpenSearchClient` access and relabel counts as saved diagnostics, not live. Preserve null as unknown. |
+| `QuickSearchHomeWidgets.java:118` `widgets()` | Stop adding the host's duplicate Quick Search home widgets. Link to `status.homeHref()` (or default `QuickSearchApi.HOME_HREF`). Keep the qbit-provided home/diagnostics widget. |
+| `QuickSearchHomeWidgets.java:133` `indexFacts()`, `:160` `summary()` and `:196` `syncTile()` | For a compact host overview, use the same `QuickSearchStatusService.get()` value: `state`, `indexes`, `pending`, `exhausted`, `lastSuccessfulSync`, `lastError`. Remove SQL aggregates and live source counts; use saved count timestamps and truncation flags. |
+| `QuickSearchHomeWidgets.java:267` `runs()`, `:315` `failedEvents()` | Replace duplicate detailed views with the home link; drill down using the qbit's own Index Run / Failed Event tables from that home. The summary API intentionally does not export detailed histories. |
+| `AdminOperationsWidgets.java:209` `quickSearch()` | Render `status.indexes()` for the compact admin overview, and use `status.homeHref()` for its footer link. Remove qbit-table SQL and source-table counts. |
+| `AdminOperationsWidgets.java:343` `searchFooter()` | Use `clusterHealth`, `pending`, `exhausted`, `lastSuccessfulSync`, `lastError`, and `checkedAt` from the same snapshot. Detailed recent-run information belongs in the qbit home. |
+
+Call with the existing request's `QContext`/`QSession`; do not elevate a host user's
+session to bypass `adminPermissionRules`. Cache hits still check permission.
+
+## UI contract
+
+| Method | Path | operationId |
+| --- | --- | --- |
+| GET | `/qqq/v1/qbit/quick-search/status` | `getQuickSearchStatus` |
+
+Extension: `quick-search`, version `1.0.0`; core dependency: `4.1.0-RC.6`.
+The requested `/quick-search` prefix cannot validate on the supplied RC.6 baseline:
+`MiddlewareSpecExtensionsMetaData.ESTABLISHED_PREFIXES` only admits esb, alerts,
+assistant, and sql-workbench. We use the default namespace without modifying core.
+
+Wire changes against develop: one additive endpoint and its schemas; no existing
+method, path, request, or response changed. Next UI `src/` contains no calls to a
+Quick Search-specific endpoint today; its core `/qqq/v1/search` integration is unchanged.
+Vendor `src/main/resources/openapi/qqq-v1/quick-search.yaml` for clients.
+
+Regenerate the fragment with:
+
+```powershell
+./mvnw.cmd -B test -Dtest=QuickSearchApiContractTest -Dquicksearch.openapi.update=true -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT
+```
+
+The local verification override is never the committed dependency version.
+
+## Verification and limits
+
+`./mvnw.cmd -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT` passes
+468 unit tests and JaCoCo, including strict HTTP server startup, permission checks,
+OpenAPI contract consistency, cached/disabled/degraded/down states, timeout recovery,
+bounded backlog/table lists and prefixed QInstance isolation. All 41 Docker integration
+tests skip locally because Docker is unavailable.
+
+The separate Checkstyle check is **not green**: the existing checked-in rules report
+471 main-source violations at HEAD and 438 in this worktree, with zero new violations.
+New source files and the two new tests pass those rules. No repository-wide header
+rewrite or unrelated style cleanup was included.

docs/TODO.md

@@ -1,5 +1,18 @@
 # TODO
 
+## Real-time error counter
+
+- [x] Count failed real-time events after commit, serialize concurrent listeners within a JVM, tolerate counter-write failures, and reset after successful basepull/full reindex; add regression tests and field/documentation help.
+
+## Architecture status API (#50 / F10)
+
+- [x] Supported Java status API, bounded cache/rate limiting, administrator authorization, home navigation, spec endpoint and committed fragment.
+- [x] Contract and strict HTTP tests; host migration mapping for kof22#164.
+- [ ] kof22#164: replace internal probes/duplicate home using `docs/STATUS-API-HOST-MIGRATION.md`.
+- [ ] Next UI: vendor the additive Quick Search status fragment when adopting RC.6.
+- [ ] Run Docker integration tests in CI; local Docker is unavailable.
+- [ ] Resolve the pre-existing repository Checkstyle baseline (438 remaining main-source violations, down from 471 at HEAD); status API adds none.
+
 ## Before tagging 1.0.0
 
 - [ ] qqq-all and Voyage adopt `1.0.0-RC.2` (items below) and report back.
@@ -9,7 +22,7 @@
 - [ ] qqq-all follow-up PR: remove the manual `instance.addQBit(...)` for quick-search in `DemoQBits`, pass `withSchedulerName("demoScheduler")`, set `withStartupMode(QuickSearchStartupMode.DEGRADED)` for the full profile test, pin 1.0.0.
 - [ ] Voyage: pin 1.0.0, set `schedulerName`, switch to `opensearchUrl` and `${env.}` credentials.
 - [ ] CI: add a second test job with `-Dopensearch.test.image=opensearchproject/opensearch:3.9.0` (needs a qqq-orb parameter for extra Maven args, or a plain job).
-- [ ] Open the qqq issue proposing `RecordSearchProviderInterface` (D1); quick-search adapter lands in 1.1.
+- [x] Core SPI proposed as QRun-IO/qqq#1042 (D1); the 1.1 adapter `QuickSearchRecordSearchProvider` is on `feature/record-search-provider`.
 - [x] Tracking issue #12 created and recorded in the PRD (2026-10-06).
 - [x] PR #8 merged 2026-10-05; Dependabot PR #1 closed as superseded.
 - [x] PR #10 merged and `1.0.0-RC.1` published to Maven Central from `release/1.0` (2026-10-05).
@@ -26,6 +39,11 @@
 - [ ] Stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review): only another all-tables reindex clears it, `AWAITING_REINDEX` rows accumulate and drift detection stays off for those tables. No data is lost. Fix needs a liveness check on the `FULL_REINDEX` run record before reconcile may reset the status, plus refusing a second concurrent full reindex.
 - [ ] Drift detection still writes `NEEDS_REINDEX` into `status`, so a few-millisecond window can overwrite `REBUILDING`; the durable fix is a separate drift field (schema change, so 1.1 at the earliest).
 
+## Before releasing 1.1.0
+
+- [x] Re-pointed the `qqq-snapshot` profile from `4.1.0-SNAPSHOT` to `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI (2026-10-11).
+- [ ] qqq-all / Website-Backend: confirm the Next UI search box is served from OpenSearch (provider registered, indexed tables have `searchFields`).
+
 ## Later (1.x)
 
 - [ ] Health widget for the admin app (data exists: `isHealthy`, `documentCount`, `lastRunStatus`).

docs/initiatives/quick-search-1.0.md

@@ -2,7 +2,7 @@
 tracker_key: "QRun-IO/qbit-quick-search#12"
 project: qbit-quick-search
 status: released-rc
-updated: 2026-10-06
+updated: 2026-10-09
 ---
 
 # Initiative: QBit Quick Search 1.0 on QQQ 4.1
@@ -11,7 +11,7 @@ updated: 2026-10-06
 
 ## Status
 
-`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the review follow-ups #13 to #17, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
+`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). `1.0.0-RC.2` followed on 2026-10-06 with the review follow-ups #13 to #17. Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
 
 ## Executive Summary
 

pom.xml

@@ -18,7 +18,9 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
-      <revision>1.0.0-SNAPSHOT</revision>
+      <qbit.guardrails.enforce>true</qbit.guardrails.enforce>
+      <qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
+      <revision>1.1.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
       <opensearch.version>3.10.0</opensearch.version>
@@ -30,7 +32,7 @@
    <parent>
       <groupId>com.kingsrook</groupId>
       <artifactId>qbit-build-parent</artifactId>
-      <version>2.0.0</version>
+      <version>2.1.0</version>
    </parent>
 
    <groupId>com.kingsrook.qbits</groupId>
@@ -70,6 +72,11 @@
    </scm>
 
    <dependencies>
+      <dependency>
+         <groupId>com.kingsrook.qqq</groupId>
+         <artifactId>qqq-middleware-javalin</artifactId>
+      </dependency>
+
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
@@ -264,13 +271,17 @@
       <!-- qqq-bom-pom 4.1.0-RC.1 ahead of the parent's 4.0.0 BOM. Override with    -->
       <!-- -Dqqq.snapshot.version=... ; deactivate with -P!qqq-snapshot. Remove the -->
       <!-- activation when the parent moves to 4.1 (ADR-0007).                      -->
+      <!-- 1.1: the record-search provider needs the core SPI                       -->
+      <!-- (RecordSearchProviderInterface), first released in qqq 4.1.0-RC.2.       -->
+      <!-- The record developer panel provider needs the developer panel         -->
+      <!-- contract (DeveloperPanelProviderInterface), first in qqq 4.1.0-RC.4.   -->
       <profile>
          <id>qqq-snapshot</id>
          <activation>
             <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
-            <qqq.snapshot.version>4.1.0-RC.1</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.6</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfig.java

@@ -18,10 +18,13 @@ package com.kingsrook.qbits.quicksearch;
 
 
 import java.net.URI;
+import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.TimeZone;
 import java.util.regex.Pattern;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -30,12 +33,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
+import org.quartz.CronExpression;
 
 
 /*******************************************************************************
@@ -52,14 +57,39 @@ import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
  ** enableMaintenanceProcesses, schedulerName plus basepull/reconcile schedules,
  ** permission rules for the admin surface, search bounds, and indexing limits.
  **
+ ** Core search: serveCoreRecordSearch (default true) registers
+ ** QuickSearchRecordSearchProvider, so QQQ's POST /qqq/v1/search (the Next UI
+ ** global search) is served from OpenSearch for indexed tables, bounded by
+ ** recordSearchTimeoutMillis.
+ **
  ** Tables: searchableEntityClasses (annotated entities) and/or searchableTables
  ** (config-driven). validate() is list-based and never throws; the producer
  ** fails production when it reports errors.
  *******************************************************************************/
 public class QuickSearchQBitConfig implements QBitConfig
 {
-   public static final String DEFAULT_ADMIN_PERMISSION_BASE_NAME = "quickSearchAdmin";
-   public static final String DEFAULT_BASEPULL_TIMESTAMP_FIELD   = "modifyDate";
+   public static final String DEFAULT_ADMIN_PERMISSION_BASE_NAME   = "quick_search_admin";
+   public static final String DEFAULT_BASEPULL_TIMESTAMP_FIELD     = "modifyDate";
+   public static final int    DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS = 2500;
+   public static final String DEFAULT_DEBUG_PERMISSION_BASE_NAME   = "quick_search_debug";
+   /** Admin permission base name before 1.1.0-RC.3 (camelCase). */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_ADMIN_PERMISSION_BASE_NAME = "quickSearchAdmin";
+   /** Debug permission base name before 1.1.0-RC.3 (camelCase). */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_DEBUG_PERMISSION_BASE_NAME = "quickSearchDebug";
+   public static final int    DEFAULT_VALIDATION_SAMPLE_SIZE       = 50;
+   public static final int    MAX_VALIDATION_SAMPLE_SIZE           = 1000;
+   public static final int    DEFAULT_DIAGNOSTIC_RUNS_PER_MINUTE   = 30;
+
+   ///////////////////////////////////////////////////////////////////////////////
+   // a live full reindex refreshes its run record after every page; a page is  //
+   // one source query plus sourceBatchSize / bulkBatchSize bulk requests, each //
+   // bounded by the transport timeouts (a few minutes at most by default),     //
+   // so 30 minutes leaves a wide margin before a slow run is judged dead, and  //
+   // basepull (every 5 minutes by default) recovers a dead one soon after      //
+   ///////////////////////////////////////////////////////////////////////////////
+   public static final Integer DEFAULT_FULL_REINDEX_STALE_MINUTES = 30;
 
    private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^[a-z0-9][a-z0-9_+.-]*$");
    private static final Set<String> AWS_SERVICE_NAMES = Set.of("es", "aoss");
@@ -95,17 +125,27 @@ public class QuickSearchQBitConfig implements QBitConfig
    private Integer basepullOverlapSeconds = 300;
    private Integer defaultBasepullIntervalMinutes = 5;
    private Integer runHistoryRetentionDays = 30;
+   private Integer fullReindexStaleMinutes = DEFAULT_FULL_REINDEX_STALE_MINUTES;
    private Integer bulkBatchSize = 500;
    private Integer sourceBatchSize = 1000;
    private Integer maxFieldLength = 10000;
    private Integer maxSearchLimit = 100;
    private Boolean applyRecordSecurityLocks = true;
+   private Boolean serveCoreRecordSearch = true;
+   private Integer recordSearchTimeoutMillis = DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS;
+   private Boolean enableDiagnosticTools = true;
+   private Boolean enableRecordDiagnosticActions = true;
+   private String validationCronExpression;
+   private String validationCronTimeZoneId;
+   private Integer validationSampleSize = DEFAULT_VALIDATION_SAMPLE_SIZE;
+   private Integer diagnosticRunsPerMinute = DEFAULT_DIAGNOSTIC_RUNS_PER_MINUTE;
    private List<Class<?>> searchableEntityClasses;
    private List<SearchableTableConfig> searchableTables;
 
    private transient IndexEventPublisher indexEventPublisher;
    private transient QPermissionRules adminPermissionRules = new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(DEFAULT_ADMIN_PERMISSION_BASE_NAME);
    private transient QPermissionRules tablePermissionRules = new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS);
+   private transient QPermissionRules debugPermissionRules = new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(DEFAULT_DEBUG_PERMISSION_BASE_NAME);
    private transient MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer;
    private transient QuickSearchRuntime runtime;
 
@@ -387,8 +427,17 @@ public class QuickSearchQBitConfig implements QBitConfig
       requirePositive(maxBulkRequestBytes, "maxBulkRequestBytes", errors);
       requirePositive(basepullRepeatSeconds, "basepullRepeatSeconds", errors);
       requirePositive(maxSearchLimit, "maxSearchLimit", errors);
+      requirePositive(recordSearchTimeoutMillis, "recordSearchTimeoutMillis", errors);
       requirePositive(maxFieldLength, "maxFieldLength", errors);
       requirePositive(runHistoryRetentionDays, "runHistoryRetentionDays", errors);
+      requirePositive(fullReindexStaleMinutes, "fullReindexStaleMinutes", errors);
+      requirePositive(validationSampleSize, "validationSampleSize", errors);
+      requirePositive(diagnosticRunsPerMinute, "diagnosticRunsPerMinute", errors);
+
+      if(validationSampleSize != null && validationSampleSize > MAX_VALIDATION_SAMPLE_SIZE)
+      {
+         errors.add("validationSampleSize must be at most " + MAX_VALIDATION_SAMPLE_SIZE + " for QuickSearchQBit");
+      }
 
       if(basepullOverlapSeconds != null && basepullOverlapSeconds < 0)
       {
@@ -412,18 +461,71 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
    /*******************************************************************************
-    **
+    ** The scheduler must exist when the instance already has schedulers. A
+    ** reconcile cron gets the checks QQQ's instance validator applies to every
+    ** cron schedule (Quartz syntax, a required and known time zone, which QQQ
+    ** never defaults), plus a scheduler that supports cron, so a bad schedule
+    ** fails produce() naming these fields instead of failing at startup.
     *******************************************************************************/
    private void validateScheduling(QInstance qInstance, List<String> errors)
    {
-      if(StringUtils.hasContent(schedulerName) && qInstance != null && !CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulers()) && qInstance.getScheduler(schedulerName) == null)
+      QSchedulerMetaData scheduler = null;
+      if(StringUtils.hasContent(schedulerName) && qInstance != null && !CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulers()))
       {
-         errors.add("Scheduler not found: " + schedulerName);
+         scheduler = qInstance.getScheduler(schedulerName);
+         if(scheduler == null)
+         {
+            errors.add("Scheduler not found: " + schedulerName);
+         }
       }
 
-      if(StringUtils.hasContent(reconcileCronExpression) && !StringUtils.hasContent(schedulerName))
+      validateCron("reconcile", reconcileCronExpression, reconcileCronTimeZoneId, scheduler, errors);
+      validateCron("validation", validationCronExpression, validationCronTimeZoneId, scheduler, errors);
+
+      if(StringUtils.hasContent(validationCronExpression) && !Boolean.TRUE.equals(enableDiagnosticTools))
       {
-         errors.add("reconcileCronExpression requires schedulerName");
+         errors.add("validationCronExpression requires enableDiagnosticTools");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A cron schedule (prefix "reconcile" or "validation"): needs schedulerName
+    ** on a scheduler that supports cron, a valid Quartz expression and a
+    ** recognized time zone ID.
+    *******************************************************************************/
+   private void validateCron(String prefix, String cronExpression, String cronTimeZoneId, QSchedulerMetaData scheduler, List<String> errors)
+   {
+      if(StringUtils.hasContent(cronExpression))
+      {
+         if(!StringUtils.hasContent(schedulerName))
+         {
+            errors.add(prefix + "CronExpression requires schedulerName");
+         }
+         else if(scheduler != null && !scheduler.supportsCronSchedules())
+         {
+            errors.add(prefix + "CronExpression requires a scheduler that supports cron schedules, such as a QuartzSchedulerMetaData; scheduler [" + schedulerName + "] is of type " + scheduler.getType());
+         }
+
+         try
+         {
+            CronExpression.validateExpression(cronExpression);
+         }
+         catch(ParseException e)
+         {
+            errors.add(prefix + "CronExpression [" + cronExpression + "] is not a valid Quartz cron expression: " + e.getMessage());
+         }
+
+         if(!StringUtils.hasContent(cronTimeZoneId))
+         {
+            errors.add(prefix + "CronExpression requires " + prefix + "CronTimeZoneId (a time zone ID such as UTC or America/Chicago)");
+         }
+      }
+
+      if(StringUtils.hasContent(cronTimeZoneId) && !Arrays.asList(TimeZone.getAvailableIDs()).contains(cronTimeZoneId))
+      {
+         errors.add(prefix + "CronTimeZoneId [" + cronTimeZoneId + "] is not a recognized time zone ID");
       }
    }
 
@@ -558,6 +660,17 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** fullReindexStaleMinutes, defaulting to DEFAULT_FULL_REINDEX_STALE_MINUTES.
+    *******************************************************************************/
+   @JsonIgnore
+   public Integer getEffectiveFullReindexStaleMinutes()
+   {
+      return (fullReindexStaleMinutes == null ? DEFAULT_FULL_REINDEX_STALE_MINUTES : fullReindexStaleMinutes);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -580,7 +693,35 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
    /*******************************************************************************
-    ** Apply the table-name prefix, if any.
+    ** Apply the table-name prefix to a table or possible-value source name.
+    ** Table names are lowercase snake_case (owner naming rule, 2026-10-12), so a
+    ** prefix is converted to snake_case and joined with "_": prefix "acme" and
+    ** quick_search_index give acme_quick_search_index.
+    *******************************************************************************/
+   public String applyTablePrefix(String name)
+   {
+      if(StringUtils.hasContent(tableNamePrefix))
+      {
+         String prefix = toSnakeCase(tableNamePrefix);
+         return (prefix.endsWith("_") ? prefix : prefix + "_") + name;
+      }
+      return name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Convert a camelCase name to lowercase snake_case.
+    *******************************************************************************/
+   static String toSnakeCase(String name)
+   {
+      return name.replaceAll("([a-z0-9])([A-Z])", "$1_$2").replace('-', '_').toLowerCase();
+   }
+
+
+
+   /*******************************************************************************
+    ** Apply the name prefix, if any, to a process, widget or app name.
     *******************************************************************************/
    public String applyPrefix(String name)
    {
@@ -594,21 +735,21 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
    /*******************************************************************************
-    ** Name of the produced quickSearchIndex table.
+    ** Name of the produced quick_search_index table.
     *******************************************************************************/
    public String getQuickSearchIndexTableName()
    {
-      return applyPrefix(QuickSearchIndex.TABLE_NAME);
+      return applyTablePrefix(QuickSearchIndex.TABLE_NAME);
    }
 
 
 
    /*******************************************************************************
-    ** Name of the produced quickSearchIndexRun table.
+    ** Name of the produced quick_search_index_run table.
     *******************************************************************************/
    public String getQuickSearchIndexRunTableName()
    {
-      return applyPrefix(QuickSearchIndexRun.TABLE_NAME);
+      return applyTablePrefix(QuickSearchIndexRun.TABLE_NAME);
    }
 
 
@@ -1579,7 +1720,9 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
    /*******************************************************************************
-    ** Getter for reconcileCronExpression: optional cron schedule for reconcile (needs a scheduler that supports cron)
+    ** Getter for reconcileCronExpression: optional Quartz cron schedule for
+    ** reconcile; needs reconcileCronTimeZoneId and a scheduler that supports
+    ** cron (Quartz)
     *******************************************************************************/
    public String getReconcileCronExpression()
    {
@@ -1611,7 +1754,9 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
    /*******************************************************************************
-    ** Getter for reconcileCronTimeZoneId: time zone for reconcileCronExpression
+    ** Getter for reconcileCronTimeZoneId: time zone ID for
+    ** reconcileCronExpression (UTC, America/Chicago); required with it, since
+    ** QQQ has no default zone for cron schedules
     *******************************************************************************/
    public String getReconcileCronTimeZoneId()
    {
@@ -1707,7 +1852,7 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
    /*******************************************************************************
-    ** Getter for runHistoryRetentionDays: delete quickSearchIndexRun rows older than this many days; null keeps them forever
+    ** Getter for runHistoryRetentionDays: delete quick_search_index_run rows older than this many days; null keeps them forever
     *******************************************************************************/
    public Integer getRunHistoryRetentionDays()
    {
@@ -1737,6 +1882,36 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Getter for fullReindexStaleMinutes: a RUNNING full reindex whose run record shows no sign of life (start or heartbeat) for this many minutes is treated as stopped; null means the default
+    *******************************************************************************/
+   public Integer getFullReindexStaleMinutes()
+   {
+      return (this.fullReindexStaleMinutes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fullReindexStaleMinutes
+    *******************************************************************************/
+   public void setFullReindexStaleMinutes(Integer fullReindexStaleMinutes)
+   {
+      this.fullReindexStaleMinutes = fullReindexStaleMinutes;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fullReindexStaleMinutes
+    *******************************************************************************/
+   public QuickSearchQBitConfig withFullReindexStaleMinutes(Integer fullReindexStaleMinutes)
+   {
+      this.fullReindexStaleMinutes = fullReindexStaleMinutes;
+      return (this);
+   }
+
+
 
    /*******************************************************************************
     ** Getter for bulkBatchSize: documents per OpenSearch bulk request
@@ -1897,6 +2072,72 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Getter for serveCoreRecordSearch: register QuickSearchRecordSearchProvider
+    ** so core RecordSearchAction (POST /qqq/v1/search) searches indexed tables in
+    ** OpenSearch, and give indexed tables core searchFields when they have none
+    *******************************************************************************/
+   public Boolean getServeCoreRecordSearch()
+   {
+      return (this.serveCoreRecordSearch);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for serveCoreRecordSearch
+    *******************************************************************************/
+   public void setServeCoreRecordSearch(Boolean serveCoreRecordSearch)
+   {
+      this.serveCoreRecordSearch = serveCoreRecordSearch;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for serveCoreRecordSearch
+    *******************************************************************************/
+   public QuickSearchQBitConfig withServeCoreRecordSearch(Boolean serveCoreRecordSearch)
+   {
+      this.serveCoreRecordSearch = serveCoreRecordSearch;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for recordSearchTimeoutMillis: wall-clock limit on the OpenSearch
+    ** request behind a core record search; past it the provider fails and core
+    ** falls back to its own search (default 2500)
+    *******************************************************************************/
+   public Integer getRecordSearchTimeoutMillis()
+   {
+      return (this.recordSearchTimeoutMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for recordSearchTimeoutMillis
+    *******************************************************************************/
+   public void setRecordSearchTimeoutMillis(Integer recordSearchTimeoutMillis)
+   {
+      this.recordSearchTimeoutMillis = recordSearchTimeoutMillis;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for recordSearchTimeoutMillis
+    *******************************************************************************/
+   public QuickSearchQBitConfig withRecordSearchTimeoutMillis(Integer recordSearchTimeoutMillis)
+   {
+      this.recordSearchTimeoutMillis = recordSearchTimeoutMillis;
+      return (this);
+   }
+
+
+
 
    /*******************************************************************************
     ** Getter for searchableEntityClasses: entity classes annotated with @QuickSearchable
@@ -2058,6 +2299,229 @@ public class QuickSearchQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Getter for enableDiagnosticTools: add the admin diagnostic tools (search playground, index validation, analyzer tester, diagnostic event and validation run tables, diagnostics widget); default true
+    *******************************************************************************/
+   public Boolean getEnableDiagnosticTools()
+   {
+      return (this.enableDiagnosticTools);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for enableDiagnosticTools
+    *******************************************************************************/
+   public void setEnableDiagnosticTools(Boolean enableDiagnosticTools)
+   {
+      this.enableDiagnosticTools = enableDiagnosticTools;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for enableDiagnosticTools
+    *******************************************************************************/
+   public QuickSearchQBitConfig withEnableDiagnosticTools(Boolean enableDiagnosticTools)
+   {
+      this.enableDiagnosticTools = enableDiagnosticTools;
+      return (this);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for enableRecordDiagnosticActions: with enableDiagnosticTools, also add the per-table record actions (inspect search document, reindex records) to every indexed table; default true
+    *******************************************************************************/
+   public Boolean getEnableRecordDiagnosticActions()
+   {
+      return (this.enableRecordDiagnosticActions);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for enableRecordDiagnosticActions
+    *******************************************************************************/
+   public void setEnableRecordDiagnosticActions(Boolean enableRecordDiagnosticActions)
+   {
+      this.enableRecordDiagnosticActions = enableRecordDiagnosticActions;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for enableRecordDiagnosticActions
+    *******************************************************************************/
+   public QuickSearchQBitConfig withEnableRecordDiagnosticActions(Boolean enableRecordDiagnosticActions)
+   {
+      this.enableRecordDiagnosticActions = enableRecordDiagnosticActions;
+      return (this);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for validationCronExpression: cron schedule for the index validation process (needs schedulerName); none by default
+    *******************************************************************************/
+   public String getValidationCronExpression()
+   {
+      return (this.validationCronExpression);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for validationCronExpression
+    *******************************************************************************/
+   public void setValidationCronExpression(String validationCronExpression)
+   {
+      this.validationCronExpression = validationCronExpression;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for validationCronExpression
+    *******************************************************************************/
+   public QuickSearchQBitConfig withValidationCronExpression(String validationCronExpression)
+   {
+      this.validationCronExpression = validationCronExpression;
+      return (this);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for validationCronTimeZoneId: time zone for validationCronExpression
+    *******************************************************************************/
+   public String getValidationCronTimeZoneId()
+   {
+      return (this.validationCronTimeZoneId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for validationCronTimeZoneId
+    *******************************************************************************/
+   public void setValidationCronTimeZoneId(String validationCronTimeZoneId)
+   {
+      this.validationCronTimeZoneId = validationCronTimeZoneId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for validationCronTimeZoneId
+    *******************************************************************************/
+   public QuickSearchQBitConfig withValidationCronTimeZoneId(String validationCronTimeZoneId)
+   {
+      this.validationCronTimeZoneId = validationCronTimeZoneId;
+      return (this);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for validationSampleSize: records sampled per table by index validation (default 50, at most 1000)
+    *******************************************************************************/
+   public Integer getValidationSampleSize()
+   {
+      return (this.validationSampleSize);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for validationSampleSize
+    *******************************************************************************/
+   public void setValidationSampleSize(Integer validationSampleSize)
+   {
+      this.validationSampleSize = validationSampleSize;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for validationSampleSize
+    *******************************************************************************/
+   public QuickSearchQBitConfig withValidationSampleSize(Integer validationSampleSize)
+   {
+      this.validationSampleSize = validationSampleSize;
+      return (this);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for diagnosticRunsPerMinute: per-user limit on diagnostic tool runs per minute (default 30)
+    *******************************************************************************/
+   public Integer getDiagnosticRunsPerMinute()
+   {
+      return (this.diagnosticRunsPerMinute);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for diagnosticRunsPerMinute
+    *******************************************************************************/
+   public void setDiagnosticRunsPerMinute(Integer diagnosticRunsPerMinute)
+   {
+      this.diagnosticRunsPerMinute = diagnosticRunsPerMinute;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for diagnosticRunsPerMinute
+    *******************************************************************************/
+   public QuickSearchQBitConfig withDiagnosticRunsPerMinute(Integer diagnosticRunsPerMinute)
+   {
+      this.diagnosticRunsPerMinute = diagnosticRunsPerMinute;
+      return (this);
+   }
+
+
+
+
+   /*******************************************************************************
+    ** Getter for debugPermissionRules: permission rules for the read-only diagnostic tools and the diagnostics widget (default: quick_search_debug.hasAccess)
+    *******************************************************************************/
+   public QPermissionRules getDebugPermissionRules()
+   {
+      return (this.debugPermissionRules);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for debugPermissionRules
+    *******************************************************************************/
+   public void setDebugPermissionRules(QPermissionRules debugPermissionRules)
+   {
+      this.debugPermissionRules = debugPermissionRules;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for debugPermissionRules
+    *******************************************************************************/
+   public QuickSearchQBitConfig withDebugPermissionRules(QPermissionRules debugPermissionRules)
+   {
+      this.debugPermissionRules = debugPermissionRules;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for runtime: live state once produced (client, publisher, discovered tables)
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java

@@ -25,20 +25,28 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
+import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
-import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
-import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
-import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
-import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
+import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -50,7 +58,9 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** processes, admin app), applies the config's table customizer and default
  ** backend, stamps sourceQBitName, then calls {@link #postProduceActions}, which
  ** discovers searchable tables, builds the runtime, registers the record-change
- ** listener and runtime service, and (in FAIL_FAST mode) prepares the index.
+ ** listener and runtime service, registers the core record-search provider
+ ** (serveCoreRecordSearch) and gives indexed tables core searchFields, and (in
+ ** FAIL_FAST mode) prepares the index.
  **
  ** Host usage, unchanged from 0.x:
  **   new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
@@ -254,6 +264,17 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
 
       qInstance.withRuntimeService(new QCodeReference(QuickSearchRuntimeService.class));
 
+      MiddlewareSpecExtensionsMetaData.ofOrWithNew(qInstance)
+         .withExtension(QuickSearchApi.extension());
+
+      addRecordDiagnosticActions(output, qInstance, discoveredTables);
+
+      if(Boolean.TRUE.equals(qBitConfig.getServeCoreRecordSearch()))
+      {
+         registerRecordSearchProvider(qInstance);
+         defaultCoreSearchFields(qInstance, discoveredTables);
+      }
+
       if(!StringUtils.hasContent(qBitConfig.getSchedulerName()) && Boolean.TRUE.equals(qBitConfig.getEnableBasepullProcess()))
       {
          LOG.warn("Quick Search has no schedulerName; the basepull and reconcile processes are registered but will not run on a schedule");
@@ -270,6 +291,116 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
 
 
 
+   /*******************************************************************************
+    ** Give every indexed table in the instance the Inspect Search Document and
+    ** Reindex in Search record actions (when enableDiagnosticTools and
+    ** enableRecordDiagnosticActions are on). A process the host already
+    ** defined with the same name is kept.
+    *******************************************************************************/
+   void addRecordDiagnosticActions(MetaDataProducerMultiOutput output, QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      if(!Boolean.TRUE.equals(qBitConfig.getEnableDiagnosticTools()) || !Boolean.TRUE.equals(qBitConfig.getEnableRecordDiagnosticActions()))
+      {
+         return;
+      }
+
+      for(QuickSearchableTableConfig tableConfig : tableConfigs)
+      {
+         if(qInstance.getTable(tableConfig.getTableName()) == null)
+         {
+            continue;
+         }
+         for(QProcessMetaData process : List.of(
+            QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(qBitConfig, tableConfig.getTableName()),
+            QuickSearchDiagnosticsProcessMetaDataHelper.buildReindexRecordsProcess(qBitConfig, tableConfig.getTableName())))
+         {
+            if(qInstance.getProcess(process.getName()) == null && output.get(QProcessMetaData.class, process.getName()) == null)
+            {
+               process.setSourceQBitName(getQBitMetaData().getName());
+               output.add(process);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Register QuickSearchRecordSearchProvider as the instance's core record
+    ** search provider. An instance holds one provider; a different one the host
+    ** already registered is kept (with a warning).
+    *******************************************************************************/
+   static void registerRecordSearchProvider(QInstance qInstance)
+   {
+      QCodeReference existing = qInstance.getRecordSearchProvider();
+      if(existing != null && !QuickSearchRecordSearchProvider.class.getName().equals(existing.getName()))
+      {
+         LOG.warn("The instance already has a record search provider; Quick Search does not replace it, so core search does not use OpenSearch",
+            logPair("provider", existing.getName()));
+         return;
+      }
+
+      qInstance.withRecordSearchProvider(new QCodeReference(QuickSearchRecordSearchProvider.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** Core record search only searches tables with searchFields, so give each
+    ** indexed table that has none the searchable fields core accepts (visible
+    ** string or integer fields; never password fields). Fields the host set are
+    ** never replaced. A table that is not in the instance yet, uses backend
+    ** variants, or has no such field is skipped with a warning.
+    *******************************************************************************/
+   public static void defaultCoreSearchFields(QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      for(QuickSearchableTableConfig tableConfig : tableConfigs)
+      {
+         QTableMetaData table = qInstance.getTable(tableConfig.getTableName());
+         if(table == null)
+         {
+            LOG.warn("Indexed table is not in the instance when Quick Search is produced, so it gets no core searchFields; set them on the table for core search to use it",
+               logPair("tableName", tableConfig.getTableName()));
+            continue;
+         }
+
+         if(table.getSearchFields() != null)
+         {
+            continue;
+         }
+
+         QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
+         if(backend != null && Boolean.TRUE.equals(backend.getUsesVariants()))
+         {
+            LOG.warn("Indexed table uses backend variants, which core record search does not support; it gets no core searchFields", logPair("tableName", table.getName()));
+            continue;
+         }
+
+         List<String> searchFields = new ArrayList<>();
+         for(String fieldName : CollectionUtils.nonNullList(tableConfig.getSearchableFields()))
+         {
+            QFieldMetaData field = table.getFields() == null ? null : table.getFields().get(fieldName);
+            if(field != null && !Boolean.TRUE.equals(field.getIsHidden()) && field.getType() != null
+               && (field.getType().isStringLike() || field.getType().isIntegral()) && !field.getType().needsMasked() && !field.getIsSensitive()
+               && !searchFields.contains(fieldName))
+            {
+               searchFields.add(fieldName);
+            }
+         }
+
+         if(searchFields.isEmpty())
+         {
+            LOG.warn("Indexed table has no string or integer searchable field, so it gets no core searchFields; set them on the table for core search to use it",
+               logPair("tableName", table.getName()));
+            continue;
+         }
+
+         table.setSearchFields(searchFields);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Discover searchable tables from @QuickSearchable entity classes. Fails when
     ** a configured field does not exist on a table that is in the instance.
@@ -386,8 +517,12 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
          QTableMetaData table                  = qInstance.getTable(stc.getTableName());
          if(table != null)
          {
-            primaryKeyField        = table.getPrimaryKeyField();
-            basepullTimestampField = (basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField)) ? null : basepullTimestampField;
+            primaryKeyField = table.getPrimaryKeyField();
+            if(basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField))
+            {
+               warnIncrementalBasepullDisabled(table, basepullTimestampField);
+               basepullTimestampField = null;
+            }
             removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
          }
 
@@ -444,7 +579,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
       {
          if(QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD.equals(basepullTimestampField))
          {
-            LOG.warn("Table has no modifyDate field; incremental basepull is disabled for it (reconcile still covers it)", logPair("tableName", table.getName()));
+            warnIncrementalBasepullDisabled(table, basepullTimestampField);
             return (null);
          }
          throw (new QException("basepullTimestampField [" + basepullTimestampField + "] does not exist on table [" + table.getName() + "]"));
@@ -455,22 +590,36 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
 
 
    /*******************************************************************************
-    ** Hidden fields are never indexed: their values would leak through
-    ** searchable text and highlights.
+    ** The startup warning, once per table, for annotated and config-driven
+    ** tables alike, when the basepull timestamp field is missing. A field
+    ** explicitly set to null is an opt-out and is not warned about.
+    *******************************************************************************/
+   private static void warnIncrementalBasepullDisabled(QTableMetaData table, String basepullTimestampField)
+   {
+      LOG.warn("Table has no basepull timestamp field; incremental basepull is disabled for it (reconcile still covers it)",
+         logPair("tableName", table.getName()), logPair("timestampField", basepullTimestampField));
+   }
+
+
+
+   /*******************************************************************************
+    ** Hidden, isSensitive and password fields are never indexed: their values
+    ** would leak through searchable text and highlights.
     *******************************************************************************/
    private static void removeHiddenFields(QTableMetaData table, List<String> searchableFields, Map<String, Integer> fieldWeights, Map<String, Boolean> fieldIncludeLabels)
    {
       List<String> hidden = new ArrayList<>();
       for(String fieldName : searchableFields)
       {
-         if(table.getFields().containsKey(fieldName) && Boolean.TRUE.equals(table.getField(fieldName).getIsHidden()))
+         QFieldMetaData field = table.getFields().get(fieldName);
+         if(field != null && (Boolean.TRUE.equals(field.getIsHidden()) || field.getIsSensitive() || (field.getType() != null && field.getType().needsMasked())))
          {
             hidden.add(fieldName);
          }
       }
       if(!hidden.isEmpty())
       {
-         LOG.warn("Hidden fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
+         LOG.warn("Hidden, sensitive and password fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
          searchableFields.removeAll(hidden);
          hidden.forEach(fieldWeights::remove);
          hidden.forEach(fieldIncludeLabels::remove);

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntime.java

@@ -17,12 +17,22 @@
 package com.kingsrook.qbits.quicksearch;
 
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigStore;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticEventRecorder;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsRateLimiter;
+import com.kingsrook.qbits.quicksearch.diagnostics.StatusCache;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
+import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -34,9 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
-import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -50,6 +58,11 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** QBitMetaData, so steps, listeners and actions resolve it from QContext with
  ** {@link #get()} rather than from JVM-global statics. Two QInstances in one
  ** JVM therefore no longer share state.
+ **
+ ** The tables declared in code (annotations, searchableTables) are kept as
+ ** the code defaults; getDiscoveredTables() and getTableConfig() return the
+ ** effective configs, which come from the config records once they are
+ ** primed (see QuickSearchConfigStore for the precedence).
  *******************************************************************************/
 public class QuickSearchRuntime
 {
@@ -57,17 +70,45 @@ public class QuickSearchRuntime
 
    private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
 
+   public static final long RECORD_SEARCH_BACKOFF_MILLIS = 30_000L;
+
+   private final StatusCache statusCache = new StatusCache();
+
    private final QuickSearchQBitConfig config;
 
-   private List<QuickSearchableTableConfig>        discoveredTables = new ArrayList<>();
-   private Map<String, QuickSearchableTableConfig> tablesByName     = new LinkedHashMap<>();
+   private List<QuickSearchableTableConfig>        codeTables       = new ArrayList<>();
+   private Map<String, QuickSearchableTableConfig> codeTablesByName = new LinkedHashMap<>();
+
+   private final QuickSearchConfigStore configStore;
 
    private volatile QuickSearchOpenSearchClient client;
    private volatile IndexEventPublisher         publisher;
-   private volatile boolean                     indexReady = false;
-   private volatile boolean                     closed     = false;
+   private volatile boolean                     indexReady                     = false;
+   private volatile boolean                     closed                         = false;
+   private volatile long                        recordSearchBackoffUntilMillis = 0L;
 
-   private final Map<String, CachedFlag> enabledCache = new ConcurrentHashMap<>();
+   private final Map<String, CachedFlag> enabledCache  = new ConcurrentHashMap<>();
+   private final Map<String, Long>       buildFailures = new ConcurrentHashMap<>();
+
+   private volatile DiagnosticEventRecorder diagnosticEventRecorder;
+   private final DiagnosticsRateLimiter     diagnosticsRateLimiter = new DiagnosticsRateLimiter();
+   private final AtomicBoolean              validationRunning      = new AtomicBoolean(false);
+   private final AtomicLong                 recordSearchFailures   = new AtomicLong();
+   private final AtomicLong                 backoffDeclinedClaims  = new AtomicLong();
+   private final Instant                    startedAt              = Instant.now();
+   private volatile Instant                 lastRecordSearchFailureTime;
+   private volatile QInstance               startInstance;
+   private volatile String                  lastRecordSearchFailureMessage;
+
+
+
+   /*******************************************************************************
+    ** Instance-owned bounded administrative status cache.
+    *******************************************************************************/
+   public StatusCache getStatusCache()
+   {
+      return (statusCache);
+   }
 
 
 
@@ -85,7 +126,8 @@ public class QuickSearchRuntime
     *******************************************************************************/
    public QuickSearchRuntime(QuickSearchQBitConfig config)
    {
-      this.config = config;
+      this.config      = config;
+      this.configStore = new QuickSearchConfigStore(this);
    }
 
 
@@ -142,36 +184,89 @@ public class QuickSearchRuntime
 
 
    /*******************************************************************************
-    ** Getter for discoveredTables (never null)
+    ** The effective searchable tables (never null): from the config records
+    ** once primed, else the code-declared ones.
     *******************************************************************************/
    public List<QuickSearchableTableConfig> getDiscoveredTables()
    {
-      return (Collections.unmodifiableList(this.discoveredTables));
+      return (Collections.unmodifiableList(configStore.getTables()));
    }
 
 
 
    /*******************************************************************************
-    ** Replace the discovered tables.
+    ** Replace the code-declared tables (the defaults priming seeds).
     *******************************************************************************/
    public void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
    {
-      this.discoveredTables = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
-      this.tablesByName     = new LinkedHashMap<>();
-      for(QuickSearchableTableConfig tableConfig : this.discoveredTables)
+      this.codeTables       = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
+      this.codeTablesByName = new LinkedHashMap<>();
+      for(QuickSearchableTableConfig tableConfig : this.codeTables)
       {
-         this.tablesByName.put(tableConfig.getTableName(), tableConfig);
+         this.codeTablesByName.put(tableConfig.getTableName(), tableConfig);
       }
+      configStore.invalidate();
    }
 
 
 
    /*******************************************************************************
-    ** Find the config for one table, or null.
+    ** The tables declared in code (annotations and searchableTables), never null.
+    *******************************************************************************/
+   public List<QuickSearchableTableConfig> getCodeTables()
+   {
+      return (Collections.unmodifiableList(this.codeTables));
+   }
+
+
+
+   /*******************************************************************************
+    ** One code-declared table's config, or null.
+    *******************************************************************************/
+   public QuickSearchableTableConfig getCodeTableConfig(String tableName)
+   {
+      return (tableName == null ? null : codeTablesByName.get(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Find the effective config for one table, or null when it is not indexed.
     *******************************************************************************/
    public QuickSearchableTableConfig getTableConfig(String tableName)
    {
-      return (tableName == null ? null : tablesByName.get(tableName));
+      return (configStore.getTable(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** The store of the effective config records.
+    *******************************************************************************/
+   public QuickSearchConfigStore getConfigStore()
+   {
+      return (configStore);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an admin has paused all indexing (search keeps serving).
+    *******************************************************************************/
+   public boolean isIndexingPaused()
+   {
+      return (configStore.isIndexingPaused());
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop every cached config (a config row changed).
+    *******************************************************************************/
+   public void invalidateConfig()
+   {
+      configStore.invalidate();
+      enabledCache.clear();
    }
 
 
@@ -256,7 +351,7 @@ public class QuickSearchRuntime
             }
             else
             {
-               publisher = new SynchronousIndexEventPublisher(getClient(), discoveredTables, config.getBulkBatchSize());
+               publisher = new SynchronousIndexEventPublisher(getClient(), this::getTableConfig, config.getBulkBatchSize());
             }
          }
          return (publisher);
@@ -267,16 +362,21 @@ public class QuickSearchRuntime
 
    /*******************************************************************************
     ** Whether a table is enabled for indexing and search, per its
-    ** quickSearchIndex row. A table with no row yet is enabled. Results are
+    ** quick_search_index row. A table with no row yet is enabled. Results are
     ** cached for one minute; a failed lookup is treated as enabled so indexing
     ** does not stop because the operational table is unreachable.
     *******************************************************************************/
    public boolean isTableEnabled(String tableName)
    {
-      if(tableName == null || !tablesByName.containsKey(tableName))
+      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
+      if(tableConfig == null)
       {
          return (false);
       }
+      if(tableConfig.getEnabled() != null)
+      {
+         return (tableConfig.getEnabled());
+      }
 
       long       now    = System.currentTimeMillis();
       CachedFlag cached = enabledCache.get(tableName);
@@ -292,6 +392,36 @@ public class QuickSearchRuntime
 
 
 
+   /*******************************************************************************
+    ** When a queued build of the table last failed on this server (0: never).
+    *******************************************************************************/
+   public long getLastBuildFailureMillis(String tableName)
+   {
+      return (buildFailures.getOrDefault(tableName, 0L));
+   }
+
+
+
+   /*******************************************************************************
+    ** Remember that a queued build of the table failed now.
+    *******************************************************************************/
+   public void recordBuildFailure(String tableName)
+   {
+      buildFailures.put(tableName, System.currentTimeMillis());
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget a table's build failure (it built).
+    *******************************************************************************/
+   public void clearBuildFailure(String tableName)
+   {
+      buildFailures.remove(tableName);
+   }
+
+
+
    /*******************************************************************************
     ** Forget cached enabled flags (after an operator toggles a table).
     *******************************************************************************/
@@ -303,7 +433,180 @@ public class QuickSearchRuntime
 
 
    /*******************************************************************************
-    ** Read the enabled flag from the quickSearchIndex row.
+    ** Record that a core record search through OpenSearch failed. For the next
+    ** RECORD_SEARCH_BACKOFF_MILLIS the record-search provider claims no tables,
+    ** so core searches them itself at once instead of waiting for another
+    ** timeout on every search while OpenSearch is down.
+    *******************************************************************************/
+   public void markRecordSearchFailed()
+   {
+      markRecordSearchFailed(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** As markRecordSearchFailed(), also counting the failure and keeping its
+    ** message and time for the diagnostics.
+    *******************************************************************************/
+   public void markRecordSearchFailed(String message)
+   {
+      recordSearchBackoffUntilMillis = System.currentTimeMillis() + RECORD_SEARCH_BACKOFF_MILLIS;
+      recordSearchFailures.incrementAndGet();
+      lastRecordSearchFailureTime    = Instant.now();
+      lastRecordSearchFailureMessage = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Milliseconds left in the current record-search backoff (0 when none).
+    *******************************************************************************/
+   public long getRecordSearchBackoffRemainingMillis()
+   {
+      return (Math.max(0L, recordSearchBackoffUntilMillis - System.currentTimeMillis()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Count a table the provider declined because of a backoff (core searched
+    ** it with its own search instead).
+    *******************************************************************************/
+   public void countBackoffDeclinedClaim()
+   {
+      backoffDeclinedClaims.incrementAndGet();
+   }
+
+
+
+   /*******************************************************************************
+    ** Record searches through OpenSearch that failed since this runtime started.
+    *******************************************************************************/
+   public long getRecordSearchFailureCount()
+   {
+      return (recordSearchFailures.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Table searches core ran itself because the provider was backing off,
+    ** since this runtime started.
+    *******************************************************************************/
+   public long getBackoffDeclinedClaimCount()
+   {
+      return (backoffDeclinedClaims.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** When the last record search through OpenSearch failed (null when none).
+    *******************************************************************************/
+   public Instant getLastRecordSearchFailureTime()
+   {
+      return (lastRecordSearchFailureTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** The last record-search failure's message (null when none).
+    *******************************************************************************/
+   public String getLastRecordSearchFailureMessage()
+   {
+      return (lastRecordSearchFailureMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** When this runtime was created.
+    *******************************************************************************/
+   public Instant getStartedAt()
+   {
+      return (startedAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** The diagnostic event recorder, built on first use.
+    *******************************************************************************/
+   public DiagnosticEventRecorder getDiagnosticEventRecorder()
+   {
+      DiagnosticEventRecorder existing = diagnosticEventRecorder;
+      if(existing != null)
+      {
+         return (existing);
+      }
+      synchronized(this)
+      {
+         if(diagnosticEventRecorder == null)
+         {
+            diagnosticEventRecorder = new DiagnosticEventRecorder(config);
+         }
+         return (diagnosticEventRecorder);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The per-user rate limiter of the diagnostic tools.
+    *******************************************************************************/
+   public DiagnosticsRateLimiter getDiagnosticsRateLimiter()
+   {
+      return (diagnosticsRateLimiter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Claim the single validation slot of this JVM; false when a validation is
+    ** already running. Release with endValidation().
+    *******************************************************************************/
+   public boolean beginValidation()
+   {
+      return (validationRunning.compareAndSet(false, true));
+   }
+
+
+
+   /*******************************************************************************
+    ** Release the validation slot.
+    *******************************************************************************/
+   public void endValidation()
+   {
+      validationRunning.set(false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether core record search is backing off after a recent failure.
+    *******************************************************************************/
+   public boolean isRecordSearchBackingOff()
+   {
+      return (System.currentTimeMillis() < recordSearchBackoffUntilMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** End a record-search backoff early (an operator or test knows OpenSearch is
+    ** back).
+    *******************************************************************************/
+   public void clearRecordSearchBackoff()
+   {
+      recordSearchBackoffUntilMillis = 0L;
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the enabled flag from the quick_search_index row.
     *******************************************************************************/
    private boolean readEnabledFlag(String tableName)
    {
@@ -345,6 +648,8 @@ public class QuickSearchRuntime
     *******************************************************************************/
    public void start() throws QException
    {
+      primeConfig();
+
       try
       {
          ensureIndexReady();
@@ -377,6 +682,55 @@ public class QuickSearchRuntime
 
 
 
+   /*******************************************************************************
+    ** Prime the config records from the code defaults (never throws; a failure
+    ** is retried on later use). Runs in a system session when no QContext is
+    ** set, as when a launcher starts the runtime service.
+    *******************************************************************************/
+   void primeConfig()
+   {
+      try
+      {
+         if(QContext.getQInstance() != null)
+         {
+            configStore.tryPrime();
+            return;
+         }
+
+         QInstance qInstance = startInstance;
+         if(qInstance == null)
+         {
+            return;
+         }
+         QContext.init(qInstance, new QSystemUserSession());
+         try
+         {
+            configStore.tryPrime();
+         }
+         finally
+         {
+            QContext.clear();
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Quick Search could not prime its config records at startup; it retries on first use", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The instance to prime against when the runtime is started without a
+    ** QContext (QuickSearchRuntimeService.start sets it).
+    *******************************************************************************/
+   public void setStartInstance(QInstance startInstance)
+   {
+      this.startInstance = startInstance;
+   }
+
+
+
    /*******************************************************************************
     ** Close the publisher and client. Safe to call more than once.
     *******************************************************************************/
@@ -387,6 +741,7 @@ public class QuickSearchRuntime
          return;
       }
       closed = true;
+      statusCache.close();
 
       if(publisher != null)
       {
@@ -400,6 +755,11 @@ public class QuickSearchRuntime
          }
       }
 
+      if(diagnosticEventRecorder != null)
+      {
+         diagnosticEventRecorder.close();
+      }
+
       if(client != null)
       {
          client.close();

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntimeService.java

@@ -61,6 +61,7 @@ public class QuickSearchRuntimeService implements QRuntimeServiceInterface
          LOG.warn("Quick Search runtime service started, but no produced Quick Search QBit was found in the QInstance");
          return;
       }
+      runtime.setStartInstance(qInstance);
       runtime.start();
    }
 

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java

@@ -18,6 +18,8 @@ package com.kingsrook.qbits.quicksearch;
 
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
 
 
 /*******************************************************************************
@@ -26,6 +28,12 @@ import java.util.Map;
  ** Describes which fields to index, optional per-field boost weights,
  ** optional per-field label-inclusion flags, basepull scheduling, and
  ** whether the table is enabled by default.
+ **
+ ** Code (annotations, searchableTables) builds these as defaults; at runtime
+ ** the effective configs are built from the quick_search_index and
+ ** quick_search_index_field records (QuickSearchConfigStore), which add each
+ ** field's role, analyzer and snippet flag, the table's enabled flag and its
+ ** index row id.
  *******************************************************************************/
 public class QuickSearchableTableConfig
 {
@@ -40,6 +48,11 @@ public class QuickSearchableTableConfig
    private String              recordLabelFormat;
    private List<String>        recordLabelFields;
    private Integer             maxFieldLength;
+   private Map<String, String>  fieldRoles;
+   private Map<String, String>  fieldAnalyzers;
+   private Map<String, Boolean> fieldIncludeInSnippet;
+   private Boolean             enabled;
+   private Integer             quickSearchIndexId;
 
 
 
@@ -382,4 +395,222 @@ public class QuickSearchableTableConfig
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for fieldRoles
+    *******************************************************************************/
+   public Map<String, String> getFieldRoles()
+   {
+      return (this.fieldRoles);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldRoles
+    *******************************************************************************/
+   public void setFieldRoles(Map<String, String> fieldRoles)
+   {
+      this.fieldRoles = fieldRoles;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldRoles
+    *******************************************************************************/
+   public QuickSearchableTableConfig withFieldRoles(Map<String, String> fieldRoles)
+   {
+      this.fieldRoles = fieldRoles;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldAnalyzers
+    *******************************************************************************/
+   public Map<String, String> getFieldAnalyzers()
+   {
+      return (this.fieldAnalyzers);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldAnalyzers
+    *******************************************************************************/
+   public void setFieldAnalyzers(Map<String, String> fieldAnalyzers)
+   {
+      this.fieldAnalyzers = fieldAnalyzers;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldAnalyzers
+    *******************************************************************************/
+   public QuickSearchableTableConfig withFieldAnalyzers(Map<String, String> fieldAnalyzers)
+   {
+      this.fieldAnalyzers = fieldAnalyzers;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldIncludeInSnippet
+    *******************************************************************************/
+   public Map<String, Boolean> getFieldIncludeInSnippet()
+   {
+      return (this.fieldIncludeInSnippet);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldIncludeInSnippet
+    *******************************************************************************/
+   public void setFieldIncludeInSnippet(Map<String, Boolean> fieldIncludeInSnippet)
+   {
+      this.fieldIncludeInSnippet = fieldIncludeInSnippet;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldIncludeInSnippet
+    *******************************************************************************/
+   public QuickSearchableTableConfig withFieldIncludeInSnippet(Map<String, Boolean> fieldIncludeInSnippet)
+   {
+      this.fieldIncludeInSnippet = fieldIncludeInSnippet;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for enabled
+    *******************************************************************************/
+   public Boolean getEnabled()
+   {
+      return (this.enabled);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for enabled
+    *******************************************************************************/
+   public void setEnabled(Boolean enabled)
+   {
+      this.enabled = enabled;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for enabled
+    *******************************************************************************/
+   public QuickSearchableTableConfig withEnabled(Boolean enabled)
+   {
+      this.enabled = enabled;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for quickSearchIndexId
+    *******************************************************************************/
+   public Integer getQuickSearchIndexId()
+   {
+      return (this.quickSearchIndexId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for quickSearchIndexId
+    *******************************************************************************/
+   public void setQuickSearchIndexId(Integer quickSearchIndexId)
+   {
+      this.quickSearchIndexId = quickSearchIndexId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for quickSearchIndexId
+    *******************************************************************************/
+   public QuickSearchableTableConfig withQuickSearchIndexId(Integer quickSearchIndexId)
+   {
+      this.quickSearchIndexId = quickSearchIndexId;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** A field's role; BODY when none is set.
+    *******************************************************************************/
+   public QuickSearchFieldRole getRole(String fieldName)
+   {
+      return (QuickSearchFieldRole.fromName(fieldRoles == null ? null : fieldRoles.get(fieldName), QuickSearchFieldRole.BODY));
+   }
+
+
+
+   /*******************************************************************************
+    ** A field's analyzer: the one set, else EXACT for an EXACT field, WORD for a
+    ** KEYWORD field and PREFIX for any other.
+    *******************************************************************************/
+   public QuickSearchFieldAnalyzer getAnalyzer(String fieldName)
+   {
+      QuickSearchFieldRole     role     = getRole(fieldName);
+      QuickSearchFieldAnalyzer fallback = role == QuickSearchFieldRole.EXACT ? QuickSearchFieldAnalyzer.EXACT
+         : (role == QuickSearchFieldRole.KEYWORD ? QuickSearchFieldAnalyzer.WORD : QuickSearchFieldAnalyzer.PREFIX);
+      if(role == QuickSearchFieldRole.EXACT)
+      {
+         return (QuickSearchFieldAnalyzer.EXACT);
+      }
+      return (QuickSearchFieldAnalyzer.fromName(fieldAnalyzers == null ? null : fieldAnalyzers.get(fieldName), fallback));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field's text goes into the highlighted snippet (searchableText).
+    ** Defaults to true, except for KEYWORD and EXACT fields and EXACT-analyzed
+    ** fields, which are matched on their own instead.
+    *******************************************************************************/
+   public boolean isIncludedInSnippet(String fieldName)
+   {
+      if(getAnalyzer(fieldName) == QuickSearchFieldAnalyzer.EXACT)
+      {
+         return (false);
+      }
+      Boolean flag = fieldIncludeInSnippet == null ? null : fieldIncludeInSnippet.get(fieldName);
+      if(flag != null)
+      {
+         return (flag);
+      }
+      QuickSearchFieldRole role = getRole(fieldName);
+      return (role != QuickSearchFieldRole.KEYWORD && role != QuickSearchFieldRole.EXACT);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether every field is plain: in the snippet with the PREFIX analyzer, so
+    ** documents and queries look exactly as they did before roles existed.
+    *******************************************************************************/
+   public boolean isAllPlainFields()
+   {
+      for(String fieldName : searchableFields == null ? List.<String>of() : searchableFields)
+      {
+         if(!isIncludedInSnippet(fieldName) || getAnalyzer(fieldName) != QuickSearchFieldAnalyzer.PREFIX)
+         {
+            return (false);
+         }
+      }
+      return (true);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java

@@ -46,6 +46,8 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
 import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
@@ -439,7 +441,9 @@ public class QuickSearchAction
             .withRecordId(source.getRecordId())
             .withRecordLabel(source.getRecordLabel())
             .withScore(rawScore != null ? rawScore.floatValue() : null)
-            .withHighlightSnippet(highlightSnippet));
+            .withHighlightSnippet(highlightSnippet)
+            .withSubtitle(source.getSubtitle())
+            .withDisplayValues(source.getDisplayValues()));
       }
 
       return (results);
@@ -453,14 +457,34 @@ public class QuickSearchAction
     *******************************************************************************/
    private static SearchResponse<OpenSearchDocument> runSearch(QuickSearchOpenSearchClient client, String term, List<String> tables, int fetchSize, int offset, List<QuickSearchableTableConfig> tableConfigs, String preference) throws QException
    {
+      long start = System.currentTimeMillis();
       try
       {
          return (client.search(term, tables, fetchSize, offset, tableConfigs, preference));
       }
       catch(OpenSearchException e)
       {
-         throw (new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e));
+         QException wrapped = new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e);
+         recordFailure(tables, term, start, wrapped);
+         throw (wrapped);
       }
+      catch(QException e)
+      {
+         recordFailure(tables, term, start, e);
+         throw (e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a failed search as a diagnostic event (the term's length only,
+    ** never the term).
+    *******************************************************************************/
+   private static void recordFailure(List<String> tables, String term, long start, Exception e)
+   {
+      DiagnosticsSupport.recordSearchFailure(QuickSearchRuntime.get(), QuickSearchDiagnosticEvent.SOURCE_QUICK_SEARCH_ACTION, tables, term,
+         System.currentTimeMillis() - start, e, "term length " + (term == null ? 0 : term.length()));
    }
 
 

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchRecordSearchProvider.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.actions;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Serves QQQ core record search (RecordSearchAction, POST /qqq/v1/search,
+ ** the Next UI global search) from the Quick Search index, for the tables
+ ** this QBit indexes. Registered on the QInstance by QuickSearchQBitProducer
+ ** when serveCoreRecordSearch is on (the default).
+ **
+ ** Core creates a new instance for every search, so this class holds no
+ ** state: it resolves the QBit's QuickSearchRuntime (client, tables, config)
+ ** from QContext.
+ **
+ ** A table is claimed when it is indexed by this QBit and enabled, and core
+ ** record search is not backing off after a recent failure. search() sends
+ ** one _msearch request with a top-N search per table and returns record ids
+ ** only, best first; core re-reads them by primary key through QueryAction,
+ ** so table permissions, record security locks and record labels are core's
+ ** job, and ids of deleted records drop out. Each table gets
+ ** IDS_PER_LIMIT_MULTIPLIER x limitPerTable ids (at most
+ ** RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE) so dropouts do not leave
+ ** the table short.
+ **
+ ** The request is bounded by recordSearchTimeoutMillis. Any failure throws,
+ ** and core then searches the claimed tables with its own LIKE search; the
+ ** failure also starts a short backoff (QuickSearchRuntime
+ ** RECORD_SEARCH_BACKOFF_MILLIS) during which no tables are claimed.
+ *******************************************************************************/
+public class QuickSearchRecordSearchProvider implements RecordSearchProviderInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordSearchProvider.class);
+
+   public static final int IDS_PER_LIMIT_MULTIPLIER = 4;
+
+
+
+   /*******************************************************************************
+    ** Claim tables this QBit indexes and that are enabled, unless core record
+    ** search is off or backing off. Cheap: an in-memory lookup plus the
+    ** runtime's cached enabled flag.
+    *******************************************************************************/
+   @Override
+   public boolean claimsTable(QTableMetaData table)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || table == null || !Boolean.TRUE.equals(runtime.getConfig().getServeCoreRecordSearch()))
+      {
+         return (false);
+      }
+
+      if(runtime.getTableConfig(table.getName()) == null)
+      {
+         return (false);
+      }
+
+      if(runtime.isRecordSearchBackingOff())
+      {
+         runtime.countBackoffDeclinedClaim();
+         return (false);
+      }
+
+      return (runtime.isTableEnabled(table.getName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Ids of the best matches in each requested table, in relevance order.
+    *******************************************************************************/
+   @Override
+   public RecordSearchOutput search(RecordSearchInput input) throws QException
+   {
+      QuickSearchRuntime    runtime = QuickSearchRuntime.getOrThrow();
+      QuickSearchQBitConfig config  = runtime.getConfig();
+
+      List<RecordSearchResult> results = new ArrayList<>();
+      String                   term    = IndexingUtils.normalizeSearchText(input.getSearchTerm());
+      if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         return (new RecordSearchOutput().withResults(results));
+      }
+
+      List<String> tableNames = new ArrayList<>();
+      for(String tableName : new LinkedHashSet<>(CollectionUtils.nonNullList(input.getTableNames())))
+      {
+         if(runtime.getTableConfig(tableName) != null)
+         {
+            tableNames.add(tableName);
+         }
+      }
+      if(tableNames.isEmpty())
+      {
+         return (new RecordSearchOutput().withResults(results));
+      }
+
+      int limitPerTable = input.getLimitPerTable() == null || input.getLimitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : input.getLimitPerTable();
+      int idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * IDS_PER_LIMIT_MULTIPLIER);
+      int timeoutMillis = config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis();
+
+      Map<String, List<String>> idsByTable;
+      long                      start = System.currentTimeMillis();
+      try
+      {
+         idsByTable = runtime.getClient().searchRecordIdsPerTable(term, tableNames, idsPerTable, runtime.getDiscoveredTables(), timeoutMillis);
+      }
+      catch(QException | RuntimeException e)
+      {
+         runtime.markRecordSearchFailed(e.getMessage());
+
+         //////////////////////////////////////////////////////////////////////
+         // the user's term is not recorded (it may hold personal data); its //
+         // length is, so a length-related failure is still visible          //
+         //////////////////////////////////////////////////////////////////////
+         DiagnosticsSupport.recordSearchFailure(runtime, QuickSearchDiagnosticEvent.SOURCE_CORE_RECORD_SEARCH, tableNames, term, System.currentTimeMillis() - start, e,
+            "term length " + term.length() + ", " + idsPerTable + " ids per table, timeout " + timeoutMillis + " ms; core searched these tables itself");
+         LOG.info("Quick Search record search failed; core searches these tables itself, and Quick Search claims none for a while", logPair("tables", tableNames),
+            logPair("backoffMillis", QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS), logPair("error", e.getMessage()));
+         throw (e instanceof QException qe ? qe : new QException("Quick Search record search failed: " + e.getMessage(), e));
+      }
+
+      for(String tableName : tableNames)
+      {
+         for(String recordId : CollectionUtils.nonNullList(idsByTable.get(tableName)))
+         {
+            results.add(new RecordSearchResult().withTableName(tableName).withRecordId(recordId));
+         }
+      }
+
+      return (new RecordSearchOutput().withResults(results));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java

@@ -17,6 +17,9 @@
 package com.kingsrook.qbits.quicksearch.actions;
 
 
+import java.util.Map;
+
+
 /*******************************************************************************
  ** A single search hit returned by a QuickSearch query, representing one
  ** matching record with its relevance score and optional highlight snippet.
@@ -30,6 +33,8 @@ public class QuickSearchResult
    private String recordLabel;
    private Float  score;
    private String highlightSnippet;
+   private String subtitle;
+   private Map<String, String> displayValues;
 
 
    /*******************************************************************************
@@ -202,4 +207,66 @@ public class QuickSearchResult
       return this;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for subtitle
+    *******************************************************************************/
+   public String getSubtitle()
+   {
+      return (this.subtitle);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for subtitle
+    *******************************************************************************/
+   public void setSubtitle(String subtitle)
+   {
+      this.subtitle = subtitle;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for subtitle
+    *******************************************************************************/
+   public QuickSearchResult withSubtitle(String subtitle)
+   {
+      this.subtitle = subtitle;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for displayValues
+    *******************************************************************************/
+   public Map<String, String> getDisplayValues()
+   {
+      return (this.displayValues);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for displayValues
+    *******************************************************************************/
+   public void setDisplayValues(Map<String, String> displayValues)
+   {
+      this.displayValues = displayValues;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for displayValues
+    *******************************************************************************/
+   public QuickSearchResult withDisplayValues(Map<String, String> displayValues)
+   {
+      this.displayValues = displayValues;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchApi.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension;
+
+
+/*******************************************************************************
+ ** Supported host navigation and UI API extension identity.
+ *******************************************************************************/
+public final class QuickSearchApi
+{
+   public static final String APP_NAME = QuickSearchAdminAppMetaDataProducer.APP_NAME;
+   public static final String HOME_HREF = "/app/" + APP_NAME;
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private QuickSearchApi()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The spec registration shared by production and contract generation.
+    *******************************************************************************/
+   public static MiddlewareSpecExtension extension()
+   {
+      return (new MiddlewareSpecExtension().withName("quick-search").withSemanticVersion("1.0.0")
+         .withDescription("Quick Search administrative status").withSpec(QuickSearchStatusSpec.class));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatus.java

@@ -0,0 +1,86 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema;
+import com.kingsrook.qqq.openapi.model.Schema;
+
+
+/*******************************************************************************
+ ** Immutable administrative snapshot. Null measurements are unknown, never zero.
+ ** Counts come from persisted diagnostics; countsCheckedAt identifies their age.
+ *******************************************************************************/
+public record QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
+                                Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
+                                Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref) implements ToSchema
+{
+   /*******************************************************************************
+    ** Published wire schema for the same immutable value used by Java hosts.
+    *******************************************************************************/
+   @Override
+   public Schema toSchema()
+   {
+      return (new QuickSearchStatusSpec().defineComponentSchemas().get("QuickSearchStatusV1"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Protect the snapshot from mutations by callers.
+    *******************************************************************************/
+   public QuickSearchStatus
+   {
+      indexes = List.copyOf(indexes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Overall availability; DEGRADED also covers incomplete measurements.
+    *******************************************************************************/
+   public enum State
+   {
+      OK, DEGRADED, DOWN, DISABLED
+   }
+
+
+
+   /*******************************************************************************
+    ** A persisted index row with counts from the latest saved validation, when
+    ** available. Otherwise documentCount is the last indexing run's count and
+    ** sourceCount/countsCheckedAt are unknown. Sync time is a successful watermark.
+    *******************************************************************************/
+   public record IndexRow(String tableName, Boolean enabled, String status, Long documentCount, Long sourceCount, Instant countsCheckedAt,
+                          Instant lastSuccessfulSync, String lastError, Integer realTimeErrorCount) implements ToSchema
+   {
+      /*******************************************************************************
+       ** Published per-table schema.
+       *******************************************************************************/
+      @Override
+      public Schema toSchema()
+      {
+         return (new QuickSearchStatusSpec().defineComponentSchemas().get("QuickSearchIndexStatusV1"));
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatusService.java

@@ -0,0 +1,70 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.StatusDiagnostics;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+
+
+/*******************************************************************************
+ ** Supported entry point for hosts. Uses the current QContext and checks the
+ ** Quick Search admin app permission on EVERY call, including cache hits.
+ *******************************************************************************/
+public final class QuickSearchStatusService
+{
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private QuickSearchStatusService()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Return a briefly cached snapshot, waiting at most two seconds for a refresh.
+    ** Call with an authenticated QSession (or an explicitly trusted system session).
+    *******************************************************************************/
+   public static QuickSearchStatus get() throws QException
+   {
+      if(QContext.getQSession() == null)
+      {
+         throw (new QPermissionDeniedException("A session is required for Quick Search status."));
+      }
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (StatusDiagnostics.unavailable(QuickSearchStatus.State.DISABLED, null, QuickSearchApi.HOME_HREF));
+      }
+      String appName = runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
+      if(QContext.getQInstance().getApp(appName) == null || !PermissionsHelper.hasAppPermission(new QueryInput(), appName))
+      {
+         throw (new QPermissionDeniedException("Quick Search admin permission is required."));
+      }
+      return (runtime.getStatusCache().get(runtime));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatusSpec.java

@@ -0,0 +1,182 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
+import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
+import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
+import com.kingsrook.qqq.openapi.model.HttpMethod;
+import com.kingsrook.qqq.openapi.model.Method;
+import com.kingsrook.qqq.openapi.model.Parameter;
+import com.kingsrook.qqq.openapi.model.Schema;
+import com.kingsrook.qqq.openapi.model.Type;
+import io.javalin.http.HttpStatus;
+
+
+/*******************************************************************************
+ ** Administrative status, available through the spec-managed UI API only.
+ *******************************************************************************/
+public class QuickSearchStatusSpec extends AbstractJsonEndpointSpec
+{
+   /*******************************************************************************
+    ** Describe the supported operation.
+    *******************************************************************************/
+   @Override
+   public BasicOperation defineBasicOperation()
+   {
+      return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/quick-search/status")
+         .withShortSummary("Get Quick Search status")
+         .withLongDescription("Requires Quick Search admin permission. Five-second instance cache; two-second refresh wait. Counts are saved diagnostics, not live source scans. Unknown values are null. Backlog counts are lower bounds when backlogTruncated is true."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Stable generated-client operation name.
+    *******************************************************************************/
+   @Override
+   protected void customizeMethod(Method method)
+   {
+      method.setOperationId("getQuickSearchStatus");
+   }
+
+
+
+   /*******************************************************************************
+    ** No request parameters.
+    *******************************************************************************/
+   @Override
+   public List<Parameter> defineRequestParameters()
+   {
+      return (List.of());
+   }
+
+
+
+   /*******************************************************************************
+    ** Versioned response component.
+    *******************************************************************************/
+   @Override
+   protected String getResponseSchemaName()
+   {
+      return ("QuickSearchStatusV1");
+   }
+
+
+
+   /*******************************************************************************
+    ** Use exactly the same authorized service as in-process hosts.
+    *******************************************************************************/
+   @Override
+   protected Object handle(JsonInput input) throws Exception
+   {
+      return (QuickSearchStatusService.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Standard framework errors, including the diagnostics rate limit.
+    *******************************************************************************/
+   @Override
+   public List<BasicResponse> defineAdditionalBasicResponses()
+   {
+      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Diagnostic rate limit exceeded", "QuickSearchErrorV1"),
+         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", "QuickSearchErrorV1"),
+         new BasicResponse(HttpStatus.FORBIDDEN, "Quick Search admin permission required", "QuickSearchErrorV1")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Explicit schemas make nullability, bounds and measurement age discoverable.
+    *******************************************************************************/
+   @Override
+   public Map<String, Schema> defineComponentSchemas()
+   {
+      Map<String, Schema> status = new TreeMap<>();
+      status.put("state", field(Type.STRING, "Overall availability; incomplete diagnostics are DEGRADED.", false)
+         .withEnumValues(List.of("OK", "DEGRADED", "DOWN", "DISABLED")));
+      status.put("clusterReachable", field(Type.BOOLEAN, "Whether OpenSearch answered; null when not probed.", true));
+      status.put("clusterHealth", field(Type.STRING, "OpenSearch health color; null when unknown.", true));
+      for(String name : List.of("nodes", "activeShards", "unassignedShards"))
+      {
+         status.put(name, field(Type.INTEGER, "Cluster health counter; null when unavailable.", true));
+      }
+      status.put("indexingPaused", field(Type.BOOLEAN, "Indexing is paused; search can still serve.", true));
+      status.put("indexes", new Schema().withType(Type.ARRAY).withItems(new Schema().withRef("#/components/schemas/QuickSearchIndexStatusV1"))
+         .withDescription("At most 100 persisted index rows, ordered by id."));
+      status.put("pending", field(Type.INTEGER, "Pending and awaiting-reindex failed events, from at most 1000 backlog rows.", true));
+      status.put("exhausted", field(Type.INTEGER, "Exhausted failed events, from the same bounded backlog read.", true));
+      status.put("backlogTruncated", field(Type.BOOLEAN, "Counts are lower bounds when more than 1000 backlog rows exist.", false));
+      status.put("tablesTruncated", field(Type.BOOLEAN, "More than 100 index rows exist.", false));
+      status.put("lastSuccessfulSync", timestamp("Newest successful indexing watermark; not a guarantee all tables are current."));
+      status.put("lastError", field(Type.STRING, "Last known indexing/search error or snapshot failure; null when absent.", true));
+      status.put("checkedAt", timestamp("Snapshot completion time; remains unchanged on cache hits.").withNullable(false));
+      status.put("homeHref", field(Type.STRING, "Quick Search home link, including the configured metadata prefix.", false));
+      Map<String, Schema> row = new TreeMap<>();
+      row.put("tableName", field(Type.STRING, "Indexed source table metadata name.", false));
+      row.put("enabled", field(Type.BOOLEAN, "Whether this index is enabled.", false));
+      row.put("status", field(Type.STRING, "Persisted indexing state, such as ACTIVE or NEEDS_REINDEX.", true));
+      row.put("documentCount", field(Type.INTEGER, "Saved validation document count, else last indexing run count; not live.", true).withFormat("int64"));
+      row.put("sourceCount", field(Type.INTEGER, "Source count from saved IndexValidator report; null without validation.", true).withFormat("int64"));
+      row.put("countsCheckedAt", timestamp("Saved validation completion; null when validation counts are unavailable."));
+      row.put("lastSuccessfulSync", timestamp("Newest successful basepull/full-reindex/reconcile watermark."));
+      row.put("lastError", field(Type.STRING, "Persisted indexing error; null when absent.", true));
+      row.put("realTimeErrorCount", field(Type.INTEGER, "Persisted real-time indexing error count.", true));
+      return (Map.of("QuickSearchStatusV1", object(status), "QuickSearchIndexStatusV1", object(row),
+         "QuickSearchErrorV1", object(Map.of("error", field(Type.STRING, "Framework error message.", false)))));
+   }
+
+
+
+   /*******************************************************************************
+    ** A response object always includes its properties, including nulls.
+    *******************************************************************************/
+   private static Schema object(Map<String, Schema> properties)
+   {
+      return (new Schema().withType(Type.OBJECT).withProperties(new TreeMap<>(properties)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A scalar with explicit unknown semantics.
+    *******************************************************************************/
+   private static Schema field(Type type, String description, Boolean nullable)
+   {
+      return (new Schema().withType(type).withDescription(description).withNullable(nullable));
+   }
+
+
+
+   /*******************************************************************************
+    ** ISO-8601 instant.
+    *******************************************************************************/
+   private static Schema timestamp(String description)
+   {
+      return (field(Type.STRING, description, true).withFormat("date-time"));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java

@@ -0,0 +1,498 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Self-priming: seeds the config records from the code defaults
+ ** (@QuickSearchable / @QuickSearchField entities and searchableTables), on
+ ** startup and until it has succeeded once per runtime.
+ **
+ ** Precedence (the records win; code only fills in what is missing):
+ ** - A table code declares that has no quick_search_index row, and no
+ **   tombstone, gets a row (source CODE, enabled per enabledByDefault) and
+ **   field rows, and is queued for its first build (status PENDING_BUILD,
+ **   which the next basepull runs). A tombstoned table is never re-added.
+ ** - On a row an admin has not edited (ownerEdited false), the basepull
+ **   interval and timestamp field follow code. enabled is set only when the
+ **   row is created, never afterwards.
+ ** - A field code declares that has no row, and no tombstone, is added; a
+ **   field row an admin has not edited follows code's weight and label flag.
+ **   Either change queues a rebuild of the table. Field rows an admin added
+ **   or edited are never changed, and nothing is ever deleted by priming:
+ **   removing an annotation leaves its rows for the admin to remove.
+ ** - Rows from before 1.1.0-RC.3 (no field rows yet) get their field rows
+ **   without a rebuild when the index was already built with those fields.
+ **
+ ** All writes are system writes (QuickSearchConfigWrites), so they are not
+ ** taken for an admin's edits. The outcome is written to quick_search_settings
+ ** (lastPrimingSummary), which the Quick Search home shows.
+ *******************************************************************************/
+public class QuickSearchConfigPrimer
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchConfigPrimer.class);
+
+   private final QuickSearchRuntime    runtime;
+   private final QuickSearchQBitConfig config;
+
+
+
+   /*******************************************************************************
+    ** What priming did.
+    **
+    ** @param newTables     labels of the tables added (and queued to build)
+    ** @param newFields     field rows added
+    ** @param updatedFields field rows updated to code's defaults
+    ** @param updatedTables index rows updated to code's defaults
+    ** @param rebuilds      labels of existing tables queued to rebuild
+    ** @param skipped       code tables or fields skipped for a tombstone
+    *******************************************************************************/
+   public record PrimingReport(List<String> newTables, int newFields, int updatedFields, int updatedTables, List<String> rebuilds, int skipped)
+   {
+      /*******************************************************************************
+       ** Whether priming changed anything.
+       *******************************************************************************/
+      public boolean hasChanges()
+      {
+         return (!newTables.isEmpty() || newFields > 0 || updatedFields > 0 || updatedTables > 0 || !rebuilds.isEmpty());
+      }
+
+
+
+      /*******************************************************************************
+       ** The one-line summary for the Quick Search home.
+       *******************************************************************************/
+      public String summary()
+      {
+         List<String> parts = new ArrayList<>();
+         if(!newTables.isEmpty())
+         {
+            parts.add("New: " + newTables.size() + (newTables.size() == 1 ? " table" : " tables") + " added and building (" + String.join(", ", newTables) + ")");
+         }
+         if(newFields > 0 || updatedFields > 0)
+         {
+            parts.add(newFields + " field" + (newFields == 1 ? "" : "s") + " added, " + updatedFields + " updated from code defaults");
+         }
+         if(!rebuilds.isEmpty())
+         {
+            parts.add("rebuilding " + String.join(", ", rebuilds));
+         }
+         if(updatedTables > 0)
+         {
+            parts.add(updatedTables + " index setting" + (updatedTables == 1 ? "" : "s") + " updated from code defaults");
+         }
+         return (parts.isEmpty() ? "No changes: the records match the code defaults." : String.join("; ", parts) + ".");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchConfigPrimer(QuickSearchRuntime runtime)
+   {
+      this.runtime = runtime;
+      this.config  = runtime.getConfig();
+   }
+
+
+
+   /*******************************************************************************
+    ** Seed the records from the code defaults. Throws when the config tables
+    ** cannot be read or written (the caller retries later).
+    *******************************************************************************/
+   public PrimingReport prime() throws QException
+   {
+      PrimingReport report = QuickSearchConfigWrites.asSystem(this::doPrime);
+      if(report.hasChanges())
+      {
+         LOG.info("Quick Search primed its config records: " + report.summary());
+      }
+      return (report);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private PrimingReport doPrime() throws QException
+   {
+      QInstance qInstance  = QContext.getQInstance();
+      String    indexTable = config.getQuickSearchIndexTableName();
+      String    fieldTable = config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME);
+
+      Set<String> tableTombstones = new HashSet<>();
+      Set<String> fieldTombstones = new HashSet<>();
+      String      tombstoneTable  = config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME);
+      if(qInstance.getTable(tombstoneTable) != null)
+      {
+         for(QRecord tombstone : queryAll(tombstoneTable))
+         {
+            String fieldName = tombstone.getValueString("fieldName");
+            if(fieldName == null)
+            {
+               tableTombstones.add(tombstone.getValueString("tableName"));
+            }
+            else
+            {
+               fieldTombstones.add(tombstone.getValueString("tableName") + "." + fieldName);
+            }
+         }
+      }
+
+      Map<String, QRecord> rowsByTable = new LinkedHashMap<>();
+      for(QRecord row : queryAll(indexTable))
+      {
+         rowsByTable.putIfAbsent(row.getValueString("tableName"), row);
+      }
+      Map<Integer, Map<String, QRecord>> fieldsByIndex = new HashMap<>();
+      for(QRecord fieldRow : queryAll(fieldTable))
+      {
+         fieldsByIndex.computeIfAbsent(fieldRow.getValueInteger("quickSearchIndexId"), k -> new LinkedHashMap<>()).putIfAbsent(fieldRow.getValueString("fieldName"), fieldRow);
+      }
+
+      List<String> newTables     = new ArrayList<>();
+      List<String> rebuilds      = new ArrayList<>();
+      int          newFields     = 0;
+      int          updatedFields = 0;
+      int          updatedTables = 0;
+      int          skipped       = 0;
+
+      for(QuickSearchableTableConfig code : runtime.getCodeTables())
+      {
+         String         tableName = code.getTableName();
+         QTableMetaData table     = qInstance.getTable(tableName);
+         if(table == null)
+         {
+            LOG.warn("A table declared searchable in code is not in the instance; it is not primed", logPair("tableName", tableName));
+            continue;
+         }
+         if(tableTombstones.contains(tableName))
+         {
+            skipped++;
+            continue;
+         }
+
+         List<QRecord> codeFields = new ArrayList<>();
+         List<String>  fieldNames = CollectionUtils.nonNullList(code.getSearchableFields());
+         for(int i = 0; i < fieldNames.size(); i++)
+         {
+            String fieldName = fieldNames.get(i);
+            if(fieldTombstones.contains(tableName + "." + fieldName))
+            {
+               skipped++;
+               continue;
+            }
+            if(!QuickSearchFieldRules.isIndexable(table, fieldName))
+            {
+               LOG.warn("A field declared searchable in code cannot be indexed; it is not primed: " + QuickSearchFieldRules.whyNotIndexable(table, fieldName),
+                  logPair("tableName", tableName), logPair("fieldName", fieldName));
+               continue;
+            }
+            codeFields.add(codeFieldRow(code, fieldName, i));
+         }
+
+         QRecord row = rowsByTable.get(tableName);
+         if(row == null)
+         {
+            boolean enabled = !Boolean.FALSE.equals(code.getEnabledByDefault());
+            QRecord inserted = insertOne(indexTable, new QRecord()
+               .withValue("tableName", tableName)
+               .withValue("enabled", enabled)
+               .withValue("basepullIntervalMinutes", code.getBasepullIntervalMinutes())
+               .withValue("basepullTimestampField", code.getBasepullTimestampField())
+               .withValue("source", QuickSearchIndex.SOURCE_CODE)
+               .withValue("ownerEdited", false)
+               .withValue("realTimeErrorCount", 0)
+               .withValue("status", enabled ? AbstractIndexingStep.STATUS_PENDING_BUILD : AbstractIndexingStep.STATUS_ACTIVE));
+            Integer indexId = inserted.getValueInteger("id");
+            codeFields.forEach(f -> f.setValue("quickSearchIndexId", indexId));
+            insertAll(fieldTable, codeFields);
+            newFields += codeFields.size();
+            newTables.add(QuickSearchFieldRules.label(table));
+            continue;
+         }
+
+         Integer                   indexId = row.getValueInteger("id");
+         Map<String, Serializable> updates = new LinkedHashMap<>();
+         if(row.getValueString("source") == null)
+         {
+            updates.put("source", QuickSearchIndex.SOURCE_CODE);
+         }
+         if(!Boolean.TRUE.equals(row.getValueBoolean("ownerEdited")))
+         {
+            if(!Objects.equals(row.getValueInteger("basepullIntervalMinutes"), code.getBasepullIntervalMinutes()) && code.getBasepullIntervalMinutes() != null)
+            {
+               updates.put("basepullIntervalMinutes", code.getBasepullIntervalMinutes());
+            }
+            if(!Objects.equals(row.getValueString("basepullTimestampField"), code.getBasepullTimestampField()))
+            {
+               updates.put("basepullTimestampField", code.getBasepullTimestampField());
+            }
+         }
+         if(updates.keySet().stream().anyMatch(k -> !"source".equals(k)))
+         {
+            updatedTables++;
+         }
+
+         Map<String, QRecord> existingFields = fieldsByIndex.getOrDefault(indexId, Map.of());
+         boolean              migration      = existingFields.isEmpty();
+         boolean              fieldsChanged  = false;
+         List<QRecord>        toInsert       = new ArrayList<>();
+         List<QRecord>        toUpdate       = new ArrayList<>();
+         for(QRecord codeField : codeFields)
+         {
+            QRecord existing = existingFields.get(codeField.getValueString("fieldName"));
+            if(existing == null)
+            {
+               codeField.setValue("quickSearchIndexId", indexId);
+               toInsert.add(codeField);
+               fieldsChanged = true;
+            }
+            else if(!Boolean.TRUE.equals(existing.getValueBoolean("ownerEdited"))
+               && (!Objects.equals(existing.getValueInteger("weight"), codeField.getValueInteger("weight"))
+               || !Objects.equals(Boolean.TRUE.equals(existing.getValueBoolean("includeLabel")), codeField.getValueBoolean("includeLabel"))))
+            {
+               toUpdate.add(new QRecord().withValue("id", existing.getValue("id"))
+                  .withValue("weight", codeField.getValueInteger("weight"))
+                  .withValue("includeLabel", codeField.getValueBoolean("includeLabel")));
+               fieldsChanged = true;
+            }
+         }
+         insertAll(fieldTable, toInsert);
+         updateAll(fieldTable, toUpdate);
+         newFields += toInsert.size();
+         updatedFields += toUpdate.size();
+
+         if(fieldsChanged && needsRebuild(row, code, migration))
+         {
+            updates.put("status", AbstractIndexingStep.STATUS_PENDING_BUILD);
+            rebuilds.add(QuickSearchFieldRules.label(table));
+         }
+
+         if(!updates.isEmpty())
+         {
+            QRecord update = new QRecord().withValue("id", indexId);
+            updates.forEach(update::setValue);
+            updateAll(indexTable, List.of(update));
+         }
+      }
+
+      PrimingReport report = new PrimingReport(newTables, newFields, updatedFields, updatedTables, rebuilds, skipped);
+      writeSettings(report);
+      return (report);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a primed field change needs the table rebuilt: not when it is
+    ** disabled or already building, and not for rows from before field
+    ** records when the index was already built with exactly code's fields
+    ** (or never built: its first basepull reads everything anyway).
+    *******************************************************************************/
+   private static boolean needsRebuild(QRecord row, QuickSearchableTableConfig code, boolean migration)
+   {
+      String status = row.getValueString("status");
+      if(Boolean.FALSE.equals(row.getValueBoolean("enabled"))
+         || AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status)
+         || AbstractIndexingStep.STATUS_BUILDING.equals(status)
+         || AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+      {
+         return (false);
+      }
+      if(migration)
+      {
+         String built = row.getValueString("searchableFieldsJson");
+         return (built != null && !built.equals(AbstractIndexingStep.buildSearchableFieldsJson(code)));
+      }
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** A field row from code's defaults (role BODY; analyzer and snippet left
+    ** to their defaults).
+    *******************************************************************************/
+   private static QRecord codeFieldRow(QuickSearchableTableConfig code, String fieldName, int sortOrder)
+   {
+      Integer weight       = code.getFieldWeights() == null ? null : code.getFieldWeights().get(fieldName);
+      Boolean includeLabel = code.getFieldIncludeLabels() == null ? null : code.getFieldIncludeLabels().get(fieldName);
+      return (new QRecord()
+         .withValue("fieldName", fieldName)
+         .withValue("weight", weight == null ? 1 : weight)
+         .withValue("includeLabel", Boolean.TRUE.equals(includeLabel))
+         .withValue("role", QuickSearchFieldRole.BODY.name())
+         .withValue("sortOrder", (sortOrder + 1) * 10)
+         .withValue("source", QuickSearchIndexField.SOURCE_CODE)
+         .withValue("ownerEdited", false));
+   }
+
+
+
+   /*******************************************************************************
+    ** Create the settings row when missing, and record the priming outcome.
+    *******************************************************************************/
+   private void writeSettings(PrimingReport report) throws QException
+   {
+      String settingsTable = config.applyTablePrefix(QuickSearchSettings.TABLE_NAME);
+      if(QContext.getQInstance().getTable(settingsTable) == null)
+      {
+         return;
+      }
+
+      QRecord values = new QRecord().withValue("lastPrimedAt", Instant.now());
+      if(report.hasChanges())
+      {
+         values.withValue("lastPrimingSummary", report.summary()).withValue("lastPrimingNewTables", report.newTables().size());
+      }
+
+      List<QRecord> rows = queryAll(settingsTable);
+      if(rows.isEmpty())
+      {
+         values.withValue("id", QuickSearchSettings.SETTINGS_ID).withValue("indexingPaused", false);
+         if(!report.hasChanges())
+         {
+            values.withValue("lastPrimingSummary", report.summary()).withValue("lastPrimingNewTables", 0);
+         }
+         insertOne(settingsTable, values);
+      }
+      else
+      {
+         values.withValue("id", rows.get(0).getValue("id"));
+         updateAll(settingsTable, List.of(values));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Every row of a table.
+    *******************************************************************************/
+   public static List<QRecord> queryAll(String tableName) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter());
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one row, failing on any record error.
+    *******************************************************************************/
+   public static QRecord insertOne(String tableName, QRecord record) throws QException
+   {
+      List<QRecord> inserted = insertAll(tableName, List.of(record));
+      if(inserted.isEmpty())
+      {
+         throw (new QException("Insert into " + tableName + " returned no record"));
+      }
+      return (inserted.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert rows, failing on any record error.
+    *******************************************************************************/
+   public static List<QRecord> insertAll(String tableName, List<QRecord> records) throws QException
+   {
+      if(records.isEmpty())
+      {
+         return (List.of());
+      }
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(tableName);
+      insertInput.setRecords(records);
+      InsertOutput  output   = new InsertAction().execute(insertInput);
+      List<QRecord> inserted = CollectionUtils.nonNullList(output.getRecords());
+      for(QRecord record : inserted)
+      {
+         if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            throw (new QException("Could not insert into " + tableName + ": " + record.getErrors().get(0).getMessage()));
+         }
+      }
+      return (inserted);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update rows, failing on any record error.
+    *******************************************************************************/
+   public static void updateAll(String tableName, List<QRecord> records) throws QException
+   {
+      if(records.isEmpty())
+      {
+         return;
+      }
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(tableName);
+      updateInput.setRecords(records);
+      for(QRecord record : CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords()))
+      {
+         if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            throw (new QException("Could not update " + tableName + ": " + record.getErrors().get(0).getMessage()));
+         }
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigStore.java

@@ -0,0 +1,503 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The effective Quick Search configuration, read from the records: which
+ ** tables are indexed (quick_search_index), with which fields, weights,
+ ** roles and analyzers (quick_search_index_field), and the system settings
+ ** (quick_search_settings: the global pause).
+ **
+ ** Precedence: the records are the source of truth once the instance has the
+ ** config tables and priming has run. Code (annotations and searchableTables)
+ ** only seeds them (QuickSearchConfigPrimer). Until priming has succeeded,
+ ** or when the config tables are not in the instance or not readable (a host
+ ** that has not added their DDL yet), the code-declared tables are used, as
+ ** before 1.1.0-RC.3.
+ **
+ ** Reads are cached for CACHE_TTL_MILLIS and dropped at once by the config
+ ** tables' customizers when a row changes on this server; other servers pick
+ ** the change up when their cache expires.
+ *******************************************************************************/
+public class QuickSearchConfigStore
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchConfigStore.class);
+
+   public static final long CACHE_TTL_MILLIS   = 30_000L;
+   public static final long PRIME_RETRY_MILLIS = 300_000L;
+
+   private static final ThreadLocal<Boolean> BUSY = ThreadLocal.withInitial(() -> false);
+
+   private final QuickSearchRuntime ownerRuntime;
+
+   private volatile Snapshot snapshot;
+   private volatile QRecord  settings;
+   private volatile long     settingsReadAtMillis = 0L;
+   private volatile boolean  primed               = false;
+   private volatile long     lastPrimeAttemptMillis = 0L;
+   private volatile long     lastFailureLogMillis   = 0L;
+
+   private final Set<String> coreSearchFieldsDefaulted = ConcurrentHashMap.newKeySet();
+
+
+
+   /*******************************************************************************
+    ** A loaded set of effective table configs.
+    *******************************************************************************/
+   private record Snapshot(List<QuickSearchableTableConfig> tables, Map<String, QuickSearchableTableConfig> byName, long loadedAtMillis)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchConfigStore(QuickSearchRuntime runtime)
+   {
+      this.ownerRuntime = runtime;
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime the config hangs off (the one the store was made for, unless
+    ** the config has since been given another, such as a wrapper of it).
+    *******************************************************************************/
+   private QuickSearchRuntime runtime()
+   {
+      QuickSearchRuntime current = ownerRuntime.getConfig() == null ? null : ownerRuntime.getConfig().getRuntime();
+      return (current != null ? current : ownerRuntime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the instance in the current context has the config tables
+    ** (quick_search_index and quick_search_index_field).
+    *******************************************************************************/
+   public boolean hasConfigTables()
+   {
+      QInstance qInstance = QContext.getQInstance();
+      QuickSearchQBitConfig config = runtime().getConfig();
+      return (qInstance != null
+         && qInstance.getTable(config.getQuickSearchIndexTableName()) != null
+         && qInstance.getTable(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME)) != null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the effective config comes from the records (else from code).
+    *******************************************************************************/
+   public boolean isRecordsMode()
+   {
+      return (primed && hasConfigTables());
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether priming has succeeded in this runtime.
+    *******************************************************************************/
+   public boolean isPrimed()
+   {
+      return (primed);
+   }
+
+
+
+   /*******************************************************************************
+    ** Record that priming succeeded (or, for tests, reset it).
+    *******************************************************************************/
+   public void setPrimed(boolean primed)
+   {
+      this.primed                 = primed;
+      this.lastPrimeAttemptMillis = 0L;
+      invalidate();
+   }
+
+
+
+   /*******************************************************************************
+    ** The effective table configs: from the records in records mode, else the
+    ** code-declared ones. Primes first when that has not happened yet (at most
+    ** once per PRIME_RETRY_MILLIS while it keeps failing). Never throws.
+    *******************************************************************************/
+   public List<QuickSearchableTableConfig> getTables()
+   {
+      Snapshot current = getSnapshot();
+      return (current == null ? runtime().getCodeTables() : current.tables());
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's effective config, or null when it is not indexed.
+    *******************************************************************************/
+   public QuickSearchableTableConfig getTable(String tableName)
+   {
+      if(tableName == null)
+      {
+         return (null);
+      }
+      Snapshot current = getSnapshot();
+      if(current == null)
+      {
+         return (runtime().getCodeTableConfig(tableName));
+      }
+      return (current.byName().get(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** The current snapshot, loading it when stale; null means "use code".
+    *******************************************************************************/
+   private Snapshot getSnapshot()
+   {
+      if(BUSY.get() || !hasConfigTables())
+      {
+         return (null);
+      }
+
+      if(!primed)
+      {
+         tryPrime();
+         if(!primed)
+         {
+            return (null);
+         }
+      }
+
+      Snapshot current = snapshot;
+      if(current != null && System.currentTimeMillis() - current.loadedAtMillis() < CACHE_TTL_MILLIS)
+      {
+         return (current);
+      }
+
+      Snapshot loaded = load();
+      if(loaded != null)
+      {
+         snapshot = loaded;
+         return (loaded);
+      }
+      return (current);
+   }
+
+
+
+   /*******************************************************************************
+    ** Prime now, unless it already succeeded or failed less than
+    ** PRIME_RETRY_MILLIS ago. Never throws.
+    *******************************************************************************/
+   public synchronized void tryPrime()
+   {
+      if(primed || BUSY.get() || !hasConfigTables())
+      {
+         return;
+      }
+      long now = System.currentTimeMillis();
+      if(lastPrimeAttemptMillis != 0 && now - lastPrimeAttemptMillis < PRIME_RETRY_MILLIS)
+      {
+         return;
+      }
+
+      BUSY.set(true);
+      try
+      {
+         new QuickSearchConfigPrimer(runtime()).prime();
+         primed                 = true;
+         lastPrimeAttemptMillis = 0L;
+      }
+      catch(Exception | LinkageError e)
+      {
+         lastPrimeAttemptMillis = now;
+         LOG.warn("Quick Search could not prime its config records; using the code-declared tables until it can (retried every "
+            + (PRIME_RETRY_MILLIS / 60_000) + " minutes)", e);
+      }
+      finally
+      {
+         BUSY.set(false);
+         invalidate();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the cached configs and settings (a config row changed).
+    *******************************************************************************/
+   public void invalidate()
+   {
+      snapshot             = null;
+      settingsReadAtMillis = 0L;
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the records into a snapshot; null (logged, at most once a minute)
+    ** when they cannot be read.
+    *******************************************************************************/
+   private Snapshot load()
+   {
+      BUSY.set(true);
+      try
+      {
+         QuickSearchQBitConfig config    = runtime().getConfig();
+         QInstance             qInstance = QContext.getQInstance();
+
+         List<QRecord> indexRows = query(config.getQuickSearchIndexTableName());
+         List<QRecord> fieldRows = query(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME));
+
+         Map<Integer, List<QRecord>> fieldsByIndex = new LinkedHashMap<>();
+         for(QRecord fieldRow : fieldRows)
+         {
+            fieldsByIndex.computeIfAbsent(fieldRow.getValueInteger("quickSearchIndexId"), k -> new ArrayList<>()).add(fieldRow);
+         }
+
+         List<QuickSearchableTableConfig>        tables = new ArrayList<>();
+         Map<String, QuickSearchableTableConfig> byName = new LinkedHashMap<>();
+         for(QRecord indexRow : indexRows)
+         {
+            QuickSearchableTableConfig tableConfig = buildTableConfig(qInstance, config, indexRow, fieldsByIndex.getOrDefault(indexRow.getValueInteger("id"), List.of()));
+            if(tableConfig != null && !byName.containsKey(tableConfig.getTableName()))
+            {
+               tables.add(tableConfig);
+               byName.put(tableConfig.getTableName(), tableConfig);
+            }
+         }
+
+         ////////////////////////////////////////////////////////////////////////
+         // tables added at runtime get core searchFields (once per table), so //
+         // the global search covers them too                                  //
+         ////////////////////////////////////////////////////////////////////////
+         if(Boolean.TRUE.equals(config.getServeCoreRecordSearch()) && qInstance != null)
+         {
+            List<QuickSearchableTableConfig> unseen = tables.stream().filter(t -> coreSearchFieldsDefaulted.add(t.getTableName())).toList();
+            if(!unseen.isEmpty())
+            {
+               QuickSearchQBitProducer.defaultCoreSearchFields(qInstance, unseen);
+            }
+         }
+
+         return (new Snapshot(Collections.unmodifiableList(tables), Collections.unmodifiableMap(byName), System.currentTimeMillis()));
+      }
+      catch(Exception | LinkageError e)
+      {
+         long now = System.currentTimeMillis();
+         if(now - lastFailureLogMillis > 60_000L)
+         {
+            lastFailureLogMillis = now;
+            LOG.warn("Could not read the Quick Search config records; keeping the last config read", e);
+         }
+         return (null);
+      }
+      finally
+      {
+         BUSY.set(false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Build one table's effective config from its index row and field rows.
+    ** A table no longer in the instance is skipped; a field that is gone or
+    ** can no longer be indexed (hidden, sensitive) is dropped with a warning.
+    *******************************************************************************/
+   static QuickSearchableTableConfig buildTableConfig(QInstance qInstance, QuickSearchQBitConfig config, QRecord indexRow, List<QRecord> fieldRows)
+   {
+      String         tableName = indexRow.getValueString("tableName");
+      QTableMetaData table     = qInstance == null || tableName == null ? null : qInstance.getTable(tableName);
+      if(table == null)
+      {
+         LOG.debug("Quick Search index row names a table that is not in the instance; skipped", logPair("tableName", tableName));
+         return (null);
+      }
+
+      List<QRecord> sorted = new ArrayList<>(fieldRows);
+      sorted.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("sortOrder") == null ? Integer.MAX_VALUE : r.getValueInteger("sortOrder"))
+         .thenComparing(r -> r.getValueInteger("id") == null ? Integer.MAX_VALUE : r.getValueInteger("id")));
+
+      List<String>         fields         = new ArrayList<>();
+      Map<String, Integer> weights        = new LinkedHashMap<>();
+      Map<String, Boolean> includeLabels  = new LinkedHashMap<>();
+      Map<String, String>  roles          = new LinkedHashMap<>();
+      Map<String, String>  analyzers      = new LinkedHashMap<>();
+      Map<String, Boolean> snippetFlags   = new LinkedHashMap<>();
+      for(QRecord fieldRow : sorted)
+      {
+         String fieldName = fieldRow.getValueString("fieldName");
+         String problem   = QuickSearchFieldRules.whyNotIndexable(table, fieldName);
+         if(problem != null)
+         {
+            LOG.warn("Quick Search field row skipped: " + problem, logPair("tableName", tableName), logPair("fieldName", fieldName));
+            continue;
+         }
+         if(fields.contains(fieldName))
+         {
+            continue;
+         }
+
+         fields.add(fieldName);
+         weights.put(fieldName, fieldRow.getValueInteger("weight") == null ? 1 : fieldRow.getValueInteger("weight"));
+         includeLabels.put(fieldName, Boolean.TRUE.equals(fieldRow.getValueBoolean("includeLabel")));
+
+         ///////////////////////////////////////////////////////////////////////
+         // only non-defaults go in the maps, so a table of plain fields has  //
+         // the same searchableFieldsJson as its code config (no false drift) //
+         ///////////////////////////////////////////////////////////////////////
+         QuickSearchFieldRole role = QuickSearchFieldRole.fromName(fieldRow.getValueString("role"), QuickSearchFieldRole.BODY);
+         if(role != QuickSearchFieldRole.BODY)
+         {
+            roles.put(fieldName, role.name());
+         }
+         if(StringUtils.hasContent(fieldRow.getValueString("analyzer")))
+         {
+            analyzers.put(fieldName, fieldRow.getValueString("analyzer"));
+         }
+         if(fieldRow.getValueBoolean("includeInSnippet") != null)
+         {
+            snippetFlags.put(fieldName, fieldRow.getValueBoolean("includeInSnippet"));
+         }
+      }
+
+      String timestampField = indexRow.getValueString("basepullTimestampField");
+      if(timestampField != null && !QuickSearchFieldRules.isTimestampField(table, timestampField))
+      {
+         LOG.warn("Quick Search basepull timestamp field is not a date-time field of the table; incremental basepull is off for it",
+            logPair("tableName", tableName), logPair("timestampField", timestampField));
+         timestampField = null;
+      }
+
+      Integer interval = indexRow.getValueInteger("basepullIntervalMinutes");
+      Boolean enabled  = indexRow.getValueBoolean("enabled");
+
+      QuickSearchableTableConfig codeConfig = config.getRuntime() == null ? null : config.getRuntime().getCodeTableConfig(tableName);
+
+      return (new QuickSearchableTableConfig()
+         .withTableName(tableName)
+         .withPrimaryKeyField(table.getPrimaryKeyField())
+         .withSearchableFields(fields)
+         .withFieldWeights(weights)
+         .withFieldIncludeLabels(includeLabels)
+         .withFieldRoles(roles)
+         .withFieldAnalyzers(analyzers)
+         .withFieldIncludeInSnippet(snippetFlags)
+         .withBasepullIntervalMinutes(interval == null ? config.getDefaultBasepullIntervalMinutes() : interval)
+         .withBasepullTimestampField(timestampField)
+         .withEnabledByDefault(codeConfig == null ? Boolean.TRUE : codeConfig.getEnabledByDefault())
+         .withEnabled(enabled == null || enabled)
+         .withQuickSearchIndexId(indexRow.getValueInteger("id"))
+         .withRecordLabelFormat(codeConfig == null ? null : codeConfig.getRecordLabelFormat())
+         .withRecordLabelFields(codeConfig == null ? null : codeConfig.getRecordLabelFields())
+         .withMaxFieldLength(config.getMaxFieldLength()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings row (id 1), cached like the configs; null when the table is
+    ** not in the instance, the row does not exist, or it cannot be read.
+    *******************************************************************************/
+   public QRecord getSettings()
+   {
+      long now = System.currentTimeMillis();
+      if(settingsReadAtMillis != 0 && now - settingsReadAtMillis < CACHE_TTL_MILLIS)
+      {
+         return (settings);
+      }
+
+      QRecord read = null;
+      String  name = runtime().getConfig().applyTablePrefix(QuickSearchSettings.TABLE_NAME);
+      if(QContext.getQInstance() != null && QContext.getQInstance().getTable(name) != null)
+      {
+         try
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(name);
+            queryInput.setFilter(new QQueryFilter().withLimit(1));
+            List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+            read = rows.isEmpty() ? null : rows.get(0);
+         }
+         catch(Exception | LinkageError e)
+         {
+            LOG.debug("Could not read the Quick Search settings; treating indexing as not paused", e);
+         }
+      }
+      settings             = read;
+      settingsReadAtMillis = now;
+      return (read);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an admin has paused all indexing (quick_search_settings).
+    *******************************************************************************/
+   public boolean isIndexingPaused()
+   {
+      QRecord row = getSettings();
+      return (row != null && Boolean.TRUE.equals(row.getValueBoolean("indexingPaused")));
+   }
+
+
+
+   /*******************************************************************************
+    ** All rows of a config table.
+    *******************************************************************************/
+   private static List<QRecord> query(String tableName) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter());
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigWrites.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Marks the QBit's own writes to its config tables (priming, status updates
+ ** made on the owner's behalf), so the table customizers can tell them from
+ ** an admin's edits: only an admin's edit marks a row ownerEdited, writes a
+ ** tombstone, or flags the index for a rebuild.
+ *******************************************************************************/
+public final class QuickSearchConfigWrites
+{
+   private static final ThreadLocal<Boolean> SYSTEM_WRITE = ThreadLocal.withInitial(() -> false);
+
+
+
+   /*******************************************************************************
+    ** Work that may throw.
+    *******************************************************************************/
+   @FunctionalInterface
+   public interface Work<T>
+   {
+      /*******************************************************************************
+       ** Do it.
+       *******************************************************************************/
+      T run() throws QException;
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private QuickSearchConfigWrites()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Run work as a system write (nested calls stay system writes).
+    *******************************************************************************/
+   public static <T> T asSystem(Work<T> work) throws QException
+   {
+      boolean previous = SYSTEM_WRITE.get();
+      SYSTEM_WRITE.set(true);
+      try
+      {
+         return (work.run());
+      }
+      finally
+      {
+         SYSTEM_WRITE.set(previous);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the current thread is making a system write.
+    *******************************************************************************/
+   public static boolean isSystemWrite()
+   {
+      return (SYSTEM_WRITE.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** The current user's name for audit-style columns (pausedBy, removedBy):
+    ** full name, else id, else "system".
+    *******************************************************************************/
+   public static String currentUserName()
+   {
+      QSession session = QContext.getQSession();
+      QUser    user    = session == null ? null : session.getUser();
+      if(user != null && StringUtils.hasContent(user.getFullName()))
+      {
+         return (user.getFullName());
+      }
+      if(user != null && StringUtils.hasContent(user.getIdReference()))
+      {
+         return (user.getIdReference());
+      }
+      return ("system");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchFieldRules.java

@@ -0,0 +1,331 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.regex.Pattern;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+
+
+/*******************************************************************************
+ ** Which tables and fields may be indexed, and the defaults offered for a new
+ ** index. Shared by the Add Search Index process, the eligible-table and
+ ** field possible-value sources, the field table's validation and priming,
+ ** so all of them agree.
+ **
+ ** A table is eligible when it is in the instance, queryable, not hidden, not
+ ** on backend variants, not produced by a QBit (qbit-internal), not a QQQ
+ ** system table (audits, saved views and reports, scheduled jobs, scripts),
+ ** not named like a secret store, and has at least one indexable field. A
+ ** field is indexable when it is not hidden, not isSensitive, not a password
+ ** and not a blob or boolean.
+ *******************************************************************************/
+public final class QuickSearchFieldRules
+{
+   public static final int MAX_DEFAULT_FIELDS = 20;
+   public static final int TITLE_WEIGHT       = 3;
+
+   private static final Set<String> SYSTEM_TABLES = Set.of(
+      "audit", "auditdetail", "audittable", "audituser", "savedview", "savedreport", "sharedsavedview", "sharedsavedreport",
+      "scheduledjob", "scheduledjobparameter", "script", "scriptrevision", "scriptrevisionfile", "scriptlog", "scriptlogline",
+      "scripttype", "scripttypefilemode", "tabletrigger", "qqqtable", "qqqtablecache", "qqqprocess", "processtrace", "cachedrecord");
+
+   private static final Pattern SECRET_NAME = Pattern.compile("(secret|credential|password|passwd|token|api_?key|session|oauth|private_?key)", Pattern.CASE_INSENSITIVE);
+
+
+
+   /*******************************************************************************
+    ** A field picked by default for a new index.
+    *******************************************************************************/
+   public record DefaultField(String fieldName, Integer weight, QuickSearchFieldRole role)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private QuickSearchFieldRules()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Why a field cannot be indexed, or null when it can.
+    *******************************************************************************/
+   public static String whyNotIndexable(QTableMetaData table, String fieldName)
+   {
+      if(table == null)
+      {
+         return ("the table is not in this application");
+      }
+      QFieldMetaData field = fieldName == null ? null : table.getFields().get(fieldName);
+      if(field == null)
+      {
+         return ("field [" + fieldName + "] does not exist on " + label(table));
+      }
+      if(Boolean.TRUE.equals(field.getIsHidden()))
+      {
+         return ("field [" + fieldName + "] is hidden");
+      }
+      if(field.getIsSensitive() || (field.getType() != null && field.getType().needsMasked()))
+      {
+         return ("field [" + fieldName + "] is sensitive (its values must not reach the search index)");
+      }
+      if(field.getType() == null || field.getType() == QFieldType.BLOB || field.getType() == QFieldType.BOOLEAN)
+      {
+         return ("field [" + fieldName + "] is a " + (field.getType() == null ? "typeless" : field.getType().name().toLowerCase(Locale.ROOT)) + " field, which cannot be searched");
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field can be indexed.
+    *******************************************************************************/
+   public static boolean isIndexable(QTableMetaData table, String fieldName)
+   {
+      return (whyNotIndexable(table, fieldName) == null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The indexable fields of a table, in the table's order.
+    *******************************************************************************/
+   public static List<QFieldMetaData> indexableFields(QTableMetaData table)
+   {
+      List<QFieldMetaData> fields = new ArrayList<>();
+      if(table != null)
+      {
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            if(isIndexable(table, field.getName()))
+            {
+               fields.add(field);
+            }
+         }
+      }
+      return (fields);
+   }
+
+
+
+   /*******************************************************************************
+    ** The fields a new index starts with: the record-label fields as TITLE
+    ** (weight 3), then the other text fields (string, text, HTML) as BODY
+    ** (weight 1), up to MAX_DEFAULT_FIELDS. The admin adjusts them afterwards on
+    ** the index's field rows.
+    *******************************************************************************/
+   public static List<DefaultField> defaultFields(QTableMetaData table)
+   {
+      List<DefaultField> fields = new ArrayList<>();
+      Set<String>        picked = new LinkedHashSet<>();
+      if(table == null)
+      {
+         return (fields);
+      }
+
+      for(String labelField : CollectionUtils.nonNullList(table.getRecordLabelFields()))
+      {
+         if(isIndexable(table, labelField) && picked.add(labelField))
+         {
+            fields.add(new DefaultField(labelField, TITLE_WEIGHT, QuickSearchFieldRole.TITLE));
+         }
+      }
+
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(fields.size() >= MAX_DEFAULT_FIELDS)
+         {
+            break;
+         }
+         if(field.getType() != null && field.getType().isStringLike() && isIndexable(table, field.getName()) && picked.add(field.getName()))
+         {
+            fields.add(new DefaultField(field.getName(), 1, QuickSearchFieldRole.BODY));
+         }
+      }
+      return (fields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field can drive incremental basepull (a date-time).
+    *******************************************************************************/
+   public static boolean isTimestampField(QTableMetaData table, String fieldName)
+   {
+      QFieldMetaData field = table == null || fieldName == null ? null : table.getFields().get(fieldName);
+      return (field != null && field.getType() == QFieldType.DATE_TIME);
+   }
+
+
+
+   /*******************************************************************************
+    ** The timestamp field a new index starts with: modifyDate when the table
+    ** has it, else the first date-time field named like a modify or update
+    ** time, else none (incremental basepull off; reconcile covers the table).
+    *******************************************************************************/
+   public static String defaultTimestampField(QTableMetaData table)
+   {
+      if(table == null)
+      {
+         return (null);
+      }
+      if(isTimestampField(table, QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD))
+      {
+         return (QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD);
+      }
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         String name = field.getName().toLowerCase(Locale.ROOT);
+         if(field.getType() == QFieldType.DATE_TIME && (name.contains("modif") || name.contains("updat")))
+         {
+            return (field.getName());
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Why a table cannot be indexed at all (for anyone), or null when it can.
+    *******************************************************************************/
+   public static String whyNotEligible(QInstance qInstance, QuickSearchQBitConfig config, String tableName)
+   {
+      QTableMetaData table = qInstance == null || tableName == null ? null : qInstance.getTable(tableName);
+      if(table == null)
+      {
+         return ("table [" + tableName + "] is not in this application");
+      }
+      if(isQuickSearchTable(config, tableName) || table.getSourceQBitName() != null)
+      {
+         return (label(table) + " is internal to a QBit");
+      }
+      String normalized = tableName.replace("_", "").toLowerCase(Locale.ROOT);
+      if(SYSTEM_TABLES.contains(normalized))
+      {
+         return (label(table) + " is a system table");
+      }
+      if(SECRET_NAME.matcher(tableName).find())
+      {
+         return (label(table) + " looks like a store of secrets");
+      }
+      if(table.getIsHidden())
+      {
+         return (label(table) + " is hidden");
+      }
+      QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
+      if(backend != null && Boolean.TRUE.equals(backend.getUsesVariants()))
+      {
+         return (label(table) + " uses backend variants, which Quick Search does not support");
+      }
+      if(!table.isCapabilityEnabled(backend, Capability.TABLE_QUERY))
+      {
+         return (label(table) + " cannot be queried");
+      }
+      if(indexableFields(table).isEmpty())
+      {
+         return (label(table) + " has no field that can be searched");
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the current user may read a table.
+    *******************************************************************************/
+   public static boolean canRead(String tableName)
+   {
+      try
+      {
+         return (PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
+      }
+      catch(Exception e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables the current user could add: eligible, readable and not
+    ** already indexed, sorted by label.
+    *******************************************************************************/
+   public static List<QTableMetaData> eligibleTables(QInstance qInstance, QuickSearchQBitConfig config, Collection<String> alreadyIndexed)
+   {
+      List<QTableMetaData> tables = new ArrayList<>();
+      if(qInstance == null)
+      {
+         return (tables);
+      }
+      for(QTableMetaData table : qInstance.getTables().values())
+      {
+         if(alreadyIndexed.contains(table.getName()) || whyNotEligible(qInstance, config, table.getName()) != null || !canRead(table.getName()))
+         {
+            continue;
+         }
+         tables.add(table);
+      }
+      tables.sort((a, b) -> label(a).compareToIgnoreCase(label(b)));
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table is one of this QBit's own tables.
+    *******************************************************************************/
+   public static boolean isQuickSearchTable(QuickSearchQBitConfig config, String tableName)
+   {
+      return (tableName != null && config != null && tableName.startsWith(config.applyTablePrefix("quick_search_")));
+   }
+
+
+
+   /*******************************************************************************
+    ** A table's label, or its name.
+    *******************************************************************************/
+   public static String label(QTableMetaData table)
+   {
+      return (table.getLabel() == null ? table.getName() : table.getLabel());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchIndexFieldTableCustomizer.java

@@ -0,0 +1,425 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+
+
+/*******************************************************************************
+ ** Customizer of quick_search_index_field, the searchable fields.
+ **
+ ** An admin's insert or edit is validated: the index row exists, the field
+ ** exists on its table and can be indexed (not hidden, not isSensitive, not a
+ ** password, blob or boolean), it is not already a field of that index, the
+ ** weight is 1 to 1000, and the role and analyzer are known. The row is marked
+ ** ownerEdited (an insert also source OWNER), so priming never overwrites it.
+ **
+ ** Changes take effect for searches and real-time indexing at once (weights
+ ** are applied at query time). After an admin's change other than the sort
+ ** order, the index row is marked NEEDS_REINDEX, so the documents indexed
+ ** before can be brought in line: Apply Search Index Changes previews the
+ ** change and rebuilds the table. Removing a field leaves a tombstone, and
+ ** adding it back clears it. Every change drops the cached config.
+ *******************************************************************************/
+public class QuickSearchIndexFieldTableCustomizer implements TableCustomizerInterface
+{
+   public static final int MAX_WEIGHT = 1000;
+
+   public static final List<String> OWNER_FIELDS    = List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel", "sortOrder");
+   public static final List<String> DOCUMENT_FIELDS = List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel");
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      Set<String> inBatch = new HashSet<>();
+      for(QRecord record : records)
+      {
+         Integer indexId = record.getValueInteger("quickSearchIndexId");
+         QRecord index   = indexRow(indexId);
+         if(index == null)
+         {
+            record.addError(new BadInputStatusMessage("Choose the search index this field belongs to"));
+            continue;
+         }
+
+         String fieldName = record.getValueString("fieldName");
+         if(!validateField(record, index, fieldName, null) || !inBatch.add(indexId + "." + fieldName))
+         {
+            if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+            {
+               record.addError(new BadInputStatusMessage("Field [" + fieldName + "] is listed twice"));
+            }
+            continue;
+         }
+
+         if(record.getValue("weight") == null)
+         {
+            record.setValue("weight", 1);
+         }
+         if(!StringUtils.hasContent(record.getValueString("role")))
+         {
+            record.setValue("role", QuickSearchFieldRole.BODY.name());
+         }
+         if(record.getValue("includeLabel") == null)
+         {
+            record.setValue("includeLabel", false);
+         }
+         if(record.getValue("sortOrder") == null)
+         {
+            record.setValue("sortOrder", nextSortOrder(indexId));
+         }
+         validateOptions(record);
+         record.setValue("source", QuickSearchIndexField.SOURCE_OWNER);
+         record.setValue("ownerEdited", true);
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old == null)
+         {
+            continue;
+         }
+         if(record.getValues().containsKey("quickSearchIndexId") && !Objects.equals(record.getValueInteger("quickSearchIndexId"), old.getValueInteger("quickSearchIndexId")))
+         {
+            record.addError(new BadInputStatusMessage("A field cannot be moved to another search index; remove it and add it there"));
+            continue;
+         }
+         if(!changes(record, old, OWNER_FIELDS))
+         {
+            continue;
+         }
+         if(record.getValues().containsKey("fieldName") && !Objects.equals(record.getValueString("fieldName"), old.getValueString("fieldName")))
+         {
+            validateField(record, indexRow(old.getValueInteger("quickSearchIndexId")), record.getValueString("fieldName"), old.getValue("id"));
+         }
+         validateOptions(record);
+         record.setValue("ownerEdited", true);
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate a field name against its index's table and the index's other
+    ** fields (excludeId: the row being edited). Adds an error and returns false
+    ** when it is not acceptable.
+    *******************************************************************************/
+   private static boolean validateField(QRecord record, QRecord index, String fieldName, Serializable excludeId) throws QException
+   {
+      if(index == null)
+      {
+         record.addError(new BadInputStatusMessage("The search index of this field no longer exists"));
+         return (false);
+      }
+      String         tableName = index.getValueString("tableName");
+      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+      String         problem   = QuickSearchFieldRules.whyNotIndexable(table, fieldName);
+      if(problem != null)
+      {
+         record.addError(new BadInputStatusMessage("Cannot index " + fieldName + ": " + problem));
+         return (false);
+      }
+
+      QQueryFilter filter = new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, index.getValue("id")))
+         .withCriteria(new QFilterCriteria("fieldName", QCriteriaOperator.EQUALS, fieldName));
+      if(excludeId != null)
+      {
+         filter.withCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, excludeId));
+      }
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(fieldTableName());
+      queryInput.setFilter(filter);
+      if(CollectionUtils.nullSafeHasContents(new QueryAction().execute(queryInput).getRecords()))
+      {
+         record.addError(new BadInputStatusMessage(fieldName + " is already a searchable field of " + tableName));
+         return (false);
+      }
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Weight 1 to MAX_WEIGHT; a known role and analyzer.
+    *******************************************************************************/
+   private static void validateOptions(QRecord record)
+   {
+      Integer weight = record.getValueInteger("weight");
+      if(record.getValues().containsKey("weight") && (weight == null || weight < 1 || weight > MAX_WEIGHT))
+      {
+         record.addError(new BadInputStatusMessage("Weight must be from 1 to " + MAX_WEIGHT));
+      }
+      String role = record.getValueString("role");
+      if(StringUtils.hasContent(role) && QuickSearchFieldRole.fromName(role, null) == null)
+      {
+         record.addError(new BadInputStatusMessage("Unknown role [" + role + "]"));
+      }
+      String analyzer = record.getValueString("analyzer");
+      if(StringUtils.hasContent(analyzer) && QuickSearchFieldAnalyzer.fromName(analyzer, null) == null)
+      {
+         record.addError(new BadInputStatusMessage("Unknown analyzer [" + analyzer + "]"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         for(QRecord record : succeeded(records))
+         {
+            QRecord index = indexRow(record.getValueInteger("quickSearchIndexId"));
+            if(index != null)
+            {
+               QuickSearchTombstones.remove(index.getValueString("tableName"), record.getValueString("fieldName"));
+               markNeedsReindex(index);
+            }
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+         Map<Integer, QRecord>      marked  = new HashMap<>();
+         for(QRecord record : succeeded(records))
+         {
+            QRecord old = oldById.get(record.getValue("id"));
+            if(old != null && changes(record, old, DOCUMENT_FIELDS))
+            {
+               Integer indexId = old.getValueInteger("quickSearchIndexId");
+               if(!marked.containsKey(indexId))
+               {
+                  QRecord index = indexRow(indexId);
+                  marked.put(indexId, index);
+                  markNeedsReindex(index);
+               }
+            }
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** An admin's removal leaves a tombstone (unless the whole index is going).
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         for(QRecord record : succeeded(records))
+         {
+            QRecord index = indexRow(record.getValueInteger("quickSearchIndexId"));
+            if(index != null)
+            {
+               QuickSearchTombstones.add(index.getValueString("tableName"), record.getValueString("fieldName"), "removed by an admin");
+               markNeedsReindex(index);
+            }
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Mark the index NEEDS_REINDEX (a system write), unless a build is already
+    ** queued or running.
+    *******************************************************************************/
+   static void markNeedsReindex(QRecord index) throws QException
+   {
+      if(index == null)
+      {
+         return;
+      }
+      String status = index.getValueString("status");
+      if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status)
+         || AbstractIndexingStep.STATUS_REBUILDING.equals(status) || AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+      {
+         return;
+      }
+      QuickSearchConfigWrites.asSystem(() ->
+      {
+         QuickSearchConfigPrimer.updateAll(QuickSearchTablePossibleValueProviders.config().getQuickSearchIndexTableName(),
+            List.of(new QRecord().withValue("id", index.getValue("id")).withValue("status", AbstractIndexingStep.STATUS_NEEDS_REINDEX)));
+         return (null);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the update changes any of the given fields.
+    *******************************************************************************/
+   private static boolean changes(QRecord record, QRecord old, List<String> fields)
+   {
+      for(String field : fields)
+      {
+         if(record.getValues().containsKey(field) && !Objects.equals(normalize(record.getValue(field)), normalize(old.getValue(field))))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare values as strings, blank as null.
+    *******************************************************************************/
+   private static String normalize(Serializable value)
+   {
+      String s = value == null ? null : String.valueOf(value);
+      return (StringUtils.hasContent(s) ? s : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The records without errors.
+    *******************************************************************************/
+   private static List<QRecord> succeeded(List<QRecord> records)
+   {
+      return (CollectionUtils.nonNullList(records).stream().filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** The index row, or null.
+    *******************************************************************************/
+   static QRecord indexRow(Integer indexId) throws QException
+   {
+      if(indexId == null)
+      {
+         return (null);
+      }
+      GetInput getInput = new GetInput(QuickSearchTablePossibleValueProviders.config().getQuickSearchIndexTableName());
+      getInput.setPrimaryKey(indexId);
+      return (new GetAction().execute(getInput).getRecord());
+   }
+
+
+
+   /*******************************************************************************
+    ** One past the index's highest sortOrder, in steps of 10.
+    *******************************************************************************/
+   private static Integer nextSortOrder(Integer indexId) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(fieldTableName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexId)));
+      int max = 0;
+      for(QRecord row : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+      {
+         max = Math.max(max, row.getValueInteger("sortOrder") == null ? 0 : row.getValueInteger("sortOrder"));
+      }
+      return (max + 10);
+   }
+
+
+
+   /*******************************************************************************
+    ** The field table's name.
+    *******************************************************************************/
+   private static String fieldTableName()
+   {
+      QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
+      return (config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchIndexTableCustomizer.java

@@ -0,0 +1,239 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+
+
+/*******************************************************************************
+ ** Customizer of quick_search_index, the searchable-table registry.
+ **
+ ** Rows are added by Add Search Index, which validates the table (eligible,
+ ** readable, not yet indexed), previews and queues the first build, and by
+ ** priming; the generic insert is turned off. Any other insert is marked
+ ** source OWNER and ownerEdited, and its schedule validated. An admin's edit of the
+ ** enabled flag, interval or timestamp field marks the row ownerEdited, so
+ ** priming never overwrites it; turning enabled back on queues a build. Any
+ ** change drops the cached config, so it takes effect at once on this server.
+ ** The QBit's own writes (QuickSearchConfigWrites) pass straight through.
+ *******************************************************************************/
+public class QuickSearchIndexTableCustomizer implements TableCustomizerInterface
+{
+   public static final List<String> CONFIG_FIELDS = List.of("enabled", "basepullIntervalMinutes", "basepullTimestampField");
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // rows are added by Add Search Index (which validates and previews) and  //
+      // by priming; the generic insert is turned off. Anything else inserting  //
+      // a row is the owner's doing: mark it so priming never overwrites it     //
+      ////////////////////////////////////////////////////////////////////////////
+      for(QRecord record : records)
+      {
+         if(record.getValue("source") == null)
+         {
+            record.setValue("source", QuickSearchIndex.SOURCE_OWNER);
+         }
+         if(record.getValue("ownerEdited") == null)
+         {
+            record.setValue("ownerEdited", true);
+         }
+         String tableName = record.getValueString("tableName");
+         if(QContext.getQInstance().getTable(tableName) != null)
+         {
+            validateSchedule(record, tableName);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old == null || !changesConfig(record, old))
+         {
+            continue;
+         }
+
+         validateSchedule(record, old.getValueString("tableName"));
+         record.setValue("ownerEdited", true);
+
+         boolean wasEnabled = !Boolean.FALSE.equals(old.getValueBoolean("enabled"));
+         boolean nowEnabled = !Boolean.FALSE.equals(record.getValues().containsKey("enabled") ? record.getValueBoolean("enabled") : old.getValueBoolean("enabled"));
+         String  status     = old.getValueString("status");
+         if(!wasEnabled && nowEnabled && !AbstractIndexingStep.STATUS_REBUILDING.equals(status) && !AbstractIndexingStep.STATUS_BUILDING.equals(status))
+         {
+            record.setValue("status", AbstractIndexingStep.STATUS_PENDING_BUILD);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the update changes the enabled flag, interval or timestamp field.
+    *******************************************************************************/
+   static boolean changesConfig(QRecord record, QRecord old)
+   {
+      for(String field : CONFIG_FIELDS)
+      {
+         if(record.getValues().containsKey(field) && !Objects.equals(normalize(record.getValue(field)), normalize(old.getValue(field))))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare values as strings, blank as null (forms send "" and "true").
+    *******************************************************************************/
+   private static String normalize(Serializable value)
+   {
+      String s = value == null ? null : String.valueOf(value);
+      return (StringUtils.hasContent(s) ? s : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** A timestamp field must be a date-time field of the table; the interval
+    ** cannot be negative.
+    *******************************************************************************/
+   private static void validateSchedule(QRecord record, String tableName)
+   {
+      String timestampField = record.getValueString("basepullTimestampField");
+      if(StringUtils.hasContent(timestampField) && !QuickSearchFieldRules.isTimestampField(QContext.getQInstance().getTable(tableName), timestampField))
+      {
+         record.addError(new BadInputStatusMessage("Timestamp field [" + timestampField + "] is not a date-time field of " + tableName));
+      }
+      Integer interval = record.getValueInteger("basepullIntervalMinutes");
+      if(record.getValues().containsKey("basepullIntervalMinutes") && interval != null && interval < 0)
+      {
+         record.addError(new BadInputStatusMessage("Update interval cannot be negative (0 means every update)"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** An admin's delete leaves a tombstone, so code defaults do not add the
+    ** table back.
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         for(QRecord record : CollectionUtils.nonNullList(records))
+         {
+            if(CollectionUtils.nullSafeIsEmpty(record.getErrors()) && record.getValueString("tableName") != null)
+            {
+               QuickSearchTombstones.add(record.getValueString("tableName"), null, "removed by an admin");
+            }
+         }
+      }
+      invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the runtime's cached config.
+    *******************************************************************************/
+   static void invalidate()
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime != null)
+      {
+         runtime.invalidateConfig();
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchSettingsTableCustomizer.java

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Customizer of quick_search_settings. Pausing stamps pausedAt and pausedBy
+ ** (the Quick Search home shows "Paused since ... by ..."); resuming clears
+ ** them and hands the changes deferred while paused (DEFERRED failed events)
+ ** to the next basepull (PENDING). Every change drops the cached settings, so
+ ** a pause takes effect at once on this server and within
+ ** QuickSearchConfigStore.CACHE_TTL_MILLIS on the others.
+ *******************************************************************************/
+public class QuickSearchSettingsTableCustomizer implements TableCustomizerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchSettingsTableCustomizer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old == null || !record.getValues().containsKey("indexingPaused"))
+         {
+            continue;
+         }
+         boolean wasPaused = Boolean.TRUE.equals(old.getValueBoolean("indexingPaused"));
+         boolean nowPaused = Boolean.TRUE.equals(record.getValueBoolean("indexingPaused"));
+         if(!wasPaused && nowPaused)
+         {
+            record.setValue("pausedAt", Instant.now());
+            record.setValue("pausedBy", QuickSearchConfigWrites.currentUserName());
+         }
+         else if(wasPaused && !nowPaused)
+         {
+            record.setValue("pausedAt", null);
+            record.setValue("pausedBy", null);
+            record.setValue("pauseReason", null);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old != null && CollectionUtils.nullSafeIsEmpty(record.getErrors()) && Boolean.TRUE.equals(old.getValueBoolean("indexingPaused"))
+            && record.getValues().containsKey("indexingPaused") && !Boolean.TRUE.equals(record.getValueBoolean("indexingPaused")))
+         {
+            releaseDeferredEvents();
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Make the events deferred while paused PENDING, so the next basepull
+    ** replays them. Never throws (they stay DEFERRED; a reconcile covers them).
+    *******************************************************************************/
+   static void releaseDeferredEvents()
+   {
+      try
+      {
+         QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
+         String                table  = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+         if(QContext.getQInstance().getTable(table) == null)
+         {
+            return;
+         }
+
+         while(true)
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table);
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_DEFERRED)).withLimit(1000));
+            List<QRecord> deferred = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+            if(deferred.isEmpty())
+            {
+               return;
+            }
+            List<QRecord> updates = new ArrayList<>();
+            deferred.forEach(r -> updates.add(new QRecord().withValue("id", r.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING)));
+            QuickSearchConfigPrimer.updateAll(table, updates);
+            LOG.info("Released changes deferred while Quick Search indexing was paused", logPair("count", updates.size()));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not release the changes deferred while indexing was paused; the next reconcile covers them", e);
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchStatus.java

@@ -0,0 +1,208 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+
+
+/*******************************************************************************
+ ** The indexing status for the Quick Search home: the global pause ("Paused
+ ** since ... by ..."), the last self-priming report ("New: 2 tables added and
+ ** building..."), the tables queued or building, and those whose field
+ ** changes wait for Apply Search Index Changes. The QBit's diagnostics widget
+ ** shows it; hosts with their own home can use read() and bannerHtml().
+ **
+ ** @param paused             whether all indexing is paused
+ ** @param pausedAt           since when (null when not paused)
+ ** @param pausedBy           by whom
+ ** @param pauseReason        the reason given, if any
+ ** @param lastPrimedAt       when self-priming last ran
+ ** @param lastPrimingSummary what it last changed
+ ** @param building           labels of tables queued for or in a build
+ ** @param needsReindex       labels of tables with field changes to apply
+ ** @param recordsMode        whether the config comes from the records (else
+ **                           code: the config tables are missing or not primed)
+ *******************************************************************************/
+public record QuickSearchStatus(boolean paused, Instant pausedAt, String pausedBy, String pauseReason, Instant lastPrimedAt, String lastPrimingSummary,
+                                List<String> building, List<String> needsReindex, boolean recordsMode)
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchStatus.class);
+
+   private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
+
+
+
+   /*******************************************************************************
+    ** Read the status. Never throws; parts that cannot be read are empty.
+    *******************************************************************************/
+   public static QuickSearchStatus read(QuickSearchRuntime runtime)
+   {
+      List<String> building     = new ArrayList<>();
+      List<String> needsReindex = new ArrayList<>();
+      if(runtime == null)
+      {
+         return (new QuickSearchStatus(false, null, null, null, null, null, building, needsReindex, false));
+      }
+
+      try
+      {
+         String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+         if(QContext.getQInstance() != null && QContext.getQInstance().getTable(indexTable) != null)
+         {
+            for(QRecord row : QuickSearchConfigPrimer.queryAll(indexTable))
+            {
+               String status = row.getValueString("status");
+               String label  = labelOf(row.getValueString("tableName"));
+               if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
+               {
+                  continue;
+               }
+               if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status)
+                  || AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+               {
+                  building.add(label);
+               }
+               else if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+               {
+                  needsReindex.add(label);
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not read the index rows for the Quick Search status", e);
+      }
+
+      QRecord settings = runtime.getConfigStore().getSettings();
+      boolean paused   = settings != null && Boolean.TRUE.equals(settings.getValueBoolean("indexingPaused"));
+      return (new QuickSearchStatus(paused,
+         settings == null ? null : settings.getValueInstant("pausedAt"),
+         settings == null ? null : settings.getValueString("pausedBy"),
+         settings == null ? null : settings.getValueString("pauseReason"),
+         settings == null ? null : settings.getValueInstant("lastPrimedAt"),
+         settings == null ? null : settings.getValueString("lastPrimingSummary"),
+         building, needsReindex, runtime.getConfigStore().isRecordsMode()));
+   }
+
+
+
+   /*******************************************************************************
+    ** "Paused since 2026-10-12T14:03:00Z by Jane Admin (reason)".
+    *******************************************************************************/
+   public String pausedText()
+   {
+      return ("Paused since " + (pausedAt == null ? "an unknown time" : WHEN.format(pausedAt)) + " by " + (pausedBy == null ? "an unknown user" : pausedBy)
+         + (StringUtils.hasContent(pauseReason) ? " (" + pauseReason + ")" : "")
+         + ". Nothing is indexed; search keeps serving the current index, and changes are indexed when indexing resumes.");
+   }
+
+
+
+   /*******************************************************************************
+    ** The building line: "Building: Blog Post, Page", or null.
+    *******************************************************************************/
+   public String buildingText()
+   {
+      if(building.isEmpty())
+      {
+         return (null);
+      }
+      return ((StringUtils.hasContent(lastPrimingSummary) && lastPrimingSummary.startsWith("New:") ? lastPrimingSummary + " " : "")
+         + "Building now or with the next update: " + String.join(", ", building) + ".");
+   }
+
+
+
+   /*******************************************************************************
+    ** The pending-changes line, or null.
+    *******************************************************************************/
+   public String needsReindexText()
+   {
+      return (needsReindex.isEmpty() ? null : "Field changes waiting to be applied (Apply Search Index Changes): " + String.join(", ", needsReindex) + ".");
+   }
+
+
+
+   /*******************************************************************************
+    ** The status as standalone HTML, for a host's own home widget: one line
+    ** per item, empty when there is nothing to report.
+    *******************************************************************************/
+   public String bannerHtml()
+   {
+      StringBuilder html = new StringBuilder();
+      if(paused)
+      {
+         html.append(banner("#b26a00", "#fff4e5", pausedText()));
+      }
+      if(buildingText() != null)
+      {
+         html.append(banner("#0b5394", "#e8f0fe", buildingText()));
+      }
+      if(needsReindexText() != null)
+      {
+         html.append(banner("#5f6368", "#f1f3f4", needsReindexText()));
+      }
+      return (html.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** One banner line.
+    *******************************************************************************/
+   private static String banner(String color, String background, String text)
+   {
+      return ("<div style=\"padding:8px 12px;margin:0 0 8px 0;border-radius:6px;border-left:4px solid " + color + ";background:" + background + ";color:#202124\">"
+         + DiagnosticsHtml.escape(text) + "</div>");
+   }
+
+
+
+   /*******************************************************************************
+    ** A table's label, or its name.
+    *******************************************************************************/
+   private static String labelOf(String tableName)
+   {
+      var table = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+      return (table == null ? String.valueOf(tableName) : QuickSearchFieldRules.label(table));
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings table name for links ("/app/" + name + "/1").
+    *******************************************************************************/
+   public static String settingsPath(QuickSearchRuntime runtime)
+   {
+      return ("/app/" + runtime.getConfig().applyTablePrefix(QuickSearchSettings.TABLE_NAME) + "/" + QuickSearchSettings.SETTINGS_ID);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchTablePossibleValueProviders.java

@@ -0,0 +1,321 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+
+
+/*******************************************************************************
+ ** Custom possible-value providers over the instance's tables and fields, for
+ ** the config screens:
+ ** - SearchableTables: the indexed tables (quick_search_searchable_table),
+ **   shown by label wherever a table name is stored;
+ ** - EligibleTables: tables the current user could add (readable, eligible,
+ **   not yet indexed), for Add Search Index;
+ ** - IndexableFields and TimestampFields: the indexable or date-time fields
+ **   of the table the form is about, read from the form's other values
+ **   (tableName, or the field row's quickSearchIndexId).
+ *******************************************************************************/
+public final class QuickSearchTablePossibleValueProviders
+{
+
+   /*******************************************************************************
+    ** Utility holder - no instances.
+    *******************************************************************************/
+   private QuickSearchTablePossibleValueProviders()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The QBit's config: the runtime's, else the static facade's.
+    *******************************************************************************/
+   static QuickSearchQBitConfig config()
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig());
+   }
+
+
+
+   /*******************************************************************************
+    ** Table possible values whose id is the table name and label its label.
+    *******************************************************************************/
+   abstract static class AbstractTableProvider implements QCustomPossibleValueProvider<String>
+   {
+      /*******************************************************************************
+       ** The table names to offer.
+       *******************************************************************************/
+      abstract List<String> tableNames() throws QException;
+
+
+
+      /*******************************************************************************
+       ** Any table in the instance, by label.
+       *******************************************************************************/
+      @Override
+      public QPossibleValue<String> getPossibleValue(Serializable idValue)
+      {
+         String         tableName = ValueUtils.getValueAsString(idValue);
+         QTableMetaData table     = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+         return (tableName == null ? null : new QPossibleValue<>(tableName, table == null ? tableName : QuickSearchFieldRules.label(table)));
+      }
+
+
+
+      /*******************************************************************************
+       ** The offered tables matching the search term (or ids).
+       *******************************************************************************/
+      @Override
+      public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+      {
+         List<QPossibleValue<String>> values = new ArrayList<>();
+         for(String tableName : tableNames())
+         {
+            QPossibleValue<String> value = getPossibleValue(tableName);
+            if(matches(input, value))
+            {
+               values.add(value);
+            }
+         }
+         return (values);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The indexed tables.
+    *******************************************************************************/
+   public static class SearchableTables extends AbstractTableProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      List<String> tableNames()
+      {
+         QuickSearchRuntime runtime = QuickSearchRuntime.get();
+         List<String>       names   = new ArrayList<>();
+         List<QuickSearchableTableConfig> tables = runtime != null ? runtime.getDiscoveredTables() : QuickSearchQBitContext.getDiscoveredTables();
+         for(QuickSearchableTableConfig tableConfig : CollectionUtils.nonNullList(tables))
+         {
+            names.add(tableConfig.getTableName());
+         }
+         return (names);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables the current user could add to Quick Search.
+    *******************************************************************************/
+   public static class EligibleTables extends AbstractTableProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      List<String> tableNames() throws QException
+      {
+         QuickSearchQBitConfig config  = config();
+         Set<String>           indexed = new LinkedHashSet<>();
+         for(QRecord row : QuickSearchConfigPrimer.queryAll(config.getQuickSearchIndexTableName()))
+         {
+            indexed.add(row.getValueString("tableName"));
+         }
+         List<String> names = new ArrayList<>();
+         QuickSearchFieldRules.eligibleTables(QContext.getQInstance(), config, indexed).forEach(t -> names.add(t.getName()));
+         return (names);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Field possible values of the form's table; id is the field name.
+    *******************************************************************************/
+   abstract static class AbstractFieldProvider implements QCustomPossibleValueProvider<String>
+   {
+      /*******************************************************************************
+       ** Whether to offer a field.
+       *******************************************************************************/
+      abstract boolean offers(QTableMetaData table, QFieldMetaData field);
+
+
+
+      /*******************************************************************************
+       ** A field name is shown as itself: without the form's values the table is
+       ** not known here.
+       *******************************************************************************/
+      @Override
+      public QPossibleValue<String> getPossibleValue(Serializable idValue)
+      {
+         String fieldName = ValueUtils.getValueAsString(idValue);
+         return (fieldName == null ? null : new QPossibleValue<>(fieldName, fieldName));
+      }
+
+
+
+      /*******************************************************************************
+       ** The offered fields of the form's table, labeled "Label (name)".
+       *******************************************************************************/
+      @Override
+      public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+      {
+         List<QPossibleValue<String>> values = new ArrayList<>();
+         QTableMetaData               table  = resolveTable(input.getOtherValues());
+         if(table == null)
+         {
+            return (values);
+         }
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            if(!offers(table, field))
+            {
+               continue;
+            }
+            QPossibleValue<String> value = new QPossibleValue<>(field.getName(), (field.getLabel() == null ? field.getName() : field.getLabel()) + " (" + field.getName() + ")");
+            if(matches(input, value))
+            {
+               values.add(value);
+            }
+         }
+         return (values);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Fields that can be indexed.
+    *******************************************************************************/
+   public static class IndexableFields extends AbstractFieldProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      boolean offers(QTableMetaData table, QFieldMetaData field)
+      {
+         return (QuickSearchFieldRules.isIndexable(table, field.getName()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Date-time fields, for the basepull timestamp.
+    *******************************************************************************/
+   public static class TimestampFields extends AbstractFieldProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      boolean offers(QTableMetaData table, QFieldMetaData field)
+      {
+         return (QuickSearchFieldRules.isTimestampField(table, field.getName()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The table a form is about: its searchTableName or tableName value, else
+    ** the table of its quickSearchIndexId row. Null when none resolves.
+    *******************************************************************************/
+   static QTableMetaData resolveTable(Map<String, Serializable> otherValues) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(otherValues == null || qInstance == null)
+      {
+         return (null);
+      }
+
+      ////////////////////////////////////////////////////////////////////////
+      // Add Search Index's searchTableName first: a process form also gets  //
+      // tableName, the table the process runs from                          //
+      ////////////////////////////////////////////////////////////////////////
+      String tableName = ValueUtils.getValueAsString(otherValues.get("searchTableName"));
+      if(!StringUtils.hasContent(tableName))
+      {
+         tableName = ValueUtils.getValueAsString(otherValues.get("tableName"));
+      }
+      if(!StringUtils.hasContent(tableName) && otherValues.get("quickSearchIndexId") != null)
+      {
+         GetInput getInput = new GetInput(config().getQuickSearchIndexTableName());
+         getInput.setPrimaryKey(ValueUtils.getValueAsInteger(otherValues.get("quickSearchIndexId")));
+         QRecord row = new GetAction().execute(getInput).getRecord();
+         tableName = row == null ? null : row.getValueString("tableName");
+      }
+      return (StringUtils.hasContent(tableName) ? qInstance.getTable(tableName) : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a value is wanted: in idList when one is given, else its label or
+    ** id contains the search term (any case).
+    *******************************************************************************/
+   static boolean matches(SearchPossibleValueSourceInput input, QPossibleValue<String> value)
+   {
+      if(value == null)
+      {
+         return (false);
+      }
+      if(CollectionUtils.nullSafeHasContents(input.getIdList()))
+      {
+         return (input.getIdList().stream().anyMatch(id -> value.getId().equals(ValueUtils.getValueAsString(id))));
+      }
+      String term = input.getSearchTerm();
+      if(!StringUtils.hasContent(term))
+      {
+         return (true);
+      }
+      String lower = term.toLowerCase(Locale.ROOT);
+      return (value.getLabel().toLowerCase(Locale.ROOT).contains(lower) || value.getId().toLowerCase(Locale.ROOT).contains(lower));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchTombstones.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+
+
+/*******************************************************************************
+ ** Reads and writes quick_search_config_tombstone: what an admin removed, so
+ ** priming does not add it back. Every write is a system write. With no
+ ** tombstone table in the instance these do nothing (and priming then cannot
+ ** tell a removed code table from a new one; add the table's DDL).
+ *******************************************************************************/
+public final class QuickSearchTombstones
+{
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private QuickSearchTombstones()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The tombstone table's name, or null when it is not in the instance.
+    *******************************************************************************/
+   static String tableName()
+   {
+      QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
+      String                name   = config == null ? null : config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME);
+      return (name != null && QContext.getQInstance() != null && QContext.getQInstance().getTable(name) != null ? name : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Remember that a table (fieldName null) or one of its fields was removed.
+    *******************************************************************************/
+   public static void add(String tableName, String fieldName, String reason) throws QException
+   {
+      String name = tableName();
+      if(name == null || !find(name, tableName, fieldName).isEmpty())
+      {
+         return;
+      }
+      QuickSearchConfigWrites.asSystem(() -> QuickSearchConfigPrimer.insertOne(name, new QRecord()
+         .withValue("tableName", tableName)
+         .withValue("fieldName", fieldName)
+         .withValue("removedBy", QuickSearchConfigWrites.currentUserName())
+         .withValue("reason", reason)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget a removal (the admin added the table or field back).
+    *******************************************************************************/
+   public static void remove(String tableName, String fieldName) throws QException
+   {
+      String name = tableName();
+      if(name == null)
+      {
+         return;
+      }
+      List<Serializable> ids = new ArrayList<>();
+      find(name, tableName, fieldName).forEach(r -> ids.add(r.getValue("id")));
+      if(ids.isEmpty())
+      {
+         return;
+      }
+      QuickSearchConfigWrites.asSystem(() ->
+      {
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(name);
+         deleteInput.setPrimaryKeys(ids);
+         return (new DeleteAction().execute(deleteInput));
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table (fieldName null) or field has a tombstone.
+    *******************************************************************************/
+   public static boolean exists(String tableName, String fieldName) throws QException
+   {
+      String name = tableName();
+      return (name != null && !find(name, tableName, fieldName).isEmpty());
+   }
+
+
+
+   /*******************************************************************************
+    ** The tombstones for one table (fieldName null) or field.
+    *******************************************************************************/
+   static List<QRecord> find(String name, String tableName, String fieldName) throws QException
+   {
+      QQueryFilter filter = new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName));
+      filter.addCriteria(fieldName == null
+         ? new QFilterCriteria("fieldName", QCriteriaOperator.IS_BLANK)
+         : new QFilterCriteria("fieldName", QCriteriaOperator.EQUALS, fieldName));
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(name);
+      queryInput.setFilter(filter);
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/AnalyzerTester.java

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+
+
+/*******************************************************************************
+ ** The analyzer tester: run _analyze on the live index with its two
+ ** analyzers. The text is analyzed as it would be indexed (standard
+ ** tokenizer, lowercase, edge n-grams of 2 to 20 characters) and as a query
+ ** (standard tokenizer, lowercase); with a query, each query token is marked
+ ** found or not among the text's indexed tokens, which is how matching works
+ ** (every query token must be found). Read-only.
+ *******************************************************************************/
+public class AnalyzerTester
+{
+
+   /*******************************************************************************
+    ** The tokens of the text (both analyzers) and of the query.
+    *******************************************************************************/
+   public record Result(String text, String query, List<AnalyzedToken> indexTokens, List<AnalyzedToken> textAsQueryTokens, List<AnalyzedToken> queryTokens)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Analyze text (and an optional query) with the index's analyzers.
+    *******************************************************************************/
+   public Result analyze(QuickSearchRuntime runtime, String text, String query) throws QException
+   {
+      QuickSearchDiagnosticsOperations ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      String                           safeText = text == null ? "" : text;
+      return (new Result(safeText, query,
+         ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, safeText),
+         ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, safeText),
+         StringUtils.hasContent(query) ? ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, query) : null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render a result as HTML.
+    *******************************************************************************/
+   public static String render(Result result)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Analyzer tokens");
+      if(result.text().length() > QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH)
+      {
+         html.status(Level.WARN, "Only the first " + QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH + " characters were analyzed.");
+      }
+
+      Set<String> indexed = new LinkedHashSet<>();
+      result.indexTokens().forEach(t -> indexed.add(t.token()));
+
+      if(result.queryTokens() != null)
+      {
+         html.subheading("Does the query match this text?");
+         List<DiagnosticsHtml.Check> checks = new ArrayList<>();
+         boolean                     all    = !result.queryTokens().isEmpty();
+         for(AnalyzedToken token : result.queryTokens())
+         {
+            boolean found = indexed.contains(token.token());
+            all &= found;
+            String why = found ? "found among the indexed tokens"
+               : token.token().length() < 2 ? "one character: never indexed (n-grams start at 2)"
+               : token.token().length() > 20 ? "longer than 20 characters: never indexed (n-grams stop at 20)"
+               : "no word in the text starts with it";
+            checks.add(new DiagnosticsHtml.Check("\"" + token.token() + "\"", found ? Level.OK : Level.FAIL, why));
+         }
+         html.status(all ? Level.OK : Level.FAIL, all ? "Every query token is found: the searchable-text clause matches."
+            : "At least one query token is missing: with AND semantics, a document with only this text does not match.");
+         html.checks(checks);
+      }
+
+      html.subheading("As indexed (" + QuickSearchOpenSearchClient.INDEX_ANALYZER + "): " + result.indexTokens().size() + " token(s)");
+      html.paragraph(join(result.indexTokens()));
+      html.subheading("As a query (" + QuickSearchOpenSearchClient.SEARCH_ANALYZER + "): " + result.textAsQueryTokens().size() + " token(s)");
+      html.paragraph(join(result.textAsQueryTokens()));
+      if(result.queryTokens() != null)
+      {
+         html.subheading("Query tokens: " + result.queryTokens().size());
+         html.paragraph(join(result.queryTokens()));
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Tokens as "token (position)" separated by commas.
+    *******************************************************************************/
+   static String join(List<AnalyzedToken> tokens)
+   {
+      if(tokens == null || tokens.isEmpty())
+      {
+         return ("(none)");
+      }
+      List<String> parts = new ArrayList<>();
+      for(AnalyzedToken token : tokens)
+      {
+         parts.add(token.token() + " @" + token.position());
+      }
+      return (String.join(", ", parts));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticEventRecorder.java

@@ -0,0 +1,259 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.net.InetAddress;
+import java.time.Instant;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Writes quick_search_diagnostic_event rows: failed searches, diagnostic tool
+ ** runs (the audit trail) and refused runs.
+ ** Best effort and off the caller's thread: events are queued (at most
+ ** QUEUE_CAPACITY) and inserted by one daemon thread as the system user, so
+ ** a slow or failing database never slows or fails a search. At most
+ ** MAX_EVENTS_PER_MINUTE events are written per JVM per minute; the rest are
+ ** counted as dropped (see getDroppedCount), so a search outage cannot flood
+ ** the table. A no-op when the diagnostic tables are not configured.
+ *******************************************************************************/
+public class DiagnosticEventRecorder implements AutoCloseable
+{
+   private static final QLogger LOG = QLogger.getLogger(DiagnosticEventRecorder.class);
+
+   public static final int MAX_EVENTS_PER_MINUTE = 120;
+   public static final int QUEUE_CAPACITY        = 500;
+
+   private final QuickSearchQBitConfig config;
+   private final ThreadPoolExecutor    executor;
+   private final AtomicLong            droppedCount = new AtomicLong();
+   private final AtomicLong            failedCount  = new AtomicLong();
+   private final String                hostName;
+
+   private long windowStartMillis = 0L;
+   private int  windowCount       = 0;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public DiagnosticEventRecorder(QuickSearchQBitConfig config)
+   {
+      this.config   = config;
+      this.hostName = resolveHostName();
+      this.executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(QUEUE_CAPACITY), runnable ->
+      {
+         Thread thread = new Thread(runnable, "quick-search-diagnostic-events");
+         thread.setDaemon(true);
+         return (thread);
+      }, (runnable, pool) -> droppedCount.incrementAndGet());
+   }
+
+
+
+   /*******************************************************************************
+    ** Queue an event for insert. Fills in host name and create date, bounds
+    ** string lengths, and uses the current session's user when none is set.
+    *******************************************************************************/
+   public void record(QuickSearchDiagnosticEvent event)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      String    tableName = config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
+      if(event == null || qInstance == null || !Boolean.TRUE.equals(config.getEnableDiagnosticTools()) || qInstance.getTable(tableName) == null)
+      {
+         return;
+      }
+
+      if(!admit())
+      {
+         droppedCount.incrementAndGet();
+         return;
+      }
+
+      if(event.getUserName() == null)
+      {
+         event.setUserName(DiagnosticsSupport.currentUserName());
+      }
+
+      event.withHostName(truncate(hostName, 100))
+         .withCreateDate(Instant.now())
+         .withSource(truncate(event.getSource(), 50))
+         .withUserName(truncate(event.getUserName(), 250))
+         .withTableNames(truncate(event.getTableNames(), 1000))
+         .withSearchTerm(truncate(event.getSearchTerm(), 100))
+         .withErrorClass(truncate(event.getErrorClass(), 250))
+         .withErrorMessage(truncate(event.getErrorMessage(), 4000))
+         .withDetail(truncate(event.getDetail(), 4000));
+
+      executor.execute(() -> insert(qInstance, tableName, event));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one event as the system user. Failures are logged and counted.
+    *******************************************************************************/
+   private void insert(QInstance qInstance, String tableName, QuickSearchDiagnosticEvent event)
+   {
+      QSession session = new QSystemUserSession();
+      try
+      {
+         QContext.init(qInstance, session);
+         new InsertAction().execute(new InsertInput(tableName).withRecordEntity(event));
+      }
+      catch(Exception e)
+      {
+         failedCount.incrementAndGet();
+         LOG.warn("Could not record a Quick Search diagnostic event", e, logPair("eventType", event.getEventType()), logPair("source", event.getSource()));
+      }
+      finally
+      {
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether one more event fits in the current minute's budget.
+    *******************************************************************************/
+   private synchronized boolean admit()
+   {
+      long now = System.currentTimeMillis();
+      if(now - windowStartMillis >= 60_000L)
+      {
+         windowStartMillis = now;
+         windowCount       = 0;
+      }
+      if(windowCount >= MAX_EVENTS_PER_MINUTE)
+      {
+         return (false);
+      }
+      windowCount++;
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Wait (up to timeoutMillis) until every queued event has been written.
+    ** For tests and orderly shutdown.
+    *******************************************************************************/
+   public void flush(long timeoutMillis)
+   {
+      try
+      {
+         executor.submit(() ->
+         {
+         }).get(timeoutMillis, TimeUnit.MILLISECONDS);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Diagnostic event flush did not complete", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Events not written because the queue or the per-minute budget was full.
+    *******************************************************************************/
+   public long getDroppedCount()
+   {
+      return (droppedCount.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Events whose insert failed.
+    *******************************************************************************/
+   public long getFailedCount()
+   {
+      return (failedCount.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop the writer after the queued events (up to 2 seconds).
+    *******************************************************************************/
+   @Override
+   public void close()
+   {
+      executor.shutdown();
+      try
+      {
+         if(!executor.awaitTermination(2, TimeUnit.SECONDS))
+         {
+            executor.shutdownNow();
+         }
+      }
+      catch(InterruptedException e)
+      {
+         executor.shutdownNow();
+         Thread.currentThread().interrupt();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** This JVM's host name (the pod name in Kubernetes).
+    *******************************************************************************/
+   static String resolveHostName()
+   {
+      String fromEnv = System.getenv("HOSTNAME");
+      if(fromEnv != null && !fromEnv.isBlank())
+      {
+         return (fromEnv);
+      }
+      try
+      {
+         return (InetAddress.getLocalHost().getHostName());
+      }
+      catch(Exception e)
+      {
+         return ("unknown");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   static String truncate(String value, int max)
+   {
+      return (value == null || value.length() <= max ? value : value.substring(0, max));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticRecordReads.java

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Source-table reads for the diagnostics, in the current session (so table
+ ** permissions and record security locks apply, as for any query the user
+ ** runs). Records are read the way indexing reads them: display values
+ ** generated and possible values translated, so a document built from them
+ ** is exactly the document indexing would write.
+ *******************************************************************************/
+public final class DiagnosticRecordReads
+{
+   public static final int MAX_IDS_PER_QUERY = 1_000;
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private DiagnosticRecordReads()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Records of a table by primary key (as strings), keyed by primary key
+    ** string, read as indexing reads them. Ids that do not exist or that the
+    ** session cannot read are absent. Ids that do not convert to the key's
+    ** type are skipped.
+    *******************************************************************************/
+   public static Map<String, QRecord> loadForIndexing(String tableName, Collection<String> recordIds) throws QException
+   {
+      QTableMetaData       table   = requireTable(tableName);
+      String               pk      = table.getPrimaryKeyField();
+      Map<String, QRecord> records = new LinkedHashMap<>();
+      List<Serializable>   ids     = typedIds(table, recordIds);
+
+      for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
+         queryInput.setShouldGenerateDisplayValues(true);
+         queryInput.setShouldTranslatePossibleValues(true);
+         for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+         {
+            records.put(record.getValueString(pk), record);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Up to limit records of a table ordered by primary key (ascending or
+    ** descending), read as indexing reads them.
+    *******************************************************************************/
+   public static List<QRecord> loadEdge(String tableName, boolean ascending, int limit) throws QException
+   {
+      QTableMetaData table      = requireTable(tableName);
+      QueryInput     queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(table.getPrimaryKeyField(), ascending)).withLimit(limit));
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldTranslatePossibleValues(true);
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The primary keys (as strings) among recordIds that exist and that the
+    ** session can read.
+    *******************************************************************************/
+   public static List<String> existingIds(String tableName, Collection<String> recordIds) throws QException
+   {
+      QTableMetaData     table  = requireTable(tableName);
+      String             pk     = table.getPrimaryKeyField();
+      List<String>       found  = new ArrayList<>();
+      List<Serializable> ids    = typedIds(table, recordIds);
+
+      for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
+         for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+         {
+            found.add(record.getValueString(pk));
+         }
+      }
+      return (found);
+   }
+
+
+
+   /*******************************************************************************
+    ** Row count of a table in the current session.
+    *******************************************************************************/
+   public static Integer count(String tableName) throws QException
+   {
+      CountInput countInput = new CountInput();
+      countInput.setTableName(tableName);
+      return (new CountAction().execute(countInput).getCount());
+   }
+
+
+
+   /*******************************************************************************
+    ** Ids converted to the primary key's type; ids that do not convert are
+    ** skipped (they cannot exist).
+    *******************************************************************************/
+   static List<Serializable> typedIds(QTableMetaData table, Collection<String> recordIds)
+   {
+      QFieldMetaData     pkField = table.getField(table.getPrimaryKeyField());
+      List<Serializable> ids     = new ArrayList<>();
+      for(String recordId : CollectionUtils.nonNullCollection(recordIds))
+      {
+         try
+         {
+            Serializable id = ValueUtils.getValueAsFieldType(pkField.getType(), recordId);
+            if(id != null && !ids.contains(id))
+            {
+               ids.add(id);
+            }
+         }
+         catch(Exception e)
+         {
+            //////////////////////////////////////////////////////////
+            // e.g. "abc" for an integer key: no such record exists //
+            //////////////////////////////////////////////////////////
+         }
+      }
+      return (ids);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QTableMetaData requireTable(String tableName) throws QException
+   {
+      QTableMetaData table = QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw (new QException("Table [" + tableName + "] is not in the instance"));
+      }
+      return (table);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsHtml.java

@@ -0,0 +1,362 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.List;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+
+/*******************************************************************************
+ ** A small HTML builder for the diagnostic screens (process HTML blocks, the
+ ** validation report field and the diagnostics widget). Every text value is
+ ** escaped. Layout rules: one column of blocks capped at a readable width,
+ ** no fixed widths, and long values (JSON, ids, text) wrap instead of
+ ** scrolling, so a screen reads from 360px to 3200px without an inner or
+ ** horizontal scrollbar. Colours are mid-tone so they read in light and dark
+ ** themes; status is always also spelled out in text.
+ *******************************************************************************/
+public class DiagnosticsHtml
+{
+   public static final int MAX_JSON_CHARS = 60_000;
+
+   private static final ObjectMapper PRETTY = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+
+   private static final String WRAP = "overflow-wrap:anywhere;word-break:break-word;";
+
+   private final StringBuilder html = new StringBuilder();
+
+
+
+   /*******************************************************************************
+    ** Severity of a check or a line, rendered as a coloured label.
+    *******************************************************************************/
+   public enum Level
+   {
+      OK("OK", "#16a34a"),
+      INFO("INFO", "#2563eb"),
+      WARN("WARN", "#d97706"),
+      FAIL("FAIL", "#dc2626");
+
+      private final String label;
+      private final String color;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       *******************************************************************************/
+      Level(String label, String color)
+      {
+         this.label = label;
+         this.color = color;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for label
+       *******************************************************************************/
+      public String getLabel()
+      {
+         return (label);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Escape text for HTML element content and attribute values.
+    *******************************************************************************/
+   public static String escape(Object value)
+   {
+      if(value == null)
+      {
+         return ("");
+      }
+      String        text = String.valueOf(value);
+      StringBuilder out  = new StringBuilder(text.length() + 16);
+      for(int i = 0; i < text.length(); i++)
+      {
+         char c = text.charAt(i);
+         switch(c)
+         {
+            case '<' -> out.append("&lt;");
+            case '>' -> out.append("&gt;");
+            case '&' -> out.append("&amp;");
+            case '"' -> out.append("&quot;");
+            case '\'' -> out.append("&#39;");
+            default -> out.append(c);
+         }
+      }
+      return (out.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Escape text but keep OpenSearch highlight tags (em) as mark elements.
+    *******************************************************************************/
+   public static String highlight(String fragment)
+   {
+      return (escape(fragment).replace("&lt;em&gt;", "<mark>").replace("&lt;/em&gt;", "</mark>"));
+   }
+
+
+
+   /*******************************************************************************
+    ** A coloured status label.
+    *******************************************************************************/
+   public static String badge(Level level)
+   {
+      return ("<strong style=\"color:" + level.color + "\">" + level.label + "</strong>");
+   }
+
+
+
+   /*******************************************************************************
+    ** Open the root block.
+    *******************************************************************************/
+   public DiagnosticsHtml open()
+   {
+      html.append("<div style=\"max-width:72rem;").append(WRAP).append("\">");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Close the root block.
+    *******************************************************************************/
+   public DiagnosticsHtml close()
+   {
+      html.append("</div>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A section heading.
+    *******************************************************************************/
+   public DiagnosticsHtml heading(String text)
+   {
+      html.append("<h3>").append(escape(text)).append("</h3>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A sub-heading.
+    *******************************************************************************/
+   public DiagnosticsHtml subheading(String text)
+   {
+      html.append("<h4>").append(escape(text)).append("</h4>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A paragraph of escaped text.
+    *******************************************************************************/
+   public DiagnosticsHtml paragraph(String text)
+   {
+      html.append("<p>").append(escape(text)).append("</p>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A paragraph starting with a status label.
+    *******************************************************************************/
+   public DiagnosticsHtml status(Level level, String text)
+   {
+      html.append("<p>").append(badge(level)).append(" ").append(escape(text)).append("</p>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A list of checks: each line a status label and text.
+    *******************************************************************************/
+   public DiagnosticsHtml checks(List<Check> checks)
+   {
+      if(checks == null || checks.isEmpty())
+      {
+         return (this);
+      }
+      html.append("<ul>");
+      for(Check check : checks)
+      {
+         html.append("<li>").append(badge(check.level())).append(" <strong>").append(escape(check.name())).append("</strong>");
+         if(check.message() != null)
+         {
+            html.append(": ").append(escape(check.message()));
+         }
+         html.append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A list of label: value lines (values escaped). Null values show as
+    ** "none".
+    *******************************************************************************/
+   public DiagnosticsHtml facts(List<String[]> facts)
+   {
+      html.append("<ul>");
+      for(String[] fact : facts)
+      {
+         html.append("<li><strong>").append(escape(fact[0])).append(":</strong> ")
+            .append(fact.length > 1 && fact[1] != null ? escape(fact[1]) : "<em>none</em>").append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A bulleted list of pre-built HTML items (callers escape their parts).
+    *******************************************************************************/
+   public DiagnosticsHtml rawList(List<String> itemsHtml)
+   {
+      if(itemsHtml == null || itemsHtml.isEmpty())
+      {
+         return (this);
+      }
+      html.append("<ul>");
+      for(String item : itemsHtml)
+      {
+         html.append("<li>").append(item).append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A preformatted block that wraps (no horizontal scrolling).
+    *******************************************************************************/
+   public DiagnosticsHtml pre(String text)
+   {
+      html.append("<pre style=\"white-space:pre-wrap;").append(WRAP).append("\">").append(escape(text)).append("</pre>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A collapsed block with a summary line and pretty-printed JSON inside.
+    *******************************************************************************/
+   public DiagnosticsHtml jsonDetails(String summary, String json)
+   {
+      html.append("<details><summary>").append(escape(summary)).append("</summary>");
+      pre(prettyJson(json));
+      html.append("</details>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A collapsed block with a summary line and preformatted text inside.
+    *******************************************************************************/
+   public DiagnosticsHtml textDetails(String summary, String text)
+   {
+      html.append("<details><summary>").append(escape(summary)).append("</summary>");
+      pre(text);
+      html.append("</details>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Append pre-built HTML (callers escape their parts).
+    *******************************************************************************/
+   public DiagnosticsHtml raw(String rawHtml)
+   {
+      html.append(rawHtml);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Pretty-print JSON (or return the text as-is when it is not JSON),
+    ** truncated to MAX_JSON_CHARS.
+    *******************************************************************************/
+   public static String prettyJson(String json)
+   {
+      if(json == null)
+      {
+         return ("");
+      }
+      String pretty;
+      try
+      {
+         pretty = PRETTY.writeValueAsString(PRETTY.readTree(json));
+      }
+      catch(Exception e)
+      {
+         pretty = json;
+      }
+      return (pretty.length() > MAX_JSON_CHARS ? pretty.substring(0, MAX_JSON_CHARS) + "\n... (truncated at " + MAX_JSON_CHARS + " characters)" : pretty);
+   }
+
+
+
+   /*******************************************************************************
+    ** Pretty-print a JSON node.
+    *******************************************************************************/
+   public static String prettyJson(JsonNode node)
+   {
+      return (node == null ? "" : prettyJson(node.toString()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The built HTML.
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return (html.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** One check: name, level and message.
+    *******************************************************************************/
+   public record Check(String name, Level level, String message)
+   {
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsRateLimiter.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.LongSupplier;
+
+
+/*******************************************************************************
+ ** Sliding one-minute window of diagnostic tool runs per key (user and
+ ** tool), per JVM. tryAcquire admits a run when fewer than the limit ran in
+ ** the last 60 seconds.
+ *******************************************************************************/
+public class DiagnosticsRateLimiter
+{
+   private static final long WINDOW_MILLIS = 60_000L;
+
+   private final Map<String, Deque<Long>> runsByKey = new ConcurrentHashMap<>();
+   private final LongSupplier             clock;
+
+
+
+   /*******************************************************************************
+    ** Constructor (system clock)
+    *******************************************************************************/
+   public DiagnosticsRateLimiter()
+   {
+      this(System::currentTimeMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor with a clock (tests)
+    *******************************************************************************/
+   public DiagnosticsRateLimiter(LongSupplier clock)
+   {
+      this.clock = clock;
+   }
+
+
+
+   /*******************************************************************************
+    ** Admit one run for the key when fewer than perMinute ran in the last
+    ** minute; a null or non-positive limit admits everything.
+    *******************************************************************************/
+   public boolean tryAcquire(String key, Integer perMinute)
+   {
+      if(perMinute == null || perMinute <= 0)
+      {
+         return (true);
+      }
+
+      long        now  = clock.getAsLong();
+      Deque<Long> runs = runsByKey.computeIfAbsent(key, k -> new ArrayDeque<>());
+      synchronized(runs)
+      {
+         while(!runs.isEmpty() && now - runs.peekFirst() >= WINDOW_MILLIS)
+         {
+            runs.pollFirst();
+         }
+         if(runs.size() >= perMinute)
+         {
+            return (false);
+         }
+         runs.addLast(now);
+         return (true);
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsSupport.java

@@ -0,0 +1,249 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.Collection;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Shared pieces of the diagnostic tools: tool names, the current user, the
+ ** per-user rate limit, and recording tool runs and search failures as
+ ** quick_search_diagnostic_event rows (the audit trail).
+ *******************************************************************************/
+public final class DiagnosticsSupport
+{
+   private static final QLogger LOG = QLogger.getLogger(DiagnosticsSupport.class);
+
+   public static final String TOOL_PLAYGROUND      = "SEARCH_PLAYGROUND";
+   public static final String TOOL_INSPECT_RECORD  = "INSPECT_RECORD";
+   public static final String TOOL_REINDEX_RECORDS = "REINDEX_RECORDS";
+   public static final String TOOL_VALIDATE_INDEX  = "VALIDATE_INDEX";
+   public static final String TOOL_ANALYZE_TEXT    = "ANALYZE_TEXT";
+
+
+
+   /*******************************************************************************
+    ** Latest saved validation, shared by the home widget and host status API.
+    ** This does not run validation or scan source tables.
+    *******************************************************************************/
+   public static QRecord latestValidation(QuickSearchQBitConfig config) throws QException
+   {
+      String tableName = config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         return (null);
+      }
+      var input = new QueryInput(tableName)
+         .withFilter(new QQueryFilter()
+            .withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
+      var rows = new QueryAction().execute(input).getRecords();
+      return (rows == null || rows.isEmpty() ? null : rows.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private DiagnosticsSupport()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The session's user id reference (else full name), or null.
+    *******************************************************************************/
+   public static String currentUserName()
+   {
+      QSession session = QContext.getQSession();
+      if(session == null || session.getUser() == null)
+      {
+         return (null);
+      }
+      return (session.getUser().getIdReference() != null ? session.getUser().getIdReference() : session.getUser().getFullName());
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime of the Quick Search QBit, or a user-facing error saying the
+    ** tool is unavailable.
+    *******************************************************************************/
+   public static QuickSearchRuntime requireRuntime() throws QUserFacingException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         throw (new QUserFacingException("Quick Search is not configured in this application, so its diagnostic tools are unavailable."));
+      }
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Admit one run of a tool for the current user, or refuse it (recording
+    ** the refusal) when the user ran diagnosticRunsPerMinute tools in the last
+    ** minute.
+    *******************************************************************************/
+   public static void checkRateLimit(QuickSearchRuntime runtime, String tool) throws QUserFacingException
+   {
+      String user = currentUserName();
+      if(runtime.getDiagnosticsRateLimiter().tryAcquire((user == null ? "anonymous" : user) + "|" + tool, runtime.getConfig().getDiagnosticRunsPerMinute()))
+      {
+         return;
+      }
+
+      runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+         .withEventType(QuickSearchDiagnosticEvent.TYPE_RATE_LIMITED)
+         .withSource(tool)
+         .withOutcome(QuickSearchDiagnosticEvent.OUTCOME_REFUSED));
+      throw (new QUserFacingException("Too many Quick Search diagnostic runs in the last minute (limit " + runtime.getConfig().getDiagnosticRunsPerMinute()
+         + " per tool). Wait a minute and try again."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a tool run (audit): who ran which tool on which tables with which
+    ** term, how long it took, and its outcome.
+    *******************************************************************************/
+   public static void recordToolRun(QuickSearchRuntime runtime, String tool, String eventType, Collection<String> tableNames, String term, long startMillis, Throwable error, String detail)
+   {
+      try
+      {
+         runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+            .withEventType(eventType)
+            .withSource(tool)
+            .withOutcome(error == null ? QuickSearchDiagnosticEvent.OUTCOME_OK : QuickSearchDiagnosticEvent.OUTCOME_ERROR)
+            .withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
+            .withSearchTerm(term)
+            .withDurationMillis((int) Math.min(Integer.MAX_VALUE, System.currentTimeMillis() - startMillis))
+            .withErrorClass(error == null ? null : error.getClass().getName())
+            .withErrorMessage(error == null ? null : error.getMessage())
+            .withDetail(detail));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not queue a diagnostic event", e);
+      }
+
+      LOG.info("Quick Search diagnostic tool run", logPair("tool", tool), logPair("user", currentUserName()), logPair("tables", tableNames),
+         logPair("millis", System.currentTimeMillis() - startMillis), logPair("error", error == null ? null : error.getMessage()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a failed search (OpenSearch error or timeout). userTerm is the
+    ** user's term: never stored, and redacted from the message and detail.
+    *******************************************************************************/
+   public static void recordSearchFailure(QuickSearchRuntime runtime, String source, Collection<String> tableNames, String userTerm, long durationMillis, Throwable error, String detail)
+   {
+      if(runtime == null)
+      {
+         return;
+      }
+      try
+      {
+         Throwable cause = rootCause(error);
+         runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+            .withEventType(QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED)
+            .withSource(source)
+            .withOutcome(QuickSearchDiagnosticEvent.OUTCOME_ERROR)
+            .withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
+            .withDurationMillis((int) Math.min(Integer.MAX_VALUE, durationMillis))
+            .withErrorClass(cause == null ? null : cause.getClass().getName())
+            .withErrorMessage(redactTerm(error == null ? null : error.getMessage(), userTerm))
+            .withDetail(redactTerm(detail, userTerm)));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not queue a search failure event", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The text with the user's search term, and each of its words of three or
+    ** more characters, replaced by a marker (case-insensitive): error messages
+    ** (ours and OpenSearch's) can echo the term, which may hold personal data.
+    *******************************************************************************/
+   public static String redactTerm(String text, String userTerm)
+   {
+      if(text == null || userTerm == null || userTerm.isBlank())
+      {
+         return (text);
+      }
+      String redacted = text.replaceAll("(?i)" + java.util.regex.Pattern.quote(userTerm.trim()), "[search term]");
+      for(String word : userTerm.trim().split("\\s+"))
+      {
+         if(word.length() >= 3)
+         {
+            redacted = redacted.replaceAll("(?i)" + java.util.regex.Pattern.quote(word), "[search term]");
+         }
+      }
+      return (redacted);
+   }
+
+
+
+   /*******************************************************************************
+    ** The innermost cause of a throwable (the one that names the real error
+    ** class: timeout, connection refused, OpenSearch error).
+    *******************************************************************************/
+   public static Throwable rootCause(Throwable error)
+   {
+      Throwable cause = error;
+      int       depth = 0;
+      while(cause != null && cause.getCause() != null && cause.getCause() != cause && depth++ < 20)
+      {
+         cause = cause.getCause();
+      }
+      return (cause);
+   }
+
+
+
+   /*******************************************************************************
+    ** A user-facing error for a tool whose OpenSearch call failed: the tool
+    ** reports the failure honestly instead of showing partial data.
+    *******************************************************************************/
+   public static QUserFacingException unavailable(String what, QException e)
+   {
+      return (new QUserFacingException(what + " is unavailable: " + e.getMessage(), e));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/IndexValidator.java

@@ -0,0 +1,792 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.RawResponse;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Validate the Quick Search index against the database and its expected
+ ** shape, and report every finding with a level (OK, INFO, WARN, FAIL):
+ ** - cluster: reachable, cluster and index health, the alias pointing at one
+ **   physical index, leftover physical indexes from interrupted rebuilds;
+ ** - mapping and settings: mapping version, field types and analyzers of the
+ **   fixed fields, the fieldValues dynamic template and every fieldValues.*
+ **   field mapped as text, the two analyzers and the edge n-gram filter;
+ ** - per table: database rows vs indexed documents, a sample of records
+ **   (the lowest and highest primary keys) checked present and current in
+ **   the index, a random sample of documents checked to still have a record,
+ **   the quick_search_index row's status and last run, failed events waiting.
+ ** Database reads run as the system user, so counts compare like with like
+ ** (the index holds every row whatever the record locks). The report holds
+ ** counts, ids and field names, never record values. Read-only, except that
+ ** persist() saves the report as a quick_search_validation_run row.
+ *******************************************************************************/
+public class IndexValidator
+{
+   private static final QLogger LOG = QLogger.getLogger(IndexValidator.class);
+
+   private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
+
+   public static final String SCOPE_CLUSTER = "cluster";
+   public static final String SCOPE_MAPPING = "mapping";
+
+   private static final int MAX_LISTED_IDS = 25;
+
+
+
+   /*******************************************************************************
+    ** One finding.
+    *******************************************************************************/
+   public record Finding(String scope, String check, Level level, String message)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Per-table numbers.
+    *******************************************************************************/
+   public record TableStats(String tableName, Integer databaseCount, Long indexCount, Integer sampled, List<String> missingIds, Map<String, List<String>> staleFields,
+                            Integer documentsSampled, List<String> extraIds)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A validation report.
+    *******************************************************************************/
+   public record Report(String status, Instant startTime, Instant endTime, Integer sampleSize, List<String> tableNames, List<Finding> findings, List<TableStats> tableStats)
+   {
+      /*******************************************************************************
+       ** Count of findings at a level.
+       *******************************************************************************/
+      public int count(Level level)
+      {
+         return ((int) findings.stream().filter(f -> f.level() == level).count());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate the named tables (empty for every indexed table) with the given
+    ** sample size per table.
+    *******************************************************************************/
+   public Report validate(QuickSearchRuntime runtime, List<String> requestedTables, int sampleSize) throws QException
+   {
+      Instant          start     = Instant.now();
+      List<Finding>    findings  = new ArrayList<>();
+      List<TableStats> stats     = new ArrayList<>();
+      List<String>     tables    = resolveTables(runtime, requestedTables, findings);
+      int              boundedN  = Math.max(1, Math.min(sampleSize, QuickSearchQBitConfig.MAX_VALIDATION_SAMPLE_SIZE));
+      QInstance        qInstance = QContext.getQInstance();
+
+      QContext.withTemporaryContext(new CapturedContext(qInstance, new QSystemUserSession()), () ->
+      {
+         QuickSearchDiagnosticsOperations ops;
+         try
+         {
+            ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+         }
+         catch(QException e)
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Client", Level.FAIL, "The OpenSearch client could not be built: " + e.getMessage()));
+            return;
+         }
+
+         if(!checkCluster(runtime, ops, findings))
+         {
+            return;
+         }
+         checkMapping(ops, findings);
+         checkSettings(ops, findings);
+
+         for(String tableName : tables)
+         {
+            stats.add(checkTable(runtime, ops, runtime.getTableConfig(tableName), boundedN, findings));
+         }
+      });
+
+      String status = findings.stream().anyMatch(f -> f.level() == Level.FAIL) ? QuickSearchValidationRun.STATUS_FAIL
+         : findings.stream().anyMatch(f -> f.level() == Level.WARN) ? QuickSearchValidationRun.STATUS_WARN : QuickSearchValidationRun.STATUS_PASS;
+      return (new Report(status, start, Instant.now(), boundedN, tables, findings, stats));
+   }
+
+
+
+   /*******************************************************************************
+    ** The requested indexed tables (unknown ones reported), else every indexed
+    ** table.
+    *******************************************************************************/
+   private static List<String> resolveTables(QuickSearchRuntime runtime, List<String> requested, List<Finding> findings)
+   {
+      List<String> tables = new ArrayList<>();
+      if(CollectionUtils.nullSafeIsEmpty(requested))
+      {
+         runtime.getDiscoveredTables().forEach(t -> tables.add(t.getTableName()));
+         return (tables);
+      }
+      for(String tableName : requested)
+      {
+         if(runtime.getTableConfig(tableName) == null)
+         {
+            findings.add(new Finding(tableName, "Configured", Level.WARN, "Quick Search does not index this table; skipped."));
+         }
+         else if(!tables.contains(tableName))
+         {
+            tables.add(tableName);
+         }
+      }
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health, alias and physical indexes. Returns false when the index
+    ** cannot be reached (the other checks are then skipped).
+    *******************************************************************************/
+   boolean checkCluster(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse health;
+      try
+      {
+         health = ops.clusterHealth();
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "OpenSearch did not answer: " + e.getMessage()));
+         return (false);
+      }
+
+      if(health.status() == 404)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Index", Level.FAIL, "The index alias [" + ops.getIndexName() + "] does not exist. Run the full reindex to create it."));
+         return (false);
+      }
+      if(!health.isOk())
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "Cluster health returned status " + health.status() + ": " + QuickSearchDiagnosticsOperations.errorReason(health.body())));
+         return (false);
+      }
+
+      findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.OK, "OpenSearch answered (" + health.body().path("cluster_name").asText("cluster") + ", "
+         + health.body().path("number_of_nodes").asInt() + " node(s))."));
+      findings.add(healthFinding("Cluster health", health.body().path("status").asText("unknown")));
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = health.body().path("indices").fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index = indexes.next();
+         findings.add(healthFinding("Index health " + index.getKey(), index.getValue().path("status").asText("unknown")));
+      }
+
+      try
+      {
+         Set<String> physical = runtime.getClient().getPhysicalIndexes();
+         if(physical.isEmpty())
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.WARN, "[" + ops.getIndexName() + "] is a concrete index, not an alias (created before 1.0). The next full reindex replaces it with an alias."));
+         }
+         else if(physical.size() > 1)
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "The alias points at " + physical.size() + " physical indexes " + physical + "; searches and counts see documents twice. Run the full reindex."));
+         }
+         else
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.OK, "The alias points at " + physical.iterator().next() + "."));
+         }
+
+         RawResponse cat = ops.physicalIndexes();
+         if(cat.isOk() && cat.body() != null)
+         {
+            for(JsonNode row : cat.body())
+            {
+               String name = row.path("index").asText();
+               if(!physical.contains(name))
+               {
+                  findings.add(new Finding(SCOPE_CLUSTER, "Leftover index", Level.WARN, "Physical index " + name + " (" + row.path("docs.count").asText("?") + " docs, "
+                     + row.path("store.size").asText("?") + ") is not behind the alias: left by an interrupted rebuild. Delete it to free space."));
+               }
+            }
+         }
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "Could not read the alias: " + e.getMessage()));
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static Finding healthFinding(String check, String status)
+   {
+      return (switch(status)
+      {
+         case "green" -> new Finding(SCOPE_CLUSTER, check, Level.OK, "green.");
+         case "yellow" -> new Finding(SCOPE_CLUSTER, check, Level.WARN, "yellow: replica shards are unassigned (expected on a single-node cluster when the index asks for replicas).");
+         default -> new Finding(SCOPE_CLUSTER, check, Level.FAIL, status + ": primary shards are unassigned; searches and writes may fail.");
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Shared cluster health interpretation for validation and the host status API.
+    *******************************************************************************/
+   public static Level healthLevel(String status)
+   {
+      return (healthFinding("Health", status).level());
+   }
+
+
+
+   /*******************************************************************************
+    ** The live mapping against the mapping this version creates.
+    *******************************************************************************/
+   void checkMapping(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse response;
+      try
+      {
+         response = ops.mapping();
+         QuickSearchDiagnosticsOperations.requireOk(response, "Mapping read failed");
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_MAPPING, "Mapping", Level.FAIL, e.getMessage()));
+         return;
+      }
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index    = indexes.next();
+         JsonNode                    mappings = index.getValue().path("mappings");
+         String                      prefix   = index.getKey() + ": ";
+
+         int version = mappings.path("_meta").path(QuickSearchOpenSearchClient.META_MAPPING_VERSION).asInt(1);
+         findings.add(version >= QuickSearchOpenSearchClient.MAPPING_VERSION
+            ? new Finding(SCOPE_MAPPING, "Mapping version", Level.OK, prefix + "version " + version + ".")
+            : new Finding(SCOPE_MAPPING, "Mapping version", Level.FAIL, prefix + "version " + version + ", this release writes " + QuickSearchOpenSearchClient.MAPPING_VERSION + ". Run the full reindex."));
+
+         JsonNode properties = mappings.path("properties");
+         List<String> problems = new ArrayList<>();
+         expectType(properties, "sourceTable", "keyword", problems);
+         expectType(properties, "recordId", "keyword", problems);
+         expectType(properties, "indexedAt", "date", problems);
+         expectText(properties.path("searchableText"), "searchableText", problems);
+         expectText(properties.path("recordLabel"), "recordLabel", problems);
+
+         boolean template = false;
+         for(JsonNode entry : mappings.path("dynamic_templates"))
+         {
+            template |= entry.has("fieldValuesAsText");
+         }
+         if(!template)
+         {
+            problems.add("the fieldValuesAsText dynamic template is missing, so new fields may be mapped as numbers or dates");
+         }
+         if(mappings.path("date_detection").asBoolean(true) || mappings.path("numeric_detection").asBoolean(false))
+         {
+            problems.add("date or numeric detection is on");
+         }
+
+         Iterator<Map.Entry<String, JsonNode>> fields = properties.path("fieldValues").path("properties").fields();
+         int                                   count  = 0;
+         while(fields.hasNext())
+         {
+            Map.Entry<String, JsonNode> field = fields.next();
+            expectText(field.getValue(), "fieldValues." + field.getKey(), problems);
+            count++;
+         }
+
+         findings.add(problems.isEmpty()
+            ? new Finding(SCOPE_MAPPING, "Field mappings", Level.OK, prefix + "fixed fields and " + count + " fieldValues field(s) mapped as expected.")
+            : new Finding(SCOPE_MAPPING, "Field mappings", Level.FAIL, prefix + String.join("; ", problems) + ". Run the full reindex to rebuild the mapping."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void expectType(JsonNode properties, String field, String type, List<String> problems)
+   {
+      String actual = properties.path(field).path("type").asText(null);
+      if(!type.equals(actual))
+      {
+         problems.add(field + " is " + (actual == null ? "not mapped" : "mapped as " + actual) + ", expected " + type);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void expectText(JsonNode mapping, String field, List<String> problems)
+   {
+      String type     = mapping.path("type").asText(null);
+      String analyzer = mapping.path("analyzer").asText(null);
+      String search   = mapping.path("search_analyzer").asText(null);
+      if(!"text".equals(type))
+      {
+         problems.add(field + " is " + (type == null ? "not mapped" : "mapped as " + type) + ", expected text");
+      }
+      else if(!QuickSearchOpenSearchClient.INDEX_ANALYZER.equals(analyzer) || !QuickSearchOpenSearchClient.SEARCH_ANALYZER.equals(search))
+      {
+         problems.add(field + " uses analyzer " + analyzer + " / search analyzer " + search + ", expected " + QuickSearchOpenSearchClient.INDEX_ANALYZER + " / " + QuickSearchOpenSearchClient.SEARCH_ANALYZER);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The live analysis settings against the ones this version creates.
+    *******************************************************************************/
+   void checkSettings(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse response;
+      try
+      {
+         response = ops.settings();
+         QuickSearchDiagnosticsOperations.requireOk(response, "Settings read failed");
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, e.getMessage()));
+         return;
+      }
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index    = indexes.next();
+         JsonNode                    settings = index.getValue().path("settings").path("index");
+         JsonNode                    analysis = settings.path("analysis");
+         List<String>                problems = new ArrayList<>();
+
+         JsonNode indexAnalyzer  = analysis.path("analyzer").path(QuickSearchOpenSearchClient.INDEX_ANALYZER);
+         JsonNode searchAnalyzer = analysis.path("analyzer").path(QuickSearchOpenSearchClient.SEARCH_ANALYZER);
+         JsonNode ngram          = analysis.path("filter").path("edge_ngram_filter");
+
+         if(!"standard".equals(indexAnalyzer.path("tokenizer").asText()) || !List.of("lowercase", "edge_ngram_filter").equals(textList(indexAnalyzer.path("filter"))))
+         {
+            problems.add(QuickSearchOpenSearchClient.INDEX_ANALYZER + " is " + (indexAnalyzer.isMissingNode() ? "missing" : indexAnalyzer.toString()));
+         }
+         if(!"standard".equals(searchAnalyzer.path("tokenizer").asText()) || !List.of("lowercase").equals(textList(searchAnalyzer.path("filter"))))
+         {
+            problems.add(QuickSearchOpenSearchClient.SEARCH_ANALYZER + " is " + (searchAnalyzer.isMissingNode() ? "missing" : searchAnalyzer.toString()));
+         }
+         if(!"edge_ngram".equals(ngram.path("type").asText()) || ngram.path("min_gram").asInt() != 2 || ngram.path("max_gram").asInt() != 20)
+         {
+            problems.add("edge_ngram_filter is " + (ngram.isMissingNode() ? "missing" : ngram.toString()));
+         }
+
+         findings.add(problems.isEmpty()
+            ? new Finding(SCOPE_MAPPING, "Analysis settings", Level.OK, index.getKey() + ": analyzers and edge n-grams (2 to 20) as expected; "
+            + settings.path("number_of_shards").asText("?") + " shard(s), " + settings.path("number_of_replicas").asText("?") + " replica(s).")
+            : new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, index.getKey() + ": " + String.join("; ", problems) + ". Run the full reindex."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static List<String> textList(JsonNode array)
+   {
+      List<String> values = new ArrayList<>();
+      array.forEach(v -> values.add(v.asText()));
+      return (values);
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts, sampled freshness and orphans of one table, plus its index row
+    ** and failed events.
+    *******************************************************************************/
+   TableStats checkTable(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, int sampleSize, List<Finding> findings)
+   {
+      String tableName = tableConfig.getTableName();
+      Integer databaseCount = null;
+      Long    indexCount    = null;
+      try
+      {
+         databaseCount = DiagnosticRecordReads.count(tableName);
+         indexCount    = runtime.getClient().countDocumentsForTable(tableName);
+         long difference = indexCount - databaseCount;
+         long tolerance  = Math.max(5, databaseCount / 100);
+         Level level = difference == 0 ? Level.OK : (Math.abs(difference) <= tolerance ? Level.WARN : Level.FAIL);
+         findings.add(new Finding(tableName, "Counts", level, "database " + databaseCount + ", index " + indexCount
+            + (difference == 0 ? "." : " (" + (difference > 0 ? difference + " more in the index" : (-difference) + " missing from the index") + "). Reconcile fixes counts.")));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Counts", Level.FAIL, e.getMessage()));
+      }
+
+      List<String>              missing     = new ArrayList<>();
+      Map<String, List<String>> staleFields = new LinkedHashMap<>();
+      int                       sampled     = 0;
+      try
+      {
+         Map<String, QRecord> sample = new LinkedHashMap<>();
+         for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, true, (sampleSize + 1) / 2))
+         {
+            sample.put(record.getValueString(tableConfig.getPrimaryKeyField()), record);
+         }
+         for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, false, sampleSize / 2))
+         {
+            sample.putIfAbsent(record.getValueString(tableConfig.getPrimaryKeyField()), record);
+         }
+         sampled = sample.size();
+
+         Map<String, JsonNode> indexed = ops.getDocuments(tableName, sample.keySet());
+         for(Map.Entry<String, QRecord> entry : sample.entrySet())
+         {
+            JsonNode source = indexed.get(entry.getKey());
+            if(source == null)
+            {
+               missing.add(entry.getKey());
+               continue;
+            }
+            List<String> differing = differingFields(tableConfig, IndexingUtils.buildDocument(entry.getValue(), tableConfig), source);
+            if(!differing.isEmpty())
+            {
+               staleFields.put(entry.getKey(), differing);
+            }
+         }
+
+         findings.add(missing.isEmpty() && staleFields.isEmpty()
+            ? new Finding(tableName, "Sampled records", Level.OK, sampled + " record(s) checked: all indexed and current.")
+            : new Finding(tableName, "Sampled records", Level.FAIL, sampled + " record(s) checked: " + missing.size() + " missing" + listIds(missing) + ", "
+            + staleFields.size() + " stale" + listIds(new ArrayList<>(staleFields.keySet())) + ". Reindex those records, or run reconcile for the table."));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Sampled records", Level.FAIL, e.getMessage()));
+      }
+
+      List<String> extra            = new ArrayList<>();
+      int          documentsSampled = 0;
+      try
+      {
+         List<String> documentIds = ops.sampleRecordIds(tableName, sampleSize);
+         documentsSampled = documentIds.size();
+         Set<String> existing = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, documentIds));
+         for(String recordId : documentIds)
+         {
+            if(!existing.contains(recordId))
+            {
+               extra.add(recordId);
+            }
+         }
+         findings.add(extra.isEmpty()
+            ? new Finding(tableName, "Sampled documents", Level.OK, documentsSampled + " random document(s) checked: every one has its record.")
+            : new Finding(tableName, "Sampled documents", Level.FAIL, documentsSampled + " random document(s) checked: " + extra.size() + " have no record (orphans)"
+            + listIds(extra) + ". Reconcile removes them."));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Sampled documents", Level.FAIL, e.getMessage()));
+      }
+
+      checkIndexRow(runtime, tableName, findings);
+      checkFailedEvents(runtime, tableName, findings);
+
+      return (new TableStats(tableName, databaseCount, indexCount, sampled, missing, staleFields, documentsSampled, extra));
+   }
+
+
+
+   /*******************************************************************************
+    ** Names of the values that differ between an expected document and an
+    ** indexed source (recordLabel, searchableText, fieldValues.x).
+    *******************************************************************************/
+   static List<String> differingFields(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, JsonNode source)
+   {
+      return (RecordInspector.diff(tableConfig, expected, new QuickSearchDiagnosticsOperations.DocumentView(true, null, null, source)).stream()
+         .filter(d -> !d.isSame())
+         .map(RecordInspector.FieldDiff::name)
+         .toList());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void checkIndexRow(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
+   {
+      try
+      {
+         String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+         if(QContext.getQInstance().getTable(indexTable) == null)
+         {
+            return;
+         }
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(indexTable);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "No Quick Search Index row yet: no basepull, reconcile or reindex has run for this table."));
+            return;
+         }
+
+         QRecord row     = rows.get(0);
+         String  status  = row.getValueString("status");
+         String  lastRun = row.getValueString("lastRunStatus");
+         if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.INFO, "Disabled: Quick Search neither indexes nor searches this table, so differences are expected."));
+         }
+         if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "Status NEEDS_REINDEX: the configured fields changed since the last reindex. Rebuild the table."));
+         }
+         else if(AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "Status REBUILDING: a full reindex is running. If it was killed, basepull returns the table to ACTIVE once its run records show no sign of life for fullReindexStaleMinutes."));
+         }
+         if(AbstractIndexingStep.RUN_FAILED.equals(lastRun))
+         {
+            findings.add(new Finding(tableName, "Last run", Level.WARN, "The last indexing run failed: " + row.getValueString("lastErrorMessage")));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read the Quick Search Index row during validation", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void checkFailedEvents(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
+   {
+      try
+      {
+         String failedTable = runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+         if(QContext.getQInstance().getTable(failedTable) == null)
+         {
+            return;
+         }
+         CountInput countInput = new CountInput();
+         countInput.setTableName(failedTable);
+         countInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+            new QFilterCriteria("status", QCriteriaOperator.IN, List.of(QuickSearchFailedEvent.STATUS_PENDING, QuickSearchFailedEvent.STATUS_EXHAUSTED))));
+         Integer count = new CountAction().execute(countInput).getCount();
+         if(count != null && count > 0)
+         {
+            findings.add(new Finding(tableName, "Failed events", Level.WARN, count + " real-time index event(s) failed and wait for replay (or are exhausted). See Quick Search Failed Event."));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not count failed events during validation", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Save a report as a quick_search_validation_run row (as the system user);
+    ** returns its id, or null when the table is not configured or the insert
+    ** fails (logged).
+    *******************************************************************************/
+   public Integer persist(QuickSearchRuntime runtime, Report report, String triggerType, String userName)
+   {
+      String tableName = runtime.getConfig().applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
+      if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
+      {
+         return (null);
+      }
+
+      QuickSearchValidationRun run = new QuickSearchValidationRun()
+         .withStatus(report.status())
+         .withTriggerType(triggerType)
+         .withUserName(userName)
+         .withTableNames(DiagnosticEventRecorder.truncate(String.join(",", report.tableNames()), 1000))
+         .withSampleSize(report.sampleSize())
+         .withTablesChecked(report.tableStats().size())
+         .withFailCount(report.count(Level.FAIL))
+         .withWarnCount(report.count(Level.WARN))
+         .withSummary(DiagnosticEventRecorder.truncate(summary(report), 250))
+         .withReportHtml(render(report))
+         .withReportJson(toJson(report))
+         .withStartTime(report.startTime())
+         .withEndTime(report.endTime())
+         .withCreateDate(Instant.now());
+
+      Integer[] id = new Integer[1];
+      try
+      {
+         QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
+         {
+            InsertOutput output = new InsertAction().execute(new InsertInput(tableName).withRecordEntity(run));
+            id[0] = output.getRecords().get(0).getValueInteger("id");
+         });
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not save the Quick Search validation report", e);
+      }
+      return (id[0]);
+   }
+
+
+
+   /*******************************************************************************
+    ** One line: status and counts.
+    *******************************************************************************/
+   public static String summary(Report report)
+   {
+      return (report.status() + ": " + report.tableStats().size() + " table(s), " + report.count(Level.FAIL) + " failed check(s), " + report.count(Level.WARN) + " warning(s).");
+   }
+
+
+
+   /*******************************************************************************
+    ** The report as JSON.
+    *******************************************************************************/
+   public static String toJson(Report report)
+   {
+      try
+      {
+         return (MAPPER.writeValueAsString(report));
+      }
+      catch(Exception e)
+      {
+         return ("{\"error\":\"report could not be serialized\"}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The report as HTML: summary, then the findings grouped by scope.
+    *******************************************************************************/
+   public static String render(Report report)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      Level overall = switch(report.status())
+      {
+         case QuickSearchValidationRun.STATUS_PASS -> Level.OK;
+         case QuickSearchValidationRun.STATUS_WARN -> Level.WARN;
+         default -> Level.FAIL;
+      };
+      html.heading("Index validation");
+      html.status(overall, summary(report));
+      html.paragraph("Ran " + report.startTime() + " to " + report.endTime() + ", " + report.sampleSize() + " sampled record(s) and document(s) per table. Database reads ran as the system user.");
+
+      Map<String, List<DiagnosticsHtml.Check>> byScope = new LinkedHashMap<>();
+      for(Finding finding : report.findings())
+      {
+         byScope.computeIfAbsent(finding.scope(), k -> new ArrayList<>()).add(new DiagnosticsHtml.Check(finding.check(), finding.level(), finding.message()));
+      }
+      for(Map.Entry<String, List<DiagnosticsHtml.Check>> scope : byScope.entrySet())
+      {
+         String heading = switch(scope.getKey())
+         {
+            case SCOPE_CLUSTER -> "Cluster and index";
+            case SCOPE_MAPPING -> "Mapping and analyzers";
+            default -> "Table " + scope.getKey();
+         };
+         html.subheading(heading);
+         html.checks(scope.getValue());
+      }
+
+      for(TableStats stats : report.tableStats())
+      {
+         if(!stats.staleFields().isEmpty())
+         {
+            List<String> items = new ArrayList<>();
+            stats.staleFields().entrySet().stream().limit(MAX_LISTED_IDS).forEach(e ->
+               items.add("#" + DiagnosticsHtml.escape(e.getKey()) + ": " + DiagnosticsHtml.escape(String.join(", ", e.getValue()))));
+            html.subheading("Stale fields in " + stats.tableName());
+            html.rawList(items);
+         }
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String listIds(List<String> ids)
+   {
+      if(ids.isEmpty())
+      {
+         return ("");
+      }
+      return (" (" + String.join(", ", ids.subList(0, Math.min(MAX_LISTED_IDS, ids.size()))) + (ids.size() > MAX_LISTED_IDS ? ", ..." : "") + ")");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/QuickSearchDeveloperPanelProvider.java

@@ -0,0 +1,535 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanel;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelAction;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelBlock;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelContext;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelKeyValue;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelLink;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelProviderInterface;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelTone;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The "Quick Search" panel of a record's developer view (QQQ's
+ ** DeveloperPanelProviderInterface, /app/{table}/{id}/dev).
+ **
+ ** Shown for tables this QBit indexes, to users who may read the Quick Search
+ ** Index table. It shows the table's index row (enabled, status, counts, last
+ ** basepull and full reindex, searchable fields), the latest index run, the
+ ** record's indexed document compared with the database (Inspect Search
+ ** Document's comparison; needs the debug permission), the record's failed
+ ** index events, and the Inspect / Reindex record actions and admin links the
+ ** user may use. Every read checks the user's permissions; an unreachable
+ ** OpenSearch shows a warning instead of failing the panel.
+ *******************************************************************************/
+public class QuickSearchDeveloperPanelProvider implements DeveloperPanelProviderInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchDeveloperPanelProvider.class);
+
+   public static final String ID    = "quickSearch";
+   public static final String TITLE = "Quick Search";
+
+   public static final String STATUS_INDEXED     = "Indexed";
+   public static final String STATUS_STALE       = "Stale";
+   public static final String STATUS_NOT_INDEXED = "Not indexed";
+   public static final String STATUS_DISABLED    = "Disabled";
+   public static final String STATUS_ENABLED     = "Enabled";
+   public static final String STATUS_UNAVAILABLE = "OpenSearch unavailable";
+
+   static final int MAX_FAILED_EVENTS = 20;
+   static final int MAX_VALUE_LENGTH  = 300;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getId()
+   {
+      return (ID);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getTitle()
+   {
+      return (TITLE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Integer getOrder()
+   {
+      return (400);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getIcon()
+   {
+      return ("search");
+   }
+
+
+
+   /*******************************************************************************
+    ** Available for a table this QBit indexes, when the user may read the Quick
+    ** Search Index table (the panel shows admin-level indexing state).
+    *******************************************************************************/
+   @Override
+   public boolean isAvailable(DeveloperPanelContext context)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || context == null || runtime.getTableConfig(context.getTableName()) == null)
+      {
+         return (false);
+      }
+      return (mayRead(runtime.getConfig().getQuickSearchIndexTableName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the panel.
+    *******************************************************************************/
+   @Override
+   public DeveloperPanel build(DeveloperPanelContext context) throws QException
+   {
+      QuickSearchRuntime          runtime     = QuickSearchRuntime.getOrThrow();
+      QuickSearchQBitConfig       config      = runtime.getConfig();
+      String                      tableName   = context.getTableName();
+      QuickSearchableTableConfig  tableConfig = runtime.getTableConfig(tableName);
+      Serializable                primaryKey  = context.getPrimaryKey();
+      String                      recordId    = primaryKey == null ? null : String.valueOf(primaryKey);
+      String                      indexTable  = config.getQuickSearchIndexTableName();
+
+      DeveloperPanel panel = new DeveloperPanel()
+         .withDescription("How Quick Search (OpenSearch) indexes this table, and this record's search document.");
+
+      //////////////////////////////////////////////////////////////
+      // the table's index row (readable: checked by isAvailable) //
+      //////////////////////////////////////////////////////////////
+      QRecord indexRow = null;
+      if(mayRead(indexTable))
+      {
+         indexRow = queryFirst(indexTable, new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+      }
+      boolean enabled = indexRow == null || !Boolean.FALSE.equals(indexRow.getValueBoolean("enabled"));
+      panel.withBlock(indexBlock(config, tableConfig, indexTable, indexRow, enabled));
+
+      //////////////////////
+      // latest index run //
+      //////////////////////
+      String runTable = config.getQuickSearchIndexRunTableName();
+      if(indexRow != null && mayRead(runTable))
+      {
+         QRecord run = queryFirst(runTable, new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexRow.getValue("id")))
+            .withOrderBy(new QFilterOrderBy("id", false)));
+         panel.withBlock(run == null
+            ? DeveloperPanelBlock.text("No index runs are recorded for this table yet.", DeveloperPanelTone.INFO).withLabel("Latest index run")
+            : runBlock(runTable, run));
+      }
+
+      /////////////////////////////////////////
+      // this record's document vs. database //
+      /////////////////////////////////////////
+      String verdict = null;
+      boolean openSearchFailed = false;
+      if(recordId != null)
+      {
+         if(!Boolean.TRUE.equals(config.getEnableDiagnosticTools()))
+         {
+            panel.withBlock(DeveloperPanelBlock.text("Quick Search diagnostic tools are disabled (enableDiagnosticTools), so this record's search document is not compared.", DeveloperPanelTone.INFO)
+               .withLabel("Search document"));
+         }
+         else if(!mayInspect(config, tableName))
+         {
+            panel.withBlock(DeveloperPanelBlock.text("You do not have the Quick Search debug permission, so this record's search document is not shown.", DeveloperPanelTone.INFO)
+               .withLabel("Search document"));
+         }
+         else
+         {
+            try
+            {
+               RecordInspector.Inspection inspection = inspect(runtime, tableName, recordId, new QueryInput(tableName));
+               verdict = inspection.verdict();
+               addInspectionBlocks(panel, inspection);
+            }
+            catch(Exception e)
+            {
+               openSearchFailed = true;
+               LOG.info("Quick Search developer panel could not read the search document", logPair("tableName", tableName), logPair("recordId", recordId), logPair("error", e.getMessage()));
+               panel.withBlock(DeveloperPanelBlock.text("OpenSearch could not be reached, so this record's search document cannot be shown: "
+                  + DiagnosticsSupport.rootCause(e).getMessage(), DeveloperPanelTone.WARNING).withLabel("Search document"));
+            }
+         }
+      }
+
+      /////////////////////////////////
+      // failed index events, if any //
+      /////////////////////////////////
+      String failedTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+      if(recordId != null && mayRead(failedTable))
+      {
+         DeveloperPanelBlock failedBlock = failedEventsBlock(failedTable, tableName, recordId);
+         if(failedBlock != null)
+         {
+            panel.withBlock(failedBlock);
+         }
+      }
+
+      //////////////////////////////
+      // actions the user may run //
+      //////////////////////////////
+      if(recordId != null)
+      {
+         DeveloperPanelBlock actions = DeveloperPanelBlock.actions("Actions");
+         addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName), tableName, "Inspect search document",
+            "Compare the indexed document with the database, and see why the record does or does not match a term.");
+         addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config, tableName), tableName, "Reindex this record",
+            "Rebuild this record's search document from the database now.");
+         if(!actions.getActions().isEmpty())
+         {
+            panel.withBlock(actions);
+         }
+      }
+
+      ///////////
+      // links //
+      ///////////
+      DeveloperPanelBlock links   = DeveloperPanelBlock.links("Links");
+      String              appName = config.applyPrefix(QuickSearchAdminAppMetaDataProducer.APP_NAME);
+      if(QContext.getQInstance().getApp(appName) != null && PermissionsHelper.hasAppPermission(new QueryInput(indexTable), appName))
+      {
+         links.withLink(new DeveloperPanelLink().withLabel("Quick Search Admin").withHref("/app/" + appName).withDescription("Index status, runs, failed events and diagnostic tools."));
+      }
+      if(indexRow != null)
+      {
+         links.withLink(new DeveloperPanelLink().withLabel("Quick Search Index row for " + tableName).withHref(recordHref(indexTable, indexRow.getValue("id"))));
+      }
+      if(!links.getLinks().isEmpty())
+      {
+         panel.withBlock(links);
+      }
+
+      setStatus(panel, enabled, verdict, openSearchFailed);
+      return (panel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect one record (overridable for tests).
+    *******************************************************************************/
+   protected RecordInspector.Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, AbstractActionInput permissionInput) throws QException
+   {
+      return (new RecordInspector().inspect(runtime, tableName, recordId, null, permissionInput));
+   }
+
+
+
+   /*******************************************************************************
+    ** The table's index row as key/values.
+    *******************************************************************************/
+   private DeveloperPanelBlock indexBlock(QuickSearchQBitConfig config, QuickSearchableTableConfig tableConfig, String indexTable, QRecord indexRow, boolean enabled)
+   {
+      DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Index");
+      if(indexRow == null)
+      {
+         block.withItem("Index row", "none yet (the first indexing run creates it)");
+      }
+      else
+      {
+         block.withItem(new DeveloperPanelKeyValue().withLabel("Index row").withValue("#" + indexRow.getValueString("id")).withHref(recordHref(indexTable, indexRow.getValue("id"))));
+      }
+      block.withItem("Enabled", enabled ? "Yes" : "No");
+      block.withItem("Status", indexRow == null ? null : indexRow.getValueString("status"));
+      block.withItem("Indexed records", indexRow == null ? null : indexRow.getValueString("recordCount"));
+      block.withItem("Last basepull", indexRow == null ? null : indexRow.getValueString("lastBasepullTime"));
+      block.withItem("Last full reindex", indexRow == null ? null : indexRow.getValueString("lastFullReindexTime"));
+      block.withItem("Searchable fields", String.join(", ", CollectionUtils.nonNullList(tableConfig.getSearchableFields())));
+      block.withItem(new DeveloperPanelKeyValue().withLabel("OpenSearch index (alias)").withValue(config.getOpensearchIndexName()).withMonospace(true));
+      block.withItem("Real-time indexing", Boolean.TRUE.equals(config.getEnableRealTimeIndexing()) ? "On" : "Off");
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** The latest index run as key/values.
+    *******************************************************************************/
+   private DeveloperPanelBlock runBlock(String runTable, QRecord run)
+   {
+      DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Latest index run")
+         .withItem(new DeveloperPanelKeyValue().withLabel("Run").withValue("#" + run.getValueString("id")).withHref(recordHref(runTable, run.getValue("id"))))
+         .withItem("Type", run.getValueString("runType"))
+         .withItem("Status", run.getValueString("status"))
+         .withItem("Started", run.getValueString("startTime"))
+         .withItem("Ended", run.getValueString("endTime"))
+         .withItem("Processed", run.getValueString("recordsProcessed"))
+         .withItem("Indexed", run.getValueString("recordsIndexed"))
+         .withItem("Errors", run.getValueString("errorCount"));
+      if(run.getValueString("errorMessage") != null)
+      {
+         block.withItem("Error", truncate(run.getValueString("errorMessage")));
+      }
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** The inspection: verdict text, document facts, field comparison table.
+    *******************************************************************************/
+   private void addInspectionBlocks(DeveloperPanel panel, RecordInspector.Inspection inspection)
+   {
+      if(inspection.error() != null)
+      {
+         panel.withBlock(DeveloperPanelBlock.text(inspection.error(), RecordInspector.VERDICT_NOT_INDEXED.equals(inspection.verdict()) ? DeveloperPanelTone.INFO : DeveloperPanelTone.WARNING)
+            .withLabel("Search document"));
+         return;
+      }
+
+      switch(inspection.verdict())
+      {
+         case RecordInspector.VERDICT_CURRENT -> panel.withBlock(DeveloperPanelBlock.text("Indexed and current: the search document matches the database.", DeveloperPanelTone.SUCCESS)
+            .withLabel("Search document"));
+         case RecordInspector.VERDICT_STALE -> panel.withBlock(DeveloperPanelBlock.text("Indexed but stale: some values differ from the database. Reindex this record to refresh it.", DeveloperPanelTone.WARNING)
+            .withLabel("Search document"));
+         default -> panel.withBlock(DeveloperPanelBlock.text("Not in the index: the global search cannot find this record. Reindex this record to add it (or wait for the next basepull or reconcile).",
+            DeveloperPanelTone.ERROR).withLabel("Search document"));
+      }
+
+      DocumentView indexed = inspection.indexed();
+      panel.withBlock(DeveloperPanelBlock.keyValues("Document")
+         .withItem(new DeveloperPanelKeyValue().withLabel("Document id").withValue(OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId())).withMonospace(true))
+         .withItem(new DeveloperPanelKeyValue().withLabel("Physical index").withValue(indexed == null ? null : indexed.physicalIndex()).withMonospace(true))
+         .withItem("Indexed version", indexed == null || indexed.version() == null ? null : String.valueOf(indexed.version()))
+         .withItem("Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)" : String.valueOf(inspection.expected().getVersion()))
+         .withItem("Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null)));
+
+      DeveloperPanelBlock table = DeveloperPanelBlock.table("Database vs index", List.of("Field", "Database", "Index", "Match"));
+      for(RecordInspector.FieldDiff diff : CollectionUtils.nonNullList(inspection.diffs()))
+      {
+         table.withRow(row(diff.name(), truncate(diff.databaseValue()), truncate(diff.indexValue()), diff.isSame() ? "yes" : "NO"));
+      }
+      panel.withBlock(table);
+   }
+
+
+
+   /*******************************************************************************
+    ** The record's failed index events, or null when there are none.
+    *******************************************************************************/
+   private DeveloperPanelBlock failedEventsBlock(String failedTable, String tableName, String recordId) throws QException
+   {
+      QueryInput queryInput = new QueryInput(failedTable);
+      queryInput.setFilter(new QQueryFilter(
+         new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+         new QFilterCriteria("recordId", QCriteriaOperator.EQUALS, recordId))
+         .withOrderBy(new QFilterOrderBy("id", false))
+         .withLimit(MAX_FAILED_EVENTS));
+      List<QRecord> events = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+      if(events.isEmpty())
+      {
+         return (null);
+      }
+
+      DeveloperPanelBlock block = DeveloperPanelBlock.table("Failed index events" + (events.size() >= MAX_FAILED_EVENTS ? " (latest " + MAX_FAILED_EVENTS + ")" : ""),
+         List.of("Id", "Action", "Status", "Attempts", "Created", "Error"));
+      for(QRecord event : events)
+      {
+         block.withRow(row(event.getValueString("id"), event.getValueString("action"), event.getValueString("status"), event.getValueString("attempts"),
+            event.getValueString("createDate"), truncate(event.getValueString("errorMessage"))));
+      }
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add a record action when the process exists and the user may run it.
+    *******************************************************************************/
+   private void addActionIfPermitted(DeveloperPanelBlock actions, String processName, String tableName, String label, String description)
+   {
+      if(QContext.getQInstance().getProcess(processName) != null && PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName))
+      {
+         actions.withAction(new DeveloperPanelAction().withLabel(label).withProcessName(processName).withTableName(tableName).withDescription(description));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the user may inspect records: the Inspect Search Document
+    ** process's permission (debugPermissionRules), evaluated on the
+    ** registered process, or on an unregistered copy when the record actions
+    ** are turned off.
+    *******************************************************************************/
+   private boolean mayInspect(QuickSearchQBitConfig config, String tableName)
+   {
+      String processName = QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName);
+      if(QContext.getQInstance().getProcess(processName) != null)
+      {
+         return (PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName));
+      }
+      return (PermissionsHelper.getPermissionCheckResult(new QueryInput(tableName), QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(config, tableName))
+         == PermissionCheckResult.ALLOW);
+   }
+
+
+
+   /*******************************************************************************
+    ** Status label and tone from the enabled flag and the inspection verdict.
+    *******************************************************************************/
+   private void setStatus(DeveloperPanel panel, boolean enabled, String verdict, boolean openSearchFailed)
+   {
+      if(!enabled)
+      {
+         panel.withStatusLabel(STATUS_DISABLED).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(openSearchFailed)
+      {
+         panel.withStatusLabel(STATUS_UNAVAILABLE).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(RecordInspector.VERDICT_CURRENT.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_INDEXED).withStatusTone(DeveloperPanelTone.SUCCESS);
+      }
+      else if(RecordInspector.VERDICT_STALE.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_STALE).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(RecordInspector.VERDICT_MISSING.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_NOT_INDEXED).withStatusTone(DeveloperPanelTone.ERROR);
+      }
+      else
+      {
+         panel.withStatusLabel(STATUS_ENABLED).withStatusTone(DeveloperPanelTone.INFO);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the table exists and the user may read it.
+    *******************************************************************************/
+   private static boolean mayRead(String tableName)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      return (qInstance != null && qInstance.getTable(tableName) != null && PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
+   }
+
+
+
+   /*******************************************************************************
+    ** First row of a query (as the user), or null.
+    *******************************************************************************/
+   private static QRecord queryFirst(String tableName, QQueryFilter filter) throws QException
+   {
+      QueryInput queryInput = new QueryInput(tableName);
+      queryInput.setFilter(filter.withLimit(1));
+      List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeHasContents(records) ? records.get(0) : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** App-relative link to a record.
+    *******************************************************************************/
+   private static String recordHref(String tableName, Serializable id)
+   {
+      return ("/app/" + tableName + "/" + id);
+   }
+
+
+
+   /*******************************************************************************
+    ** A table row (null cells as empty strings).
+    *******************************************************************************/
+   private static List<String> row(String... cells)
+   {
+      List<String> row = new ArrayList<>();
+      for(String cell : cells)
+      {
+         row.add(cell == null ? "" : cell);
+      }
+      return (row);
+   }
+
+
+
+   /*******************************************************************************
+    ** Long values cut for display.
+    *******************************************************************************/
+   static String truncate(String value)
+   {
+      if(value == null || value.length() <= MAX_VALUE_LENGTH)
+      {
+         return (value);
+      }
+      return (value.substring(0, MAX_VALUE_LENGTH) + "... (" + value.length() + " characters)");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/QuickSearchDiagnosticsWidgetRenderer.java

@@ -0,0 +1,253 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** The Search Diagnostics widget: the record-search provider's state on the
+ ** server that renders it, the most recent search failures (from every
+ ** server, via Quick Search Diagnostic Event), the latest validation run,
+ ** and where the tools are. Every part says plainly when it has nothing to
+ ** show or cannot be read.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchDiagnosticsWidgetRenderer.class);
+
+   public static final int RECENT_FAILURES = 10;
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", "<p>Quick Search is not configured in this application.</p>")));
+      }
+      return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", renderHtml(runtime))));
+   }
+
+
+
+   /*******************************************************************************
+    ** The widget's HTML.
+    *******************************************************************************/
+   static String renderHtml(QuickSearchRuntime runtime)
+   {
+      QuickSearchQBitConfig config = runtime.getConfig();
+      DiagnosticsHtml       html   = new DiagnosticsHtml().open();
+
+      renderIndexingStatus(html, runtime);
+
+      SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(QContext.getQInstance(), runtime);
+      html.subheading("Global search on this server");
+      if(!Boolean.TRUE.equals(state.quickSearchProvider()) || !Boolean.TRUE.equals(state.serveCoreRecordSearch()))
+      {
+         html.status(Level.INFO, "Quick Search does not serve the global search here (" + (state.registeredProvider() == null ? "no provider registered" : "provider " + state.registeredProvider())
+            + "); core searches every table itself.");
+      }
+      else if(state.backoffRemainingMillis() > 0)
+      {
+         html.status(Level.FAIL, "Backing off for " + (state.backoffRemainingMillis() / 1000) + " s after an OpenSearch failure: the global search uses core's database search meanwhile. Last error: "
+            + state.lastFailureMessage());
+      }
+      else
+      {
+         html.status(Level.OK, "Served by OpenSearch through Quick Search (timeout " + state.timeoutMillis() + " ms).");
+      }
+      html.facts(List.of(
+         new String[] { "Failed OpenSearch searches since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) },
+         new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) },
+         new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() }));
+
+      renderRecentFailures(html, config);
+      renderLastValidation(html, config);
+
+      html.subheading("Tools");
+      html.paragraph("On Quick Search Index (Actions): Add Search Index, Apply Search Index Changes, Disable and Remove Search Index, "
+         + "Search Playground, Validate Search Index, Test Search Analyzer. Searchable fields are edited on each index record. "
+         + "On Quick Search Settings: Pause and Resume Indexing; on Quick Search Index Run: Cancel Index Run. "
+         + "On any indexed record and on Quick Search Failed Event rows: Inspect Search Document and Reindex in Search. "
+         + "Every run is recorded in Quick Search Diagnostic Event.");
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Indexing: paused (since when, by whom), what self-priming added and is
+    ** building, and field changes waiting to be applied.
+    *******************************************************************************/
+   static void renderIndexingStatus(DiagnosticsHtml html, QuickSearchRuntime runtime)
+   {
+      QuickSearchStatus status = QuickSearchStatus.read(runtime);
+      html.subheading("Indexing");
+      String banner = status.bannerHtml();
+      if(!banner.isEmpty())
+      {
+         html.raw(banner);
+      }
+      if(status.paused())
+      {
+         html.raw("<p><a href=\"" + DiagnosticsHtml.escape(QuickSearchStatus.settingsPath(runtime)) + "\">Quick Search Settings</a>: Resume Indexing (Actions).</p>");
+      }
+      else
+      {
+         html.status(Level.OK, "Indexing is running (real-time and scheduled). Pause it from "
+            + "Quick Search Settings (Actions: Pause Indexing); search keeps serving while paused.");
+      }
+      if(!status.recordsMode())
+      {
+         html.status(Level.INFO, "The searchable tables come from code: the Quick Search config tables are not in this application yet, or not primed. "
+            + "See docs/MIGRATION-1.1.md (data-driven config).");
+      }
+      else if(status.lastPrimedAt() != null)
+      {
+         html.paragraph("Self-priming ran " + status.lastPrimedAt() + ": " + (status.lastPrimingSummary() == null ? "no changes." : status.lastPrimingSummary()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderRecentFailures(DiagnosticsHtml html, QuickSearchQBitConfig config)
+   {
+      String tableName = config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
+      html.subheading("Recent search failures (all servers)");
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         html.paragraph("Not recorded: the diagnostic event table is not configured.");
+         return;
+      }
+
+      try
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("eventType", QCriteriaOperator.EQUALS, QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED))
+            .withOrderBy(new QFilterOrderBy("id", false))
+            .withLimit(RECENT_FAILURES));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            html.status(Level.OK, "None recorded (failures are kept for " + config.getRunHistoryRetentionDays() + " days).");
+            return;
+         }
+
+         List<String> items = new ArrayList<>();
+         for(QRecord row : rows)
+         {
+            items.add("<a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
+               + DiagnosticsHtml.escape(row.getValueString("createDate")) + "</a> on " + DiagnosticsHtml.escape(row.getValueString("hostName"))
+               + ": " + DiagnosticsHtml.escape(row.getValueString("source")) + ", " + DiagnosticsHtml.escape(row.getValueString("durationMillis")) + " ms, "
+               + DiagnosticsHtml.escape(shortClass(row.getValueString("errorClass"))) + ": " + DiagnosticsHtml.escape(abbreviate(row.getValueString("errorMessage"), 240)));
+         }
+         html.rawList(items);
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read recent search failures for the diagnostics widget", e);
+         html.status(Level.WARN, "Could not read the diagnostic events: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderLastValidation(DiagnosticsHtml html, QuickSearchQBitConfig config)
+   {
+      String tableName = config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
+      html.subheading("Latest index validation");
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         html.paragraph("Not kept: the validation run table is not configured.");
+         return;
+      }
+
+      try
+      {
+         QRecord row = DiagnosticsSupport.latestValidation(config);
+         if(row == null)
+         {
+            html.paragraph("None yet. Run Validate Search Index from Quick Search Index (Actions)"
+               + (config.getValidationCronExpression() == null ? "." : "; it is also scheduled (" + config.getValidationCronExpression() + ")."));
+            return;
+         }
+
+         String  status = row.getValueString("status");
+         Level   level  = QuickSearchValidationRun.STATUS_PASS.equals(status) ? Level.OK : (QuickSearchValidationRun.STATUS_WARN.equals(status) ? Level.WARN : Level.FAIL);
+         html.raw("<p>" + DiagnosticsHtml.badge(level) + " <a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
+            + DiagnosticsHtml.escape(row.getValueString("startTime")) + "</a> (" + DiagnosticsHtml.escape(row.getValueString("triggerType")) + "): "
+            + DiagnosticsHtml.escape(row.getValueString("summary")) + "</p>");
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read the latest validation run for the diagnostics widget", e);
+         html.status(Level.WARN, "Could not read the validation runs: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String shortClass(String className)
+   {
+      return (className == null ? "error" : className.substring(className.lastIndexOf('.') + 1));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String abbreviate(String text, int max)
+   {
+      return (text == null ? "" : (text.length() > max ? text.substring(0, max) + "..." : text));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/RecordInspector.java

@@ -0,0 +1,410 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Check;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** Inspect one record's search document: the document as indexed next to the
+ ** document indexing would write from the database now (field by field),
+ ** and, given a term, why the record does or does not match it.
+ ** The match analysis walks the same gates as the global search: term
+ ** length, table indexed and enabled, read permission, record readable,
+ ** document present and current, every query token present among the
+ ** document's indexed tokens (edge n-grams of 2 to 20 characters, AND
+ ** semantics), explain, and the record's rank against the provider's
+ ** per-table id window. Read-only.
+ *******************************************************************************/
+public class RecordInspector
+{
+   public static final String VERDICT_CURRENT     = "CURRENT";
+   public static final String VERDICT_STALE       = "STALE";
+   public static final String VERDICT_MISSING     = "MISSING";
+   public static final String VERDICT_NOT_INDEXED = "NOT_INDEXED";
+   public static final String VERDICT_NO_RECORD   = "NO_RECORD";
+
+   private static final int RANK_WINDOW = RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE;
+
+
+
+   /*******************************************************************************
+    ** One compared value: name, value from the database, value in the index.
+    *******************************************************************************/
+   public record FieldDiff(String name, String databaseValue, String indexValue)
+   {
+      /*******************************************************************************
+       ** Whether the two values are equal.
+       *******************************************************************************/
+      public boolean isSame()
+      {
+         return (Objects.equals(databaseValue, indexValue));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The inspection of one record.
+    *******************************************************************************/
+   public record Inspection(String tableName, String recordId, String verdict, List<FieldDiff> diffs, DocumentView indexed, OpenSearchDocument expected,
+                            List<Check> matchChecks, ExplainResult explain, String error)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect one record, with an optional term to analyze.
+    *******************************************************************************/
+   public Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, String term, AbstractActionInput permissionInput) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         return (new Inspection(tableName, recordId, VERDICT_NOT_INDEXED, List.of(), null, null, List.of(), null, "Quick Search does not index table [" + tableName + "]."));
+      }
+
+      Map<String, QRecord> records = DiagnosticRecordReads.loadForIndexing(tableName, List.of(recordId));
+      QRecord              record  = records.get(recordId);
+      if(record == null && !records.isEmpty())
+      {
+         record = records.values().iterator().next();
+      }
+      if(record == null)
+      {
+         return (new Inspection(tableName, recordId, VERDICT_NO_RECORD, List.of(), null, null, List.of(), null,
+            "Record " + recordId + " does not exist, or you cannot read it."));
+      }
+
+      QuickSearchDiagnosticsOperations ops      = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      OpenSearchDocument               expected = IndexingUtils.buildDocument(record, tableConfig);
+      DocumentView                     indexed  = ops.getDocument(tableName, recordId);
+      List<FieldDiff>                  diffs    = diff(tableConfig, expected, indexed);
+
+      String verdict = !Boolean.TRUE.equals(indexed.found()) ? VERDICT_MISSING : (diffs.stream().allMatch(FieldDiff::isSame) ? VERDICT_CURRENT : VERDICT_STALE);
+
+      List<Check>   checks  = new ArrayList<>();
+      ExplainResult explain = null;
+      String        cleaned = IndexingUtils.normalizeSearchText(term);
+      if(StringUtils.hasContent(cleaned))
+      {
+         explain = analyzeMatch(runtime, ops, tableConfig, recordId, cleaned, verdict, expected, indexed, checks, permissionInput);
+      }
+
+      return (new Inspection(tableName, recordId, verdict, diffs, indexed, expected, checks, explain, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare the expected document with the indexed one: label, searchable
+    ** text, then each configured field (in configuration order) and any extra
+    ** field only the index holds.
+    *******************************************************************************/
+   static List<FieldDiff> diff(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, DocumentView indexed)
+   {
+      JsonNode source = indexed == null || indexed.source() == null ? null : indexed.source();
+
+      List<FieldDiff> diffs = new ArrayList<>();
+      diffs.add(new FieldDiff("recordLabel", expected == null ? null : blankToNull(expected.getRecordLabel()), text(source, "recordLabel")));
+      diffs.add(new FieldDiff("searchableText", expected == null ? null : blankToNull(expected.getSearchableText()), text(source, "searchableText")));
+
+      Set<String> fieldNames = new LinkedHashSet<>(CollectionUtils.nonNullList(tableConfig.getSearchableFields()));
+      if(source != null && source.has("fieldValues"))
+      {
+         source.get("fieldValues").fieldNames().forEachRemaining(fieldNames::add);
+      }
+      for(String field : fieldNames)
+      {
+         Object expectedValue = expected == null || expected.getFieldValues() == null ? null : expected.getFieldValues().get(field);
+         diffs.add(new FieldDiff("fieldValues." + field, expectedValue == null ? null : String.valueOf(expectedValue),
+            source == null ? null : text(source.path("fieldValues"), field)));
+      }
+      return (diffs);
+   }
+
+
+
+   /*******************************************************************************
+    ** Walk the global search's gates for this record and term, adding a check
+    ** per gate; returns the explain result when the document exists.
+    *******************************************************************************/
+   private ExplainResult analyzeMatch(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, String recordId, String term, String verdict,
+      OpenSearchDocument expected, DocumentView indexed, List<Check> checks, AbstractActionInput permissionInput) throws QException
+   {
+      String tableName = tableConfig.getTableName();
+
+      checks.add(term.length() < QuickSearchAction.MIN_TERM_LENGTH
+         ? new Check("Term length", Level.FAIL, "Terms shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters never reach OpenSearch.")
+         : new Check("Term length", Level.OK, term.length() + " characters."));
+
+      boolean enabled = runtime.isTableEnabled(tableName);
+      checks.add(enabled
+         ? new Check("Table enabled", Level.OK, "The table is enabled in Quick Search Index.")
+         : new Check("Table enabled", Level.FAIL, "The table is disabled in Quick Search Index, so Quick Search does not search it (core searches it with its own database search)."));
+
+      boolean readable = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
+      checks.add(readable
+         ? new Check("Table permission", Level.OK, "You can read this table.")
+         : new Check("Table permission", Level.FAIL, "You cannot read this table, so the global search skips it for you."));
+
+      checks.add(new Check("Record access", Level.OK, "You can read this record (record security locks allow it)."));
+
+      switch(verdict)
+      {
+         case VERDICT_MISSING -> checks.add(new Check("Indexed document", Level.FAIL,
+            "The record has no document in the index, so OpenSearch cannot return it. Use Reindex in Search on the record (or wait for the next basepull / reconcile)."));
+         case VERDICT_STALE -> checks.add(new Check("Indexed document", Level.WARN,
+            "The indexed document differs from the database (see the comparison); the search matches the indexed values. Reindex the record to refresh it."));
+         default -> checks.add(new Check("Indexed document", Level.OK, "The indexed document matches the database."));
+      }
+
+      List<AnalyzedToken> queryTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
+      Set<String>         indexedGrams = Boolean.TRUE.equals(indexed.found()) ? tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, text(indexed.source(), "searchableText")))
+         : Set.of();
+      Set<String>         expectedGrams = expected == null ? Set.of() : tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, expected.getSearchableText()));
+
+      if(queryTokens.isEmpty())
+      {
+         checks.add(new Check("Term tokens", Level.FAIL, "The term has no searchable characters after analysis."));
+      }
+      for(AnalyzedToken token : queryTokens)
+      {
+         String value = token.token();
+         if(value.length() < 2)
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "One character: the index holds edge n-grams of 2 to 20 characters, so a single character never matches, and every token must match."));
+         }
+         else if(value.length() > 20)
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Longer than 20 characters: the index holds edge n-grams of at most 20 characters, so this token never matches. Search a shorter prefix."));
+         }
+         else if(indexedGrams.contains(value))
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.OK, "Found in the indexed text (as a word or a word prefix)."));
+         }
+         else if(expectedGrams.contains(value))
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.WARN, "Not in the indexed text, but in the database's current values: it will match once the record is reindexed."));
+         }
+         else
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Not in this record's searchable text: no searchable field holds a word starting with it. Every token must match (AND)."));
+         }
+      }
+
+      ExplainResult explain = null;
+      if(Boolean.TRUE.equals(indexed.found()))
+      {
+         explain = ops.explain(term, tableName, recordId, runtime.getDiscoveredTables());
+         checks.add(Boolean.TRUE.equals(explain.matched())
+            ? new Check("OpenSearch explain", Level.OK, "The search query matches this document with score " + String.format("%.4f", explain.score()) + ".")
+            : new Check("OpenSearch explain", Level.FAIL, "The search query does not match this document."));
+
+         if(Boolean.TRUE.equals(explain.matched()))
+         {
+            checks.add(rankCheck(runtime, ops, tableName, recordId, term));
+         }
+      }
+      return (explain);
+   }
+
+
+
+   /*******************************************************************************
+    ** Where the record ranks among the table's matches, against the number of
+    ** ids the provider asks for with the default per-table limit.
+    *******************************************************************************/
+   private Check rankCheck(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, String tableName, String recordId, String term) throws QException
+   {
+      int         window = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
+      SearchProbe probe  = ops.probeSearch(term, tableName, RANK_WINDOW, runtime.getDiscoveredTables());
+      int         rank   = 0;
+      for(ProbeHit hit : probe.hits())
+      {
+         rank++;
+         if(recordId.equals(hit.recordId()))
+         {
+            if(rank <= RecordSearchAction.DEFAULT_LIMIT_PER_TABLE)
+            {
+               return (new Check("Rank", Level.OK, "#" + rank + " of " + probe.totalHits() + " matches in this table: within the global search's default " + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " results."));
+            }
+            if(rank <= window)
+            {
+               return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: Quick Search returns it, but the global search shows only the first "
+                  + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " readable results per table by default. A more specific term ranks it higher."));
+            }
+            return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: beyond the top " + window + " ids the provider requests per table, so the global search does not show it."));
+         }
+      }
+      return (new Check("Rank", Level.WARN, "Not in the top " + RANK_WINDOW + " of " + probe.totalHits() + " matches in this table, so the global search does not show it."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render inspections as HTML.
+    *******************************************************************************/
+   public static String render(List<Inspection> inspections, String term)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      for(Inspection inspection : inspections)
+      {
+         html.heading("Search document for " + inspection.tableName() + " #" + inspection.recordId());
+         if(inspection.error() != null)
+         {
+            html.status(Level.FAIL, inspection.error());
+            continue;
+         }
+
+         switch(inspection.verdict())
+         {
+            case VERDICT_CURRENT -> html.status(Level.OK, "Indexed and current: the document matches the database.");
+            case VERDICT_STALE -> html.status(Level.WARN, "Indexed but stale: some values differ from the database. Reindex in Search refreshes it.");
+            default -> html.status(Level.FAIL, "Not in the index. Reindex in Search adds it.");
+         }
+
+         DocumentView indexed = inspection.indexed();
+         List<String[]> facts = new ArrayList<>();
+         facts.add(new String[] { "Document id", OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId()) });
+         facts.add(new String[] { "Physical index", indexed == null ? null : indexed.physicalIndex() });
+         facts.add(new String[] { "Indexed version", indexed == null || indexed.version() == null ? null : indexed.version() + versionNote(indexed.version()) });
+         facts.add(new String[] { "Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)"
+            : inspection.expected().getVersion() + versionNote(inspection.expected().getVersion()) });
+         facts.add(new String[] { "Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null) });
+         html.facts(facts);
+
+         html.subheading("Database vs index");
+         List<String> items = new ArrayList<>();
+         for(FieldDiff diff : inspection.diffs())
+         {
+            items.add(DiagnosticsHtml.badge(diff.isSame() ? Level.OK : Level.WARN) + " <strong>" + DiagnosticsHtml.escape(diff.name()) + "</strong>"
+               + (diff.isSame() ? ": " + valueHtml(diff.databaseValue())
+               : "<br>database: " + valueHtml(diff.databaseValue()) + "<br>index: " + valueHtml(diff.indexValue())));
+         }
+         html.rawList(items);
+
+         if(!inspection.matchChecks().isEmpty())
+         {
+            html.subheading("Why does (or doesn't) this record match \"" + term + "\"?");
+            html.checks(inspection.matchChecks());
+            if(inspection.explain() != null && inspection.explain().explanationText() != null)
+            {
+               html.textDetails("Score explanation", inspection.explain().explanationText());
+            }
+         }
+
+         if(indexed != null && indexed.source() != null)
+         {
+            html.jsonDetails("Indexed document (JSON)", indexed.source().toString());
+         }
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String valueHtml(String value)
+   {
+      if(value == null)
+      {
+         return ("<em>none</em>");
+      }
+      String shown = value.length() > 500 ? value.substring(0, 500) + "... (" + value.length() + " characters)" : value;
+      return ("<code>" + DiagnosticsHtml.escape(shown) + "</code>");
+   }
+
+
+
+   /*******************************************************************************
+    ** " (time)" for a version that looks like epoch millis.
+    *******************************************************************************/
+   private static String versionNote(Long version)
+   {
+      return (version != null && version > 946_684_800_000L ? " (" + java.time.Instant.ofEpochMilli(version) + ")" : "");
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static Set<String> tokenSet(List<AnalyzedToken> tokens)
+   {
+      Set<String> set = new LinkedHashSet<>();
+      for(AnalyzedToken token : tokens)
+      {
+         set.add(token.token());
+      }
+      return (set);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String text(JsonNode node, String field)
+   {
+      if(node == null || !node.has(field) || node.get(field).isNull())
+      {
+         return (null);
+      }
+      return (blankToNull(node.get(field).isTextual() ? node.get(field).asText() : node.get(field).toString()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String blankToNull(String value)
+   {
+      return (value == null || value.isEmpty() ? null : value);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/RecordReindexer.java

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** Reindex chosen records of one table now: each record the user can read is
+ ** rebuilt from the database and written to the index (versioned, so a newer
+ ** document is never overwritten); a record that no longer exists has its
+ ** document removed; a record that exists but that the user cannot read is
+ ** skipped. The one diagnostic tool that writes to the index; it never
+ ** changes a record. A disabled table is refused.
+ *******************************************************************************/
+public class RecordReindexer
+{
+   public static final String OUTCOME_INDEXED = "INDEXED";
+   public static final String OUTCOME_DELETED = "DELETED";
+   public static final String OUTCOME_SKIPPED = "SKIPPED";
+   public static final String OUTCOME_FAILED  = "FAILED";
+
+
+
+   /*******************************************************************************
+    ** What happened to one record.
+    *******************************************************************************/
+   public record Outcome(String recordId, String outcome, String message)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Reindex the records of a table by id.
+    *******************************************************************************/
+   public List<Outcome> reindex(QuickSearchRuntime runtime, String tableName, List<String> recordIds) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Quick Search does not index table [" + tableName + "]"));
+      }
+      if(!runtime.isTableEnabled(tableName))
+      {
+         throw (new QException("Table [" + tableName + "] is disabled in Quick Search Index; enable it before reindexing its records"));
+      }
+      if(runtime.isIndexingPaused())
+      {
+         throw (new QException("Quick Search indexing is paused; resume it in Quick Search Settings before reindexing records"));
+      }
+
+      Set<String>          ids      = new LinkedHashSet<>(recordIds);
+      Map<String, QRecord> readable = DiagnosticRecordReads.loadForIndexing(tableName, ids);
+      Set<String>          validIds = new HashSet<>();
+      DiagnosticRecordReads.typedIds(QContext.getQInstance().getTable(tableName), ids).forEach(id -> validIds.add(String.valueOf(id)));
+
+      ////////////////////////////////////////////////////////////////////
+      // ids the user cannot read: exists at all (system view)? if not, //
+      // the record is gone and its document should go too              //
+      ////////////////////////////////////////////////////////////////////
+      List<String> unreadable = ids.stream().filter(id -> !readable.containsKey(id)).toList();
+      Set<String>  existing   = new HashSet<>();
+      if(!unreadable.isEmpty())
+      {
+         List<String> found = new ArrayList<>();
+         QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
+            found.addAll(DiagnosticRecordReads.existingIds(tableName, unreadable)));
+         existing.addAll(found);
+      }
+
+      QuickSearchOpenSearchClient client   = runtime.getClient();
+      List<Outcome>               outcomes = new ArrayList<>();
+
+      List<OpenSearchDocument> docs = new ArrayList<>();
+      for(String id : ids)
+      {
+         QRecord record = readable.get(id);
+         if(record != null)
+         {
+            OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
+            if(doc != null)
+            {
+               docs.add(doc);
+            }
+         }
+      }
+
+      BulkIndexResult indexResult = client.indexDocuments(docs, Math.max(1, runtime.getConfig().getBulkBatchSize() == null ? 500 : runtime.getConfig().getBulkBatchSize()));
+      Set<String>     failedIds   = new HashSet<>();
+      for(String failure : indexResult.getErrors())
+      {
+         ////////////////////////////////////////////////////
+         // failures read "table:recordId: reason"          //
+         ////////////////////////////////////////////////////
+         int    reasonStart = failure.indexOf(": ");
+         String documentId  = reasonStart < 0 ? failure : failure.substring(0, reasonStart);
+         failedIds.add(documentId.substring(documentId.indexOf(':') + 1));
+      }
+
+      List<String> toDelete = new ArrayList<>();
+      for(String id : ids)
+      {
+         if(readable.containsKey(id))
+         {
+            outcomes.add(failedIds.contains(id) ? new Outcome(id, OUTCOME_FAILED, "OpenSearch rejected the document; see the run's errors")
+               : new Outcome(id, OUTCOME_INDEXED, "Rebuilt from the database and written to the index."));
+         }
+         else if(!validIds.contains(id))
+         {
+            outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "Not a valid id for this table."));
+         }
+         else if(existing.contains(id))
+         {
+            outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "The record exists but you cannot read it, so it was not reindexed."));
+         }
+         else
+         {
+            toDelete.add(OpenSearchDocument.buildDocumentId(tableName, id));
+            outcomes.add(new Outcome(id, OUTCOME_DELETED, "The record no longer exists; its document (if any) was removed."));
+         }
+      }
+
+      if(!toDelete.isEmpty())
+      {
+         BulkIndexResult deleteResult = client.deleteDocuments(toDelete, 500);
+         if(!deleteResult.getErrors().isEmpty())
+         {
+            throw (new QException("Removing documents of deleted records failed: " + String.join("; ", deleteResult.getErrors())));
+         }
+      }
+
+      if(!indexResult.getErrors().isEmpty())
+      {
+         outcomes.add(new Outcome("-", OUTCOME_FAILED, "Index errors: " + String.join("; ", indexResult.getErrors())));
+      }
+
+      client.refreshIndex();
+      return (outcomes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Render outcomes as HTML.
+    *******************************************************************************/
+   public static String render(String tableName, List<Outcome> outcomes)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Reindex in search: " + tableName);
+      long indexed = outcomes.stream().filter(o -> OUTCOME_INDEXED.equals(o.outcome())).count();
+      long deleted = outcomes.stream().filter(o -> OUTCOME_DELETED.equals(o.outcome())).count();
+      long skipped = outcomes.stream().filter(o -> OUTCOME_SKIPPED.equals(o.outcome())).count();
+      long failed  = outcomes.stream().filter(o -> OUTCOME_FAILED.equals(o.outcome())).count();
+      html.status(failed > 0 ? Level.FAIL : (skipped > 0 ? Level.WARN : Level.OK),
+         indexed + " reindexed, " + deleted + " removed (record gone), " + skipped + " skipped, " + failed + " failed. The documents are searchable now.");
+
+      List<String> items = new ArrayList<>();
+      for(Outcome outcome : outcomes)
+      {
+         Level level = switch(outcome.outcome())
+         {
+            case OUTCOME_INDEXED, OUTCOME_DELETED -> Level.OK;
+            case OUTCOME_SKIPPED -> Level.WARN;
+            default -> Level.FAIL;
+         };
+         items.add(DiagnosticsHtml.badge(level) + " #" + DiagnosticsHtml.escape(outcome.recordId()) + ": " + DiagnosticsHtml.escape(outcome.message()));
+      }
+      html.rawList(items);
+      return (html.close().toString());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/SearchPathDiagnosis.java

@@ -0,0 +1,178 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+
+
+/*******************************************************************************
+ ** Which path the global search (core RecordSearchAction, POST
+ ** /qqq/v1/search) takes for each table, and why: OpenSearch through the
+ ** Quick Search record-search provider, core's own database search, or not
+ ** searched at all. Mirrors RecordSearchAction's table selection and
+ ** QuickSearchRecordSearchProvider.claimsTable without calling them, so a
+ ** diagnosis changes no state (no backoff, no counters).
+ *******************************************************************************/
+public final class SearchPathDiagnosis
+{
+   public static final String PATH_OPENSEARCH = "OpenSearch (Quick Search provider)";
+   public static final String PATH_CORE       = "Core database search";
+   public static final String PATH_NONE       = "Not searched";
+
+
+
+   /*******************************************************************************
+    ** The record-search provider's state in this JVM.
+    *******************************************************************************/
+   public record ProviderState(String registeredProvider, Boolean quickSearchProvider, Boolean serveCoreRecordSearch, Long backoffRemainingMillis,
+                               Long failureCount, Instant lastFailureTime, String lastFailureMessage, Long backoffDeclinedClaims, Integer timeoutMillis, Instant runtimeStartedAt)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's path and the reason for it.
+    *******************************************************************************/
+   public record TablePath(String tableName, String tableLabel, Boolean indexed, Boolean enabled, Boolean readable, Boolean coreSearchable, String path, String reason)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private SearchPathDiagnosis()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The provider's state from the instance and the runtime.
+    *******************************************************************************/
+   public static ProviderState providerState(QInstance qInstance, QuickSearchRuntime runtime)
+   {
+      QCodeReference        provider = qInstance.getRecordSearchProvider();
+      QuickSearchQBitConfig config   = runtime.getConfig();
+      return (new ProviderState(
+         provider == null ? null : provider.getName(),
+         provider != null && QuickSearchRecordSearchProvider.class.getName().equals(provider.getName()),
+         Boolean.TRUE.equals(config.getServeCoreRecordSearch()),
+         runtime.getRecordSearchBackoffRemainingMillis(),
+         runtime.getRecordSearchFailureCount(),
+         runtime.getLastRecordSearchFailureTime(),
+         runtime.getLastRecordSearchFailureMessage(),
+         runtime.getBackoffDeclinedClaimCount(),
+         config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis(),
+         runtime.getStartedAt()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The path of each named table (unknown names are reported as not
+    ** searched), as seen by the session of permissionInput.
+    *******************************************************************************/
+   public static List<TablePath> diagnose(QInstance qInstance, QuickSearchRuntime runtime, List<String> tableNames, AbstractActionInput permissionInput)
+   {
+      ProviderState   state = providerState(qInstance, runtime);
+      List<TablePath> paths = new ArrayList<>();
+      for(String tableName : CollectionUtils.nonNullList(tableNames))
+      {
+         paths.add(diagnoseTable(qInstance, runtime, state, tableName, permissionInput));
+      }
+      return (paths);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static TablePath diagnoseTable(QInstance qInstance, QuickSearchRuntime runtime, ProviderState state, String tableName, AbstractActionInput permissionInput)
+   {
+      QTableMetaData table = qInstance.getTable(tableName);
+      if(table == null)
+      {
+         return (new TablePath(tableName, tableName, false, false, false, false, PATH_NONE, "There is no table with this name."));
+      }
+
+      boolean indexed        = runtime.getTableConfig(tableName) != null;
+      boolean enabled        = indexed && runtime.isTableEnabled(tableName);
+      boolean readable       = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
+      boolean coreSearchable = !CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) && table.isCapabilityEnabled(qInstance.getBackendForTable(tableName), Capability.TABLE_QUERY);
+      String  label          = table.getLabel() == null ? tableName : table.getLabel();
+
+      if(!coreSearchable)
+      {
+         return (new TablePath(tableName, label, indexed, enabled, readable, false, PATH_NONE,
+            "Global search skips this table: it has no core searchFields (or does not support query)."));
+      }
+      if(!readable)
+      {
+         return (new TablePath(tableName, label, indexed, enabled, false, true, PATH_NONE,
+            "Global search skips this table for you: you do not have read permission on it."));
+      }
+      if(!indexed)
+      {
+         return (new TablePath(tableName, label, false, false, true, true, PATH_CORE, "Quick Search does not index this table."));
+      }
+      if(!Boolean.TRUE.equals(state.serveCoreRecordSearch()))
+      {
+         return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, "serveCoreRecordSearch is off in the Quick Search configuration."));
+      }
+      if(!Boolean.TRUE.equals(state.quickSearchProvider()))
+      {
+         String reason = state.registeredProvider() == null
+            ? "No record search provider is registered on the instance."
+            : "Another record search provider is registered (" + state.registeredProvider() + "), so Quick Search does not serve global search.";
+         return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, reason));
+      }
+      if(!enabled)
+      {
+         return (new TablePath(tableName, label, true, false, true, true, PATH_CORE, "The table is disabled in Quick Search Index (enabled = false)."));
+      }
+      if(state.backoffRemainingMillis() != null && state.backoffRemainingMillis() > 0)
+      {
+         return (new TablePath(tableName, label, true, true, true, true, PATH_CORE,
+            "Quick Search is backing off for " + Math.max(1, state.backoffRemainingMillis() / 1000) + " more second(s) after an OpenSearch failure"
+               + (state.lastFailureTime() == null ? "" : " at " + state.lastFailureTime())
+               + (state.lastFailureMessage() == null ? "" : ": " + state.lastFailureMessage())));
+      }
+
+      return (new TablePath(tableName, label, true, true, true, true, PATH_OPENSEARCH,
+         "Claimed by the Quick Search provider. If OpenSearch fails or takes longer than " + state.timeoutMillis()
+            + " ms, core searches this table itself for that request and Quick Search backs off for "
+            + (QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS / 1000) + " seconds."));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/SearchPlayground.java

@@ -0,0 +1,654 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** The search playground: runs a term the way the global search does, as the
+ ** current user, and shows everything behind it.
+ ** 1. Which path serves each table and why (SearchPathDiagnosis).
+ ** 2. The provider's own OpenSearch request (_msearch of record ids), timed,
+ **    without starting a backoff when it fails.
+ ** 3. The real global search (core RecordSearchAction) as the current user:
+ **    what users see, per table, timed. Like any user search, a provider
+ **    failure here starts the provider's normal backoff.
+ ** 4. Per indexed table, the production query with scores, highlights and
+ **    matched clauses, which hits users can read (record locks), which ones
+ **    the global search shows, the query DSL and the raw response (hits the
+ **    user cannot read are redacted to their id).
+ ** 5. Optionally, explain for one hit, and the term's tokens.
+ ** Read-only: no index or record is changed.
+ *******************************************************************************/
+public class SearchPlayground
+{
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+
+
+   /*******************************************************************************
+    ** What to run: term, tables (empty for every searchable table), results per
+    ** table as in the global search, and an optional hit to explain
+    ** ("table:recordId").
+    *******************************************************************************/
+   public record Input(String term, List<String> tableNames, Integer limitPerTable, String explainDocumentId)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's probe: the search, which hit ids the user can read, which
+    ** the global search showed, and an error when the probe failed.
+    *******************************************************************************/
+   public record TableProbe(String tableName, SearchProbe probe, Set<String> readableIds, Set<String> shownIds, Integer hiddenCount, String error)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Everything the playground found.
+    *******************************************************************************/
+   public record Report(String term, List<String> notices, List<AnalyzedToken> termTokens, String termTokensError,
+                        SearchPathDiagnosis.ProviderState providerState, List<SearchPathDiagnosis.TablePath> paths,
+                        Map<String, List<String>> providerIds, Map<String, Integer> providerHiddenCounts, Long providerMillis, String providerError,
+                        Map<String, List<RecordSearchResult>> coreResults, Long coreMillis, String coreError,
+                        List<TableProbe> probes, String explainDocumentId, ExplainResult explain, String explainError,
+                        Integer limitPerTable, Integer idsPerTable)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the playground for the input as the current session.
+    *******************************************************************************/
+   public Report run(QuickSearchRuntime runtime, Input input, AbstractActionInput permissionInput)
+   {
+      QInstance             qInstance = QContext.getQInstance();
+      QuickSearchQBitConfig config    = runtime.getConfig();
+      List<String>          notices   = new ArrayList<>();
+
+      String term          = IndexingUtils.normalizeSearchText(input.term());
+      int    limitPerTable = input.limitPerTable() == null || input.limitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : Math.min(input.limitPerTable(), RecordSearchAction.MAX_LIMIT_PER_TABLE);
+      int    idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
+
+      List<String> tables = resolveTables(qInstance, runtime, input.tableNames(), notices);
+      List<SearchPathDiagnosis.TablePath> paths = SearchPathDiagnosis.diagnose(qInstance, runtime, tables, permissionInput);
+      SearchPathDiagnosis.ProviderState   state = SearchPathDiagnosis.providerState(qInstance, runtime);
+
+      if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         notices.add("The term is shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters: the Quick Search provider returns nothing for it, so only core's own search can match.");
+      }
+      if(term.length() > RecordSearchAction.MAX_SEARCH_TERM_LENGTH)
+      {
+         notices.add("The term is longer than " + RecordSearchAction.MAX_SEARCH_TERM_LENGTH + " characters: the global search rejects it.");
+      }
+
+      QuickSearchDiagnosticsOperations ops = null;
+      String                           opsError = null;
+      try
+      {
+         ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      }
+      catch(QException e)
+      {
+         opsError = e.getMessage();
+         notices.add("The OpenSearch client could not be built: " + e.getMessage());
+      }
+
+      ////////////////////////
+      // term tokens        //
+      ////////////////////////
+      List<AnalyzedToken> termTokens      = null;
+      String              termTokensError = opsError;
+      if(ops != null && StringUtils.hasContent(term))
+      {
+         try
+         {
+            termTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
+         }
+         catch(QException e)
+         {
+            termTokensError = e.getMessage();
+         }
+      }
+
+      ///////////////////////////////////////////////////////////
+      // the provider's own request, for the tables it claims  //
+      ///////////////////////////////////////////////////////////
+      List<String> claimed = paths.stream().filter(p -> SearchPathDiagnosis.PATH_OPENSEARCH.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
+      Map<String, List<String>> providerIds    = null;
+      Map<String, Integer>      providerHidden = new LinkedHashMap<>();
+      Long                      providerMillis = null;
+      String                    providerError  = null;
+      if(!claimed.isEmpty() && ops != null && term.length() >= QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         int  timeout = state.timeoutMillis();
+         long start   = System.currentTimeMillis();
+         try
+         {
+            providerIds = runtime.getClient().searchRecordIdsPerTable(term, claimed, idsPerTable, runtime.getDiscoveredTables(), timeout);
+         }
+         catch(QException | RuntimeException e)
+         {
+            providerError = e.getClass().getSimpleName() + ": " + e.getMessage();
+         }
+         providerMillis = System.currentTimeMillis() - start;
+         providerIds    = providerIds == null ? null : readableOnly(providerIds, providerHidden);
+      }
+
+      ///////////////////////////////////////////////////////
+      // the real global search, as the current user       //
+      ///////////////////////////////////////////////////////
+      Map<String, List<RecordSearchResult>> coreResults = new LinkedHashMap<>();
+      Long                                  coreMillis  = null;
+      String                                coreError   = null;
+      List<String> searched = paths.stream().filter(p -> !SearchPathDiagnosis.PATH_NONE.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
+      if(!searched.isEmpty() && StringUtils.hasContent(term))
+      {
+         long start = System.currentTimeMillis();
+         try
+         {
+            RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm(term).withTableNames(searched).withLimitPerTable(limitPerTable));
+            for(RecordSearchResult result : CollectionUtils.nonNullList(output.getResults()))
+            {
+               coreResults.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
+            }
+         }
+         catch(Exception e)
+         {
+            coreError = e.getClass().getSimpleName() + ": " + e.getMessage();
+         }
+         coreMillis = System.currentTimeMillis() - start;
+      }
+
+      //////////////////////////////////////////
+      // per indexed table, the probe search  //
+      //////////////////////////////////////////
+      List<TableProbe> probes = new ArrayList<>();
+      for(SearchPathDiagnosis.TablePath path : paths)
+      {
+         if(!Boolean.TRUE.equals(path.indexed()) || !Boolean.TRUE.equals(path.readable()) || term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+         {
+            continue;
+         }
+         if(ops == null)
+         {
+            probes.add(new TableProbe(path.tableName(), null, Set.of(), Set.of(), 0, opsError));
+            continue;
+         }
+         probes.add(probeTable(ops, runtime, term, path.tableName(), idsPerTable, coreResults.get(path.tableName())));
+      }
+
+      //////////////////////
+      // explain one hit  //
+      //////////////////////
+      ExplainResult explain      = null;
+      String        explainError = null;
+      String        explainId    = StringUtils.hasContent(input.explainDocumentId()) ? input.explainDocumentId().trim() : null;
+      if(explainId != null && ops != null)
+      {
+         try
+         {
+            explain = explainHit(ops, runtime, term, explainId);
+         }
+         catch(QException e)
+         {
+            explainError = e.getMessage();
+         }
+      }
+
+      return (new Report(term, notices, termTokens, termTokensError, state, paths, providerIds, providerHidden, providerMillis, providerError,
+         coreResults, coreMillis, coreError, probes, explainId, explain, explainError, limitPerTable, idsPerTable));
+   }
+
+
+
+   /*******************************************************************************
+    ** The requested tables that exist (else every table with core searchFields
+    ** or a Quick Search index), in instance order.
+    *******************************************************************************/
+   static List<String> resolveTables(QInstance qInstance, QuickSearchRuntime runtime, List<String> requested, List<String> notices)
+   {
+      List<String> tables = new ArrayList<>();
+      if(!CollectionUtils.nullSafeIsEmpty(requested))
+      {
+         for(String tableName : new LinkedHashSet<>(requested))
+         {
+            if(qInstance.getTable(tableName) == null)
+            {
+               notices.add("Ignored unknown table [" + tableName + "].");
+            }
+            else
+            {
+               tables.add(tableName);
+            }
+         }
+         return (tables);
+      }
+
+      for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values())
+      {
+         if(!CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) || runtime.getTableConfig(table.getName()) != null)
+         {
+            tables.add(table.getName());
+         }
+      }
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Probe one table and mark which hits the user can read and which ones the
+    ** global search showed.
+    *******************************************************************************/
+   private TableProbe probeTable(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String tableName, int size, List<RecordSearchResult> shown)
+   {
+      try
+      {
+         SearchProbe  probe = ops.probeSearch(term, tableName, size, runtime.getDiscoveredTables());
+         List<String> ids   = probe.hits().stream().map(ProbeHit::recordId).toList();
+
+         Set<String> readable = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, ids));
+         Set<String> shownIds = new HashSet<>();
+         for(RecordSearchResult result : CollectionUtils.nonNullList(shown))
+         {
+            shownIds.add(String.valueOf(result.getRecordId()));
+         }
+
+         List<ProbeHit> visible = probe.hits().stream().filter(h -> readable.contains(h.recordId())).toList();
+         int            hidden  = probe.hits().size() - visible.size();
+         SearchProbe redacted = new SearchProbe(probe.tableName(), probe.requestJson(), redactResponse(probe.responseJson(), readable),
+            probe.tookMillis(), probe.wallMillis(), hidden == 0 ? probe.totalHits() : null, probe.totalIsLowerBound(), visible);
+         return (new TableProbe(tableName, redacted, readable, shownIds, hidden, null));
+      }
+      catch(QException e)
+      {
+         return (new TableProbe(tableName, null, Set.of(), Set.of(), 0, e.getMessage()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The provider's ids per table, keeping only records the user can read;
+    ** how many were dropped goes in hiddenCounts. Ids of records the user
+    ** cannot read are never shown (they would reveal what a locked record
+    ** contains).
+    *******************************************************************************/
+   static Map<String, List<String>> readableOnly(Map<String, List<String>> idsByTable, Map<String, Integer> hiddenCounts)
+   {
+      Map<String, List<String>> readable = new LinkedHashMap<>();
+      for(Map.Entry<String, List<String>> entry : idsByTable.entrySet())
+      {
+         List<String> ids = entry.getValue() == null ? List.of() : entry.getValue();
+         Set<String>  ok;
+         try
+         {
+            ok = new HashSet<>(DiagnosticRecordReads.existingIds(entry.getKey(), ids));
+         }
+         catch(QException e)
+         {
+            ok = Set.of();
+         }
+         Set<String> finalOk = ok;
+         readable.put(entry.getKey(), ids.stream().filter(finalOk::contains).toList());
+         hiddenCounts.put(entry.getKey(), (int) ids.stream().filter(id -> !finalOk.contains(id)).count());
+      }
+      return (readable);
+   }
+
+
+
+   /*******************************************************************************
+    ** The raw response without the hits the user cannot read: they are
+    ** removed whole (id, score, source, highlights, matched queries), and
+    ** when any was removed the total and max score are redacted too, so the
+    ** response reveals nothing about records the user cannot read.
+    *******************************************************************************/
+   static String redactResponse(String responseJson, Set<String> readableIds)
+   {
+      try
+      {
+         JsonNode root    = MAPPER.readTree(responseJson);
+         JsonNode hitList = root.path("hits").path("hits");
+         int      removed = 0;
+         if(hitList.isArray())
+         {
+            ArrayNode array = (ArrayNode) hitList;
+            for(int i = array.size() - 1; i >= 0; i--)
+            {
+               String recordId = array.get(i).path("_source").path("recordId").asText(null);
+               if(recordId == null || !readableIds.contains(recordId))
+               {
+                  array.remove(i);
+                  removed++;
+               }
+            }
+         }
+         if(removed > 0 && root.path("hits") instanceof ObjectNode hits)
+         {
+            hits.put("total", "redacted: " + removed + " hit(s) you cannot read were removed");
+            hits.remove("max_score");
+         }
+         return (root.toString());
+      }
+      catch(Exception e)
+      {
+         return ("{\"redacted\":\"response could not be parsed for redaction\"}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Explain one hit ("table:recordId") when the table is indexed and the user
+    ** can read the record.
+    *******************************************************************************/
+   private ExplainResult explainHit(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String documentId) throws QException
+   {
+      int colon = documentId.indexOf(':');
+      if(colon <= 0 || colon == documentId.length() - 1)
+      {
+         throw (new QException("Give the hit as table:recordId (its document id), for example blogPost:42"));
+      }
+      String tableName = documentId.substring(0, colon);
+      String recordId  = documentId.substring(colon + 1);
+
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Table [" + tableName + "] is not indexed by Quick Search"));
+      }
+      if(DiagnosticRecordReads.existingIds(tableName, List.of(recordId)).isEmpty())
+      {
+         throw (new QException("Record " + recordId + " of [" + tableName + "] does not exist or you cannot read it"));
+      }
+      return (ops.explain(term, tableName, recordId, runtime.getDiscoveredTables()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render a report as HTML.
+    *******************************************************************************/
+   public static String render(Report report)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Search playground: \"" + report.term() + "\"");
+      html.paragraph("Run as you, at " + Instant.now() + ". The global search shows up to " + report.limitPerTable() + " results per table; the Quick Search provider asks OpenSearch for the top "
+         + report.idsPerTable() + " ids per table and core keeps the first " + report.limitPerTable() + " you can read.");
+
+      for(String notice : report.notices())
+      {
+         html.status(Level.WARN, notice);
+      }
+
+      renderProviderState(html, report.providerState());
+
+      html.subheading("Path per table");
+      List<String> pathItems = new ArrayList<>();
+      for(SearchPathDiagnosis.TablePath path : report.paths())
+      {
+         Level level = SearchPathDiagnosis.PATH_OPENSEARCH.equals(path.path()) ? Level.OK : (SearchPathDiagnosis.PATH_CORE.equals(path.path()) ? Level.INFO : Level.WARN);
+         pathItems.add(DiagnosticsHtml.badge(level) + " <strong>" + DiagnosticsHtml.escape(path.tableLabel()) + "</strong> (" + DiagnosticsHtml.escape(path.tableName()) + "): "
+            + DiagnosticsHtml.escape(path.path()) + ". " + DiagnosticsHtml.escape(path.reason()));
+      }
+      if(pathItems.isEmpty())
+      {
+         html.paragraph("No table is searchable by the global search.");
+      }
+      html.rawList(pathItems);
+
+      html.subheading("Term tokens (search analyzer)");
+      if(report.termTokensError() != null)
+      {
+         html.status(Level.FAIL, "Could not analyze the term: " + report.termTokensError());
+      }
+      else if(report.termTokens() != null)
+      {
+         html.paragraph(tokenSummary(report.termTokens()));
+         for(AnalyzedToken token : report.termTokens())
+         {
+            if(token.token().length() < 2)
+            {
+               html.status(Level.WARN, "\"" + token.token() + "\" is one character: the index holds edge n-grams of 2 to 20 characters, so it can never match, and every term must match.");
+            }
+            else if(token.token().length() > 20)
+            {
+               html.status(Level.WARN, "\"" + token.token() + "\" is longer than 20 characters: the index holds edge n-grams of at most 20 characters, so it can never match.");
+            }
+         }
+      }
+
+      html.subheading("Quick Search provider request");
+      if(report.providerMillis() == null)
+      {
+         html.paragraph("Not run: the provider claims none of these tables (see the paths above), or the term is too short.");
+      }
+      else if(report.providerError() != null)
+      {
+         html.status(Level.FAIL, "Failed after " + report.providerMillis() + " ms: " + report.providerError() + ". A user search hitting this falls back to core's database search for these tables.");
+      }
+      else
+      {
+         html.status(Level.OK, "Answered in " + report.providerMillis() + " ms (wall clock, limit " + report.providerState().timeoutMillis() + " ms).");
+         List<String> items = new ArrayList<>();
+         for(Map.Entry<String, List<String>> entry : report.providerIds().entrySet())
+         {
+            Integer hidden = report.providerHiddenCounts().getOrDefault(entry.getKey(), 0);
+            items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + entry.getValue().size() + " id(s) you can read"
+               + (hidden > 0 ? " (" + hidden + " more you cannot read, not shown)" : "")
+               + (entry.getValue().isEmpty() ? "" : ": " + DiagnosticsHtml.escape(String.join(", ", entry.getValue().subList(0, Math.min(20, entry.getValue().size()))))
+               + (entry.getValue().size() > 20 ? ", ..." : "")));
+         }
+         html.rawList(items);
+      }
+
+      html.subheading("Global search results (what you see)");
+      if(report.coreMillis() == null)
+      {
+         html.paragraph("Not run: no table is searchable, or the term is empty.");
+      }
+      else if(report.coreError() != null)
+      {
+         html.status(Level.FAIL, "The global search failed after " + report.coreMillis() + " ms: " + report.coreError());
+      }
+      else
+      {
+         int total = report.coreResults().values().stream().mapToInt(List::size).sum();
+         html.status(total == 0 ? Level.INFO : Level.OK, total + " result(s) in " + report.coreMillis() + " ms.");
+         List<String> items = new ArrayList<>();
+         for(Map.Entry<String, List<RecordSearchResult>> entry : report.coreResults().entrySet())
+         {
+            List<String> labels = new ArrayList<>();
+            for(RecordSearchResult result : entry.getValue())
+            {
+               labels.add(DiagnosticsHtml.escape(result.getRecordLabel() == null ? String.valueOf(result.getRecordId()) : result.getRecordLabel()) + " (#" + DiagnosticsHtml.escape(result.getRecordId()) + ")");
+            }
+            items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + String.join("; ", labels));
+         }
+         html.rawList(items);
+      }
+
+      html.subheading("OpenSearch detail per indexed table");
+      if(report.probes().isEmpty())
+      {
+         html.paragraph("No indexed table you can read was searched, or the term is too short.");
+      }
+      for(TableProbe tableProbe : report.probes())
+      {
+         renderProbe(html, tableProbe);
+      }
+
+      if(report.explainDocumentId() != null)
+      {
+         html.subheading("Explain " + report.explainDocumentId());
+         if(report.explainError() != null)
+         {
+            html.status(Level.FAIL, report.explainError());
+         }
+         else if(report.explain() != null && !Boolean.TRUE.equals(report.explain().documentFound()))
+         {
+            html.status(Level.FAIL, "There is no document " + report.explainDocumentId() + " in the index.");
+         }
+         else if(report.explain() != null)
+         {
+            html.status(Boolean.TRUE.equals(report.explain().matched()) ? Level.OK : Level.WARN,
+               Boolean.TRUE.equals(report.explain().matched()) ? "Matches with score " + report.explain().score() + "." : "Does not match this term.");
+            html.textDetails("Score explanation", report.explain().explanationText());
+            html.jsonDetails("Explain response (JSON)", report.explain().json());
+         }
+      }
+
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderProviderState(DiagnosticsHtml html, SearchPathDiagnosis.ProviderState state)
+   {
+      html.subheading("Record search provider (this server)");
+      List<String[]> facts = new ArrayList<>();
+      facts.add(new String[] { "Registered provider", state.registeredProvider() == null ? null : state.registeredProvider() + (Boolean.TRUE.equals(state.quickSearchProvider()) ? " (Quick Search)" : " (not Quick Search)") });
+      facts.add(new String[] { "serveCoreRecordSearch", String.valueOf(state.serveCoreRecordSearch()) });
+      facts.add(new String[] { "Timeout", state.timeoutMillis() + " ms" });
+      facts.add(new String[] { "Backing off", state.backoffRemainingMillis() > 0 ? "yes, " + (state.backoffRemainingMillis() / 1000) + " s left" : "no" });
+      facts.add(new String[] { "Failures since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) });
+      facts.add(new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) });
+      facts.add(new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() });
+      html.facts(facts);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderProbe(DiagnosticsHtml html, TableProbe tableProbe)
+   {
+      html.raw("<h4>" + DiagnosticsHtml.escape(tableProbe.tableName()) + "</h4>");
+      if(tableProbe.error() != null)
+      {
+         html.status(Level.FAIL, "OpenSearch search failed: " + tableProbe.error());
+         return;
+      }
+
+      SearchProbe probe = tableProbe.probe();
+      html.paragraph((probe.totalHits() == null ? "Matching documents: not shown (some are records you cannot read)"
+         : probe.totalHits() + (Boolean.TRUE.equals(probe.totalIsLowerBound()) ? "+" : "") + " matching document(s)")
+         + "; OpenSearch took " + probe.tookMillis() + " ms (" + probe.wallMillis() + " ms wall clock). Showing the " + probe.hits().size() + " top hit(s) you can read.");
+      if(tableProbe.hiddenCount() != null && tableProbe.hiddenCount() > 0)
+      {
+         html.status(Level.INFO, tableProbe.hiddenCount() + " of the top hits are records you cannot read (deleted, or hidden by a record security lock); they are not shown.");
+      }
+
+      List<String> items    = new ArrayList<>();
+      int          position = 0;
+      for(ProbeHit hit : probe.hits())
+      {
+         position++;
+         StringBuilder item = new StringBuilder();
+         item.append("#").append(position).append(" <strong>")
+            .append(DiagnosticsHtml.escape(hit.recordLabel() == null ? "(no label)" : hit.recordLabel()))
+            .append("</strong> &middot; document <code>").append(DiagnosticsHtml.escape(OpenSearchDocument.buildDocumentId(tableProbe.tableName(), hit.recordId()))).append("</code>")
+            .append(" &middot; score ").append(hit.score() == null ? "n/a" : String.format("%.3f", hit.score()))
+            .append(" &middot; ").append(tableProbe.shownIds().contains(hit.recordId()) ? "shown in global search" : "not shown (beyond the per-table limit)");
+         if(!hit.matchedClauses().isEmpty())
+         {
+            List<String> fields = hit.matchedClauses().stream().map(c -> c.startsWith(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX) ? c.substring(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX.length()) : c).toList();
+            item.append("<br>Matched: ").append(DiagnosticsHtml.escape(String.join(", ", fields)));
+         }
+         for(Map.Entry<String, List<String>> highlight : hit.highlights().entrySet())
+         {
+            item.append("<br><em>").append(DiagnosticsHtml.escape(highlight.getKey())).append(":</em> ")
+               .append(DiagnosticsHtml.highlight(String.join(" ... ", highlight.getValue())));
+         }
+         items.add(item.toString());
+      }
+      if(items.isEmpty())
+      {
+         html.paragraph("No document of this table that you can read matches every term.");
+      }
+      html.rawList(items);
+      html.jsonDetails("Query DSL", probe.requestJson());
+      html.jsonDetails("Raw response (hits you cannot read removed)", probe.responseJson());
+   }
+
+
+
+   /*******************************************************************************
+    ** "a, b, c" of token strings.
+    *******************************************************************************/
+   static String tokenSummary(List<AnalyzedToken> tokens)
+   {
+      if(tokens.isEmpty())
+      {
+         return ("No tokens: the term has no searchable characters.");
+      }
+      List<String> parts = new ArrayList<>();
+      for(AnalyzedToken token : tokens)
+      {
+         parts.add("\"" + token.token() + "\"");
+      }
+      return (tokens.size() + " token(s), all of which must match: " + String.join(", ", parts));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java

@@ -0,0 +1,125 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+
+
+/*******************************************************************************
+ ** Instance-owned five-second cache with one in-flight refresh. A stuck backend
+ ** cannot grow a queue or create more workers. Runtime shutdown stops the worker.
+ *******************************************************************************/
+public class StatusCache implements AutoCloseable
+{
+   private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable ->
+   {
+      Thread thread = new Thread(runnable, "quick-search-status");
+      thread.setDaemon(true);
+      return (thread);
+   });
+   private Future<QuickSearchStatus> refresh;
+   private QuickSearchStatus cached;
+   private long expiresAt;
+   private long refreshStartedAt;
+
+
+   /*******************************************************************************
+    ** Authorization belongs to the public service, before reaching this cache.
+    ** Only refreshes consume the existing per-user diagnostic rate limit.
+    *******************************************************************************/
+   public synchronized QuickSearchStatus get(QuickSearchRuntime runtime) throws QException
+   {
+      if(cached != null && System.nanoTime() < expiresAt)
+      {
+         return (cached);
+      }
+      String homeHref = "/app/" + runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
+      // A refresh that finished while nobody was polling must not resurrect an old snapshot.
+      if(refresh != null && refresh.isDone() && System.nanoTime() - refreshStartedAt >= TimeUnit.SECONDS.toNanos(5))
+      {
+         refresh = null;
+      }
+      if(refresh == null)
+      {
+         DiagnosticsSupport.checkRateLimit(runtime, "STATUS");
+         QInstance instance = QContext.getQInstance();
+         refreshStartedAt = System.nanoTime();
+         refresh = executor.submit(() ->
+         {
+            QContext.init(instance, new QSystemUserSession(instance));
+            try
+            {
+               return (StatusDiagnostics.read(runtime));
+            }
+            finally
+            {
+               QContext.clear();
+            }
+         });
+      }
+      try
+      {
+         cached = refresh.get(2, TimeUnit.SECONDS);
+         refresh = null;
+      }
+      catch(TimeoutException e)
+      {
+         // Leave the one worker in flight; never pile up abandoned requests.
+         cached = StatusDiagnostics.unavailable(QuickSearchStatus.State.DEGRADED, "Status refresh timed out; measurements are unknown.", homeHref);
+      }
+      catch(InterruptedException e)
+      {
+         Thread.currentThread().interrupt();
+         throw (new QException("Status refresh interrupted", e));
+      }
+      catch(ExecutionException e)
+      {
+         refresh = null;
+         cached = StatusDiagnostics.unavailable(QuickSearchStatus.State.DEGRADED, "Status diagnostics could not be read.", homeHref);
+      }
+      expiresAt = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+      return (cached);
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop the bounded worker when its runtime closes.
+    *******************************************************************************/
+   @Override
+   public void close()
+   {
+      executor.shutdownNow();
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusDiagnostics.java

@@ -0,0 +1,213 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.State;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Cheap projection of existing diagnostics: one health request, bounded metadata
+ ** reads, saved IndexValidator counts. No source scans or per-table remote calls.
+ *******************************************************************************/
+public final class StatusDiagnostics
+{
+   public static final int MAX_TABLES = 100;
+   public static final int MAX_BACKLOG = 1000;
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private StatusDiagnostics()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Read as the trusted diagnostics worker after the caller's admin check.
+    *******************************************************************************/
+   public static QuickSearchStatus read(QuickSearchRuntime runtime) throws Exception
+   {
+      String homeHref = "/app/" + runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
+      List<QRecord> rows = query(runtime.getConfig().getQuickSearchIndexTableName(), new QQueryFilter()
+         .withOrderBy(new QFilterOrderBy("id")).withLimit(MAX_TABLES + 1));
+      boolean truncated = rows.size() > MAX_TABLES;
+      QRecord validation = DiagnosticsSupport.latestValidation(runtime.getConfig());
+      String json = validation == null ? null : validation.getValueString("reportJson");
+      JsonNode report = json == null || json.length() > 1_000_000 ? MAPPER.createObjectNode() : MAPPER.readTree(json);
+      Instant countsAt = validation == null ? null : validation.getValueInstant("endTime");
+      List<IndexRow> indexes = new ArrayList<>();
+      Instant lastSync = null;
+      String lastError = runtime.getLastRecordSearchFailureMessage();
+      Instant lastErrorAt = runtime.getLastRecordSearchFailureTime();
+      boolean degraded = truncated || runtime.isRecordSearchBackingOff() || (rows.isEmpty() && !runtime.getCodeTables().isEmpty());
+      for(QRecord row : rows.subList(0, Math.min(rows.size(), MAX_TABLES)))
+      {
+         String tableName = row.getValueString("tableName");
+         Long documents = row.getValueLong("documentCount");
+         Long source = null;
+         Instant measuredAt = null;
+         for(JsonNode stats : report.path("tableStats"))
+         {
+            if(tableName.equals(stats.path("tableName").asText()))
+            {
+               documents = stats.path("indexCount").isNumber() ? stats.path("indexCount").longValue() : null;
+               source = stats.path("databaseCount").isNumber() ? stats.path("databaseCount").longValue() : null;
+               measuredAt = countsAt;
+               break;
+            }
+         }
+         Instant sync = latest(row.getValueInstant("lastBasepullTime"), latest(row.getValueInstant("lastFullReindexTime"), row.getValueInstant("lastReconcileTime")));
+         lastSync = latest(lastSync, sync);
+         String error = row.getValueString("lastErrorMessage");
+         if(error != null && !error.isBlank() && (lastErrorAt == null || (row.getValueInstant("modifyDate") != null && row.getValueInstant("modifyDate").isAfter(lastErrorAt))))
+         {
+            lastError = error;
+            lastErrorAt = row.getValueInstant("modifyDate");
+         }
+         boolean enabled = !Boolean.FALSE.equals(row.getValueBoolean("enabled"));
+         Integer realTimeErrors = row.getValueInteger("realTimeErrorCount");
+         degraded |= enabled && (!"ACTIVE".equals(row.getValueString("status")) || "FAILED".equals(row.getValueString("lastRunStatus"))
+            || (realTimeErrors != null && realTimeErrors > 0) || (source != null && documents != null && !source.equals(documents)));
+         indexes.add(new IndexRow(tableName, enabled, row.getValueString("status"), documents, source, measuredAt, sync, error, realTimeErrors));
+      }
+      List<QRecord> backlog = query(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME),
+         new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("PENDING", "EXHAUSTED", "AWAITING_REINDEX")))
+            .withOrderBy(new QFilterOrderBy("id")).withLimit(MAX_BACKLOG + 1));
+      int pending = 0;
+      int exhausted = 0;
+      for(QRecord row : backlog.subList(0, Math.min(backlog.size(), MAX_BACKLOG)))
+      {
+         if("EXHAUSTED".equals(row.getValueString("status")))
+         {
+            exhausted++;
+         }
+         else
+         {
+            pending++;
+         }
+      }
+      boolean paused = runtime.isIndexingPaused();
+      degraded |= paused || pending > 0 || exhausted > 0;
+      Boolean reachable = null;
+      String health = null;
+      Integer nodes = null;
+      Integer active = null;
+      Integer unassigned = null;
+      State state = degraded ? State.DEGRADED : State.OK;
+      if(!truncated && !indexes.isEmpty() && indexes.stream().noneMatch(row -> Boolean.TRUE.equals(row.enabled())))
+      {
+         state = State.DISABLED;
+      }
+      else
+      {
+         try
+         {
+            var response = new QuickSearchDiagnosticsOperations(runtime.getClient()).clusterHealth();
+            reachable = true;
+            QuickSearchDiagnosticsOperations.requireOk(response, "Cluster health unavailable");
+            JsonNode body = response.body();
+            health = body.path("status").asText("unknown");
+            nodes = number(body, "number_of_nodes");
+            active = number(body, "active_shards");
+            unassigned = number(body, "unassigned_shards");
+            if(IndexValidator.healthLevel(health) != DiagnosticsHtml.Level.OK || body.path("timed_out").asBoolean())
+            {
+               state = "red".equals(health) ? State.DOWN : State.DEGRADED;
+            }
+         }
+         catch(QException e)
+         {
+            if(reachable == null)
+            {
+               reachable = false;
+            }
+            state = State.DOWN;
+            lastError = "OpenSearch health is unavailable; see Quick Search diagnostics.";
+         }
+      }
+      return (new QuickSearchStatus(state, reachable, health, nodes, active, unassigned, paused, indexes, pending, exhausted, backlog.size() > MAX_BACKLOG,
+         truncated, lastSync, lastError, Instant.now(), homeHref));
+   }
+
+
+
+   /*******************************************************************************
+    ** Unknown measurements for disabled or failed snapshots.
+    *******************************************************************************/
+   public static QuickSearchStatus unavailable(State state, String error, String homeHref)
+   {
+      return (new QuickSearchStatus(state, null, null, null, null, null, null, List.of(), null, null, false, false, null, error, Instant.now(), homeHref));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bounded metadata queries; failures propagate rather than looking healthy.
+    *******************************************************************************/
+   private static List<QRecord> query(String tableName, QQueryFilter filter) throws QException
+   {
+      return (CollectionUtils.nonNullList(new QueryAction().execute(new QueryInput(tableName).withFilter(filter)).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Preserve unknown counters from incomplete cluster responses.
+    *******************************************************************************/
+   private static Integer number(JsonNode body, String field)
+   {
+      return (body.path(field).isNumber() ? body.path(field).intValue() : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Newest non-null timestamp.
+    *******************************************************************************/
+   private static Instant latest(Instant first, Instant second)
+   {
+      return (first == null ? second : second == null || first.isAfter(second) ? first : second);
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java

@@ -19,7 +19,9 @@ package com.kingsrook.qbits.quicksearch.listeners;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
@@ -39,6 +41,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
 import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
@@ -51,22 +55,28 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** Real-time indexing hook. Registered once per QInstance; applies to the
  ** tables this QBit indexes.
  **
- ** Events are built while the write's transaction is open (so updates can be
- ** re-read in full), but published only after that transaction commits, via
+ ** Events are built while the write's transaction is open, but published only
+ ** after that transaction commits, via
  ** QBackendTransaction.addAfterCommitCallback. A write without a transaction
  ** is already committed, so it publishes immediately. A rolled-back write never
  ** reaches the index.
  **
- ** This listener never fails the host's write. When the re-read of updated
- ** records or publishing fails (including a LinkageError from a missing
+ ** Inserted and updated records are re-read inside that transaction, with
+ ** display values and possible-value labels, so a real-time document matches
+ ** the one basepull, reconcile and full reindex build. The event's own records
+ ** are not enough: for an update they are the sparse input, and for an insert
+ ** they carry only raw values.
+ **
+ ** This listener never fails the host's write. When the re-read of inserted or
+ ** updated records or publishing fails (including a LinkageError from a missing
  ** optional library while the client is built), each affected (table, record,
- ** action) is written to quickSearchFailedEvent for the scheduled basepull to
+ ** action) is written to quick_search_failed_event for the scheduled basepull to
  ** replay. A failed re-read is recorded after the commit too, so a rolled-back
  ** write leaves no row.
  **
  ** While a full reindex rebuilds into a new physical index, the alias still
  ** points at the old one, so a published delete never reaches the new index.
- ** Such deletes are also written to quickSearchFailedEvent with status
+ ** Such deletes are also written to quick_search_failed_event with status
  ** AWAITING_REINDEX; the reindex applies them after the alias swap.
  *******************************************************************************/
 public class QuickSearchRecordChangeListener implements RecordChangeListenerInterface
@@ -84,6 +94,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
       QuickSearchRuntime runtime = QuickSearchRuntime.get();
       return (runtime != null
          && Boolean.TRUE.equals(runtime.getConfig().getEnableRealTimeIndexing())
+         && !QuickSearchFieldRules.isQuickSearchTable(runtime.getConfig(), tableName)
          && runtime.getTableConfig(tableName) != null);
    }
 
@@ -110,13 +121,25 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
       String primaryKeyField = tableConfig.getPrimaryKeyField() != null ? tableConfig.getPrimaryKeyField() : "id";
 
+      ///////////////////////////////////////////////////////////////////////////
+      // paused: index nothing, but keep the written keys (DEFERRED), which     //
+      // resuming hands to the basepull; search keeps serving the old documents //
+      ///////////////////////////////////////////////////////////////////////////
+      if(runtime.isIndexingPaused())
+      {
+         List<IndexEvent> deferred = event.getType() == RecordChangeType.DELETE
+            ? toDeleteEvents(tableName, primaryKeyField, event.getRecords())
+            : toIndexEvents(tableName, primaryKeyField, event.getRecords());
+         runAfterCommit(event, () -> recordDeferred(runtime, deferred));
+         return;
+      }
+
       List<IndexEvent> indexEvents  = new ArrayList<>();
       List<IndexEvent> deleteEvents = new ArrayList<>();
 
       switch(event.getType())
       {
-         case INSERT -> indexEvents.addAll(toIndexEvents(tableName, primaryKeyField, event.getRecords()));
-         case UPDATE ->
+         case INSERT, UPDATE ->
          {
             try
             {
@@ -127,10 +150,11 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
                ///////////////////////////////////////////////////////////////////////
                // log now: with a transaction, nothing else is logged until commit, //
                // and a rollback (a transaction the failed re-read aborted, too)    //
-               // would leave no trace. Then record the updated keys after commit,  //
+               // would leave no trace. Then record the written keys after commit,  //
                // so basepull re-reads and indexes them.                            //
                ///////////////////////////////////////////////////////////////////////
-               LOG.warn("Could not re-read updated records for indexing; recording them for replay once the write commits", e,
+               LOG.warn("Could not re-read " + (event.getType() == RecordChangeType.INSERT ? "inserted" : "updated")
+                     + " records for indexing; recording them for replay once the write commits", e,
                   logPair("tableName", tableName), logPair("recordCount", event.getRecords().size()));
                List<IndexEvent> failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords());
                runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e));
@@ -240,7 +264,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
 
    /*******************************************************************************
-    ** Write one quickSearchFailedEvent row per event. Never throws.
+    ** Write one quick_search_failed_event row per event. Never throws.
     *******************************************************************************/
    static void recordFailures(QuickSearchRuntime runtime, List<IndexEvent> events, Throwable cause)
    {
@@ -255,13 +279,14 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
       try
       {
-         if(!hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)))
+         if(!hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)))
          {
-            LOG.error("quickSearchFailedEvent table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
+            LOG.error("quick_search_failed_event table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
             return;
          }
 
          insertFailedEventRows(runtime, events, message, QuickSearchFailedEvent.STATUS_PENDING);
+         incrementRealTimeErrorCounts(runtime, events);
       }
       catch(Exception e)
       {
@@ -271,6 +296,79 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
 
 
+   /*******************************************************************************
+    ** QQQ UpdateInput has no arithmetic update. Serialize this read-modify-write
+    ** across listeners in this JVM, after the source transaction has committed.
+    ** Counter failures must not prevent replay or fail the source write.
+    *******************************************************************************/
+   private static synchronized void incrementRealTimeErrorCounts(QuickSearchRuntime runtime, List<IndexEvent> events)
+   {
+      Map<String, Integer> counts = new HashMap<>();
+      for(IndexEvent event : events)
+      {
+         counts.merge(event.getTableName(), 1, Integer::sum);
+      }
+
+      for(Map.Entry<String, Integer> entry : counts.entrySet())
+      {
+         try
+         {
+            String     indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(indexTable);
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, entry.getKey())).withLimit(1));
+            List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+            if(CollectionUtils.nullSafeIsEmpty(rows))
+            {
+               continue;
+            }
+
+            QRecord row      = rows.get(0);
+            Integer previous = row.getValueInteger("realTimeErrorCount");
+            UpdateInput updateInput = new UpdateInput();
+            updateInput.setTableName(indexTable);
+            updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id"))
+               .withValue("realTimeErrorCount", Math.addExact(previous == null ? 0 : previous, entry.getValue()))));
+            List<QRecord> updated = QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)).getRecords();
+            if(CollectionUtils.nullSafeIsEmpty(updated) || CollectionUtils.nullSafeHasContents(updated.get(0).getErrors()))
+            {
+               throw (new QException("Counter update returned errors"));
+            }
+         }
+         catch(Exception | LinkageError e)
+         {
+            LOG.warn("Could not increment real-time indexing error count", e, logPair("tableName", entry.getKey()));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Keep changes made while indexing is paused, for the basepull to replay
+    ** when it resumes. Never throws.
+    *******************************************************************************/
+   static void recordDeferred(QuickSearchRuntime runtime, List<IndexEvent> events)
+   {
+      if(events.isEmpty())
+      {
+         return;
+      }
+      try
+      {
+         if(hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)))
+         {
+            insertFailedEventRows(runtime, events, "deferred: indexing was paused", QuickSearchFailedEvent.STATUS_DEFERRED);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not keep changes made while indexing is paused; the next reconcile covers them", e, logPair("eventCount", events.size()));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Whether a full reindex is rebuilding the deletes' table. The status is
     ** read from the database (not cached) so a reindex on another node is seen.
@@ -286,7 +384,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
       String tableName = deleteEvents.get(0).getTableName();
       try
       {
-         return (hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
+         return (hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
       }
       catch(Exception e)
       {
@@ -341,7 +439,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
       }
 
       UpdateInput updateInput = new UpdateInput();
-      updateInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
+      updateInput.setTableName(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
       updateInput.setRecords(updates);
       new UpdateAction().execute(updateInput);
    }
@@ -349,7 +447,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
 
    /*******************************************************************************
-    ** Whether the table's quickSearchIndex row is REBUILDING.
+    ** Whether the table's quick_search_index row is REBUILDING.
     *******************************************************************************/
    private static boolean isRebuilding(QuickSearchRuntime runtime, String tableName) throws QException
    {
@@ -369,7 +467,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
 
    /*******************************************************************************
-    ** Insert one quickSearchFailedEvent row per event; returns the inserted rows.
+    ** Insert one quick_search_failed_event row per event; returns the inserted rows.
     *******************************************************************************/
    static List<QRecord> insertFailedEventRows(QuickSearchRuntime runtime, List<IndexEvent> events, String message, String status) throws QException
    {
@@ -386,7 +484,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
       }
 
       InsertInput insertInput = new InsertInput();
-      insertInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
+      insertInput.setTableName(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
       insertInput.setRecords(rows);
       return (CollectionUtils.nonNullList(new InsertAction().execute(insertInput).getRecords()));
    }
@@ -444,8 +542,9 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
 
    /*******************************************************************************
-    ** Updated records are "as the backend returned them", which for RDBMS is the
-    ** sparse input. Re-read the full rows inside the same transaction, with
+    ** The event's records are "as the backend returned them": for an update on
+    ** RDBMS that is the sparse input, and for an insert the raw values with no
+    ** display values. Re-read the full rows inside the same transaction, with
     ** display values so possible-value labels are indexed (which can run host
     ** possible-value providers).
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchAdminAppMetaDataProducer.java

@@ -17,24 +17,32 @@
 package com.kingsrook.qbits.quicksearch.metadata;
 
 
+import java.util.ArrayList;
+import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
 
 
 /*******************************************************************************
- ** Produces the quickSearchAdmin app from the tables and processes the other
- ** component producers added to the production output. Runs last.
+ ** Produces the quickSearchAdmin app from the tables the other component
+ ** producers added to the production output. Processes are not app children
+ ** (never in the nav): they run from their tables' Actions. Runs last.
  *******************************************************************************/
 public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataProducer<QAppMetaData, QuickSearchQBitConfig>
 {
@@ -62,24 +70,26 @@ public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataPr
          return (app);
       }
 
-      for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME, QuickSearchFailedEvent.TABLE_NAME })
+      QWidgetMetaData diagnosticsWidget = produced.get(QWidgetMetaData.class, config.applyPrefix(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME));
+      if(diagnosticsWidget != null)
       {
-         QTableMetaData table = produced.get(QTableMetaData.class, config.applyPrefix(tableName));
+         app.withWidgets(new ArrayList<>(List.of(diagnosticsWidget.getName())));
+      }
+
+      for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexField.TABLE_NAME, QuickSearchSettings.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME,
+         QuickSearchFailedEvent.TABLE_NAME, QuickSearchValidationRun.TABLE_NAME, QuickSearchDiagnosticEvent.TABLE_NAME, QuickSearchConfigTombstone.TABLE_NAME })
+      {
+         QTableMetaData table = produced.get(QTableMetaData.class, config.applyTablePrefix(tableName));
          if(table != null)
          {
             app.withChild(table);
          }
       }
 
-      for(String processName : new String[] { QuickSearchProcessMetaDataHelper.BASEPULL_PROCESS_NAME, QuickSearchProcessMetaDataHelper.FULL_REINDEX_PROCESS_NAME, QuickSearchProcessMetaDataHelper.RECONCILE_PROCESS_NAME })
-      {
-         QProcessMetaData process = produced.get(QProcessMetaData.class, config.applyPrefix(processName));
-         if(process != null)
-         {
-            app.withChild(process);
-         }
-      }
-
+      ////////////////////////////////////////////////////////////////////////
+      // processes are never in the nav: Basepull, Full Reindex, Reconcile   //
+      // and the config processes run from their tables' Actions             //
+      ////////////////////////////////////////////////////////////////////////
       return (app);
    }
 

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigMetaDataHelper.java

@@ -0,0 +1,384 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchIndexFieldTableCustomizer;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchSettingsTableCustomizer;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchTablePossibleValueProviders;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDocumentsChoice;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+
+
+/*******************************************************************************
+ ** Builds the data-driven config metadata (1.1.0-RC.3): the field, settings
+ ** and tombstone tables, the index-to-fields join and child-record widget,
+ ** and the possible-value sources the config screens use.
+ *******************************************************************************/
+public class QuickSearchConfigMetaDataHelper
+{
+   public static final String ELIGIBLE_TABLE_PVS   = "quick_search_eligible_table";
+   public static final String INDEXABLE_FIELD_PVS  = "quick_search_indexable_field";
+   public static final String TIMESTAMP_FIELD_PVS  = "quick_search_timestamp_field";
+   public static final String FIELD_ROLE_PVS       = "quick_search_field_role";
+   public static final String FIELD_ANALYZER_PVS   = "quick_search_field_analyzer";
+   public static final String DOCUMENTS_CHOICE_PVS = "quick_search_documents_choice";
+   public static final String CONFIG_SOURCE_PVS    = "quick_search_config_source";
+   public static final String INDEX_FIELDS_JOIN    = "quick_search_index_join_quick_search_index_field";
+   public static final String INDEX_FIELDS_WIDGET  = "quickSearchIndexFields";
+   public static final String FIELDS_ASSOCIATION   = "fields";
+   public static final String INDEX_RUNS_JOIN      = "quick_search_index_join_quick_search_index_run";
+   public static final String INDEX_RUNS_WIDGET    = "quickSearchIndexRuns";
+   public static final String RUN_TYPE_PVS         = "quick_search_run_type";
+   public static final String RUN_STATUS_PVS       = "quick_search_run_status";
+   public static final String EVENT_STATUS_PVS     = "quick_search_failed_event_status";
+   public static final String EVENT_ACTION_PVS     = "quick_search_failed_event_action";
+
+
+
+   /*******************************************************************************
+    ** quick_search_index_field: the searchable fields of each index row.
+    *******************************************************************************/
+   public static QTableMetaData buildIndexFieldTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
+         .withLabel("Quick Search Index Field")
+         .withIcon(new QIcon("text_fields"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("fieldName")
+         .withUniqueKey(new UniqueKey("quickSearchIndexId", "fieldName"))
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withIsRequired(true)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)))
+         .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withLabel("Field").withIsRequired(true)
+            .withPossibleValueSourceName(config.applyTablePrefix(INDEXABLE_FIELD_PVS)))
+         .withField(new QFieldMetaData("weight", QFieldType.INTEGER).withDefaultValue(1))
+         .withField(new QFieldMetaData("role", QFieldType.STRING).withDefaultValue(QuickSearchFieldRole.BODY.name())
+            .withPossibleValueSourceName(config.applyTablePrefix(FIELD_ROLE_PVS)))
+         .withField(new QFieldMetaData("analyzer", QFieldType.STRING).withLabel("Analyzer (blank: by role)")
+            .withPossibleValueSourceName(config.applyTablePrefix(FIELD_ANALYZER_PVS)))
+         .withField(new QFieldMetaData("includeInSnippet", QFieldType.BOOLEAN).withLabel("In Snippet (blank: by role)"))
+         .withField(new QFieldMetaData("includeLabel", QFieldType.BOOLEAN).withLabel("Index With Field Label").withDefaultValue(false))
+         .withField(new QFieldMetaData("sortOrder", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false).withPossibleValueSourceName(config.applyTablePrefix(CONFIG_SOURCE_PVS)))
+         .withField(new QFieldMetaData("ownerEdited", QFieldType.BOOLEAN).withLabel("Edited by Owner").withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Field", new QIcon("text_fields"), Tier.T1, List.of("id", "quickSearchIndexId", "fieldName", "weight", "role")))
+         .withSection(new QFieldSection("matching", "Matching", new QIcon("tune"), Tier.T2, List.of("analyzer", "includeInSnippet", "includeLabel", "sortOrder")))
+         .withSection(new QFieldSection("origin", "Origin", new QIcon("history"), Tier.T3, List.of("source", "ownerEdited", "createDate", "modifyDate"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_settings: the one row of system settings (the global pause
+    ** and the last priming report). Rows are never inserted or deleted by
+    ** users; the QBit creates the row when it primes.
+    *******************************************************************************/
+   public static QTableMetaData buildSettingsTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchSettings.TABLE_NAME))
+         .withLabel("Quick Search Settings")
+         .withIcon(new QIcon("settings"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
+         .withRecordLabelFormat("Quick Search Settings")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchSettingsTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchSettingsTableCustomizer.class))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("indexingPaused", QFieldType.BOOLEAN).withLabel("Indexing Paused"))
+         .withField(new QFieldMetaData("pauseReason", QFieldType.STRING).withLabel("Pause Reason"))
+         .withField(new QFieldMetaData("pausedAt", QFieldType.DATE_TIME).withLabel("Paused Since").withIsEditable(false))
+         .withField(new QFieldMetaData("pausedBy", QFieldType.STRING).withLabel("Paused By").withIsEditable(false))
+         .withField(new QFieldMetaData("lastPrimedAt", QFieldType.DATE_TIME).withLabel("Last Primed").withIsEditable(false))
+         .withField(new QFieldMetaData("lastPrimingSummary", QFieldType.TEXT).withLabel("Last Priming").withIsEditable(false))
+         .withField(new QFieldMetaData("lastPrimingNewTables", QFieldType.INTEGER).withLabel("Tables Added by Last Priming").withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("pause", "Indexing", new QIcon("pause_circle"), Tier.T1, List.of("id", "indexingPaused", "pauseReason", "pausedAt", "pausedBy")))
+         .withSection(new QFieldSection("priming", "Self-priming", new QIcon("auto_mode"), Tier.T2, List.of("lastPrimedAt", "lastPrimingSummary", "lastPrimingNewTables")))
+         .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_config_tombstone: what an admin removed, so priming does
+    ** not add it back. Written by the QBit; deleting a row lets code defaults
+    ** seed that table or field again.
+    *******************************************************************************/
+   public static QTableMetaData buildTombstoneTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME))
+         .withLabel("Quick Search Removed Config")
+         .withIcon(new QIcon("delete_sweep"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("tableName", "fieldName")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
+         .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withLabel("Field (blank: whole table)").withIsEditable(false))
+         .withField(new QFieldMetaData("removedBy", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("reason", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Removed").withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_index to its field rows.
+    *******************************************************************************/
+   public static QJoinMetaData buildIndexFieldsJoin(QuickSearchQBitConfig config)
+   {
+      return (new QJoinMetaData()
+         .withName(config.applyTablePrefix(INDEX_FIELDS_JOIN))
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
+         .withRightTable(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
+         .withJoinOn(new JoinOn("id", "quickSearchIndexId"))
+         .withOrderBy(new QFilterOrderBy("sortOrder"))
+         .withOrderBy(new QFilterOrderBy("id")));
+   }
+
+
+
+   /*******************************************************************************
+    ** The field rows on the index record view, where fields are added,
+    ** edited and removed.
+    *******************************************************************************/
+   public static QWidgetMetaData buildIndexFieldsWidget(QuickSearchQBitConfig config)
+   {
+      QWidgetMetaData widget = ChildRecordListRenderer.widgetMetaDataBuilder(buildIndexFieldsJoin(config))
+         .withName(config.applyPrefix(INDEX_FIELDS_WIDGET))
+         .withLabel("Searchable Fields")
+         .withCanAddChildRecord(true)
+         .withDisabledFieldsForNewChildRecords(Set.of("quickSearchIndexId"))
+         .withDefaultValuesForNewChildRecordsFromParentFields(Map.of("quickSearchIndexId", "id"))
+         .withOnlyIncludeFieldNames(List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel", "source"))
+         .withMaxRows(100)
+         .getWidgetMetaData();
+      return (widget);
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_index to its run history, newest first.
+    *******************************************************************************/
+   public static QJoinMetaData buildIndexRunsJoin(QuickSearchQBitConfig config)
+   {
+      return (new QJoinMetaData()
+         .withName(config.applyTablePrefix(INDEX_RUNS_JOIN))
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
+         .withRightTable(config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME))
+         .withJoinOn(new JoinOn("id", "quickSearchIndexId"))
+         .withOrderBy(new QFilterOrderBy("id", false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The index record's recent runs (read only).
+    *******************************************************************************/
+   public static QWidgetMetaData buildIndexRunsWidget(QuickSearchQBitConfig config)
+   {
+      return (ChildRecordListRenderer.widgetMetaDataBuilder(buildIndexRunsJoin(config))
+         .withName(config.applyPrefix(INDEX_RUNS_WIDGET))
+         .withLabel("Recent Runs")
+         .withCanAddChildRecord(false)
+         .withOnlyIncludeFieldNames(List.of("id", "runType", "status", "startTime", "endTime", "recordsProcessed", "recordsIndexed", "errorCount"))
+         .withMaxRows(10)
+         .getWidgetMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    ** The custom possible-value sources over the instance's tables and fields.
+    *******************************************************************************/
+   public static List<QPossibleValueSource> buildCustomPossibleValueSources(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         custom(config.applyTablePrefix(ELIGIBLE_TABLE_PVS), QuickSearchTablePossibleValueProviders.EligibleTables.class),
+         custom(config.applyTablePrefix(INDEXABLE_FIELD_PVS), QuickSearchTablePossibleValueProviders.IndexableFields.class),
+         custom(config.applyTablePrefix(TIMESTAMP_FIELD_PVS), QuickSearchTablePossibleValueProviders.TimestampFields.class)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The enum possible-value sources (role, analyzer, documents choice,
+    ** config source).
+    *******************************************************************************/
+   public static List<QPossibleValueSource> buildEnumPossibleValueSources(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         QPossibleValueSource.newForEnum(config.applyTablePrefix(FIELD_ROLE_PVS), QuickSearchFieldRole.values()),
+         QPossibleValueSource.newForEnum(config.applyTablePrefix(FIELD_ANALYZER_PVS), QuickSearchFieldAnalyzer.values()),
+         QPossibleValueSource.newForEnum(config.applyTablePrefix(DOCUMENTS_CHOICE_PVS), QuickSearchDocumentsChoice.values()),
+         stringEnum(config.applyTablePrefix(RUN_TYPE_PVS), "BASEPULL", "Update (basepull)", "RECONCILE", "Reconcile", "FULL_REINDEX", "Full reindex / build"),
+         stringEnum(config.applyTablePrefix(RUN_STATUS_PVS), "RUNNING", "Running", "COMPLETED", "Completed", "FAILED", "Failed", "CANCELLED", "Cancelled"),
+         stringEnum(config.applyTablePrefix(EVENT_STATUS_PVS), "PENDING", "Pending (retried by the next update)", "EXHAUSTED", "Exhausted (gave up)",
+            "AWAITING_REINDEX", "Awaiting full reindex", "DEFERRED", "Deferred (indexing paused)"),
+         stringEnum(config.applyTablePrefix(EVENT_ACTION_PVS), "INDEX", "Index", "DELETE", "Delete"),
+         new QPossibleValueSource()
+            .withName(config.applyTablePrefix(CONFIG_SOURCE_PVS))
+            .withType(QPossibleValueSourceType.ENUM)
+            .withEnumValues(List.of(
+               new QPossibleValue<>(QuickSearchIndex.SOURCE_CODE, "Code default"),
+               new QPossibleValue<>(QuickSearchIndex.SOURCE_OWNER, "Added by owner")))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Add what quick_search_index refers to (its field, settings and tombstone
+    ** tables, join, field-list widget and possible-value sources) to an
+    ** instance whose tables are built with the helpers instead of produced by
+    ** the QBit (tests, hand-assembled instances). Anything already there is
+    ** kept.
+    *******************************************************************************/
+   public static void addCompanionMetaData(QInstance qInstance, QuickSearchQBitConfig config)
+   {
+      for(QTableMetaData table : List.of(buildIndexFieldTable(config), buildSettingsTable(config), buildTombstoneTable(config)))
+      {
+         if(qInstance.getTable(table.getName()) == null)
+         {
+            qInstance.addTable(table);
+         }
+      }
+      QJoinMetaData join = buildIndexFieldsJoin(config);
+      if(qInstance.getJoin(join.getName()) == null)
+      {
+         qInstance.addJoin(join);
+      }
+      for(QJoinMetaData extraJoin : List.of(buildIndexRunsJoin(config)))
+      {
+         if(qInstance.getJoin(extraJoin.getName()) == null)
+         {
+            qInstance.addJoin(extraJoin);
+         }
+      }
+      for(QWidgetMetaData widget : List.of(buildIndexFieldsWidget(config), buildIndexRunsWidget(config)))
+      {
+         if(qInstance.getWidget(widget.getName()) == null)
+         {
+            qInstance.addWidget(widget);
+         }
+      }
+
+      List<QPossibleValueSource> sources = new ArrayList<>(buildCustomPossibleValueSources(config));
+      sources.addAll(buildEnumPossibleValueSources(config));
+      sources.add(new QPossibleValueSource()
+         .withName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
+         .withType(QPossibleValueSourceType.CUSTOM)
+         .withIdType(QFieldType.STRING)
+         .withCustomCodeReference(new QCodeReference(QuickSearchTablePossibleValueProviders.SearchableTables.class)));
+      for(QPossibleValueSource source : sources)
+      {
+         if(qInstance.getPossibleValueSource(source.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(source);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An enum source of string ids and labels (id, label, id, label...).
+    *******************************************************************************/
+   private static QPossibleValueSource stringEnum(String name, String... idsAndLabels)
+   {
+      List<QPossibleValue<?>> values = new ArrayList<>();
+      for(int i = 0; i + 1 < idsAndLabels.length; i += 2)
+      {
+         values.add(new QPossibleValue<>(idsAndLabels[i], idsAndLabels[i + 1]));
+      }
+      return (new QPossibleValueSource().withName(name).withType(QPossibleValueSourceType.ENUM).withEnumValues(values));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QPossibleValueSource custom(String name, Class<?> provider)
+   {
+      return (new QPossibleValueSource()
+         .withName(name)
+         .withType(QPossibleValueSourceType.CUSTOM)
+         .withIdType(QFieldType.STRING)
+         .withCustomCodeReference(new QCodeReference(provider)));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigMetaDataProducer.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the data-driven config's join, field-list widget and
+ ** possible-value sources (see QuickSearchConfigMetaDataHelper).
+ *******************************************************************************/
+public class QuickSearchConfigMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig       config = getQBitConfig();
+      MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexFieldsJoin(config));
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexFieldsWidget(config));
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexRunsJoin(config));
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexRunsWidget(config));
+      for(QPossibleValueSource possibleValueSource : QuickSearchConfigMetaDataHelper.buildCustomPossibleValueSources(config))
+      {
+         output.add(possibleValueSource);
+      }
+      for(QPossibleValueSource possibleValueSource : QuickSearchConfigMetaDataHelper.buildEnumPossibleValueSources(config))
+      {
+         output.add(possibleValueSource);
+      }
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (150);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigProcessMetaDataHelper.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDocumentsChoice;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.processes.config.AddSearchIndexSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.ApplyIndexChangesSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.CancelRunSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.SetIndexingPausedSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.StopIndexingSteps;
+
+
+/*******************************************************************************
+ ** The config processes. Each is a standard streamed-ETL process with a
+ ** review step (the transform's summary: what will happen, counts, problems)
+ ** and a result step, run from its table's Actions (never from the nav):
+ ** - Add Search Index, on quick_search_index (no selection needed);
+ ** - Disable / Remove Search Index and Apply Search Index Changes, on the
+ **   selected quick_search_index rows;
+ ** - Cancel Index Run, on the selected quick_search_index_run rows;
+ ** - Pause / Resume Indexing, on the quick_search_settings row.
+ ** All need the admin permission (quick_search_admin by default).
+ *******************************************************************************/
+public class QuickSearchConfigProcessMetaDataHelper
+{
+   public static final String ADD_INDEX_PROCESS_NAME     = "quickSearchAddIndex";
+   public static final String DISABLE_INDEX_PROCESS_NAME = "quickSearchDisableIndex";
+   public static final String REMOVE_INDEX_PROCESS_NAME  = "quickSearchRemoveIndex";
+   public static final String APPLY_CHANGES_PROCESS_NAME = "quickSearchApplyIndexChanges";
+   public static final String CANCEL_RUN_PROCESS_NAME    = "quickSearchCancelRun";
+   public static final String PAUSE_PROCESS_NAME         = "quickSearchPauseIndexing";
+   public static final String RESUME_PROCESS_NAME        = "quickSearchResumeIndexing";
+
+
+
+   /*******************************************************************************
+    ** Every config process.
+    *******************************************************************************/
+   public static List<QProcessMetaData> buildProcesses(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         buildAddIndexProcess(config),
+         buildStopIndexingProcess(config, DISABLE_INDEX_PROCESS_NAME, "Disable Search Index", "toggle_off", StopIndexingSteps.MODE_DISABLE, QuickSearchDocumentsChoice.KEEP),
+         buildStopIndexingProcess(config, REMOVE_INDEX_PROCESS_NAME, "Remove Search Index", "delete", StopIndexingSteps.MODE_REMOVE, QuickSearchDocumentsChoice.DELETE),
+         buildApplyChangesProcess(config),
+         buildCancelRunProcess(config),
+         buildSetPausedProcess(config, PAUSE_PROCESS_NAME, "Pause Indexing", "pause_circle", true),
+         buildSetPausedProcess(config, RESUME_PROCESS_NAME, "Resume Indexing", "play_circle", false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Add Search Index: the valid way to add a table.
+    *******************************************************************************/
+   public static QProcessMetaData buildAddIndexProcess(QuickSearchQBitConfig config)
+   {
+      String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
+      QProcessMetaData process = base(config, ADD_INDEX_PROCESS_NAME, "Add Search Index", "add_circle", indexTable,
+         AddSearchIndexSteps.Extract.class, AddSearchIndexSteps.Transform.class, AddSearchIndexSteps.Load.class, 0, List.of(
+            new QFieldMetaData(AddSearchIndexSteps.RECORD_TABLE, QFieldType.STRING).withLabel("Table")
+               .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)),
+            new QFieldMetaData("estimatedRecords", QFieldType.INTEGER).withLabel("Records to Index"),
+            new QFieldMetaData("fieldCount", QFieldType.INTEGER).withLabel("Fields"),
+            new QFieldMetaData(AddSearchIndexSteps.FIELD_INTERVAL, QFieldType.INTEGER).withLabel("Update Interval (Minutes)"),
+            new QFieldMetaData(AddSearchIndexSteps.FIELD_TIMESTAMP, QFieldType.STRING).withLabel("Timestamp Field"),
+            new QFieldMetaData(AddSearchIndexSteps.FIELD_ENABLED, QFieldType.BOOLEAN)));
+
+      process.withStep(0, new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Choose a table")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text",
+            "Only tables you can read, that are not indexed yet and are not internal or secret are offered. The review step shows the fields the index starts with "
+               + "and how many records its first build indexes; change the fields afterwards under Searchable Fields on the index."))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_TABLE_NAME, QFieldType.STRING).withLabel("Table").withIsRequired(true)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.ELIGIBLE_TABLE_PVS)))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_INTERVAL, QFieldType.INTEGER).withLabel("Update Interval (Minutes)")
+            .withDefaultValue(config.getDefaultBasepullIntervalMinutes()))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_TIMESTAMP, QFieldType.STRING).withLabel("Timestamp Field (blank: detect)")
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.TIMESTAMP_FIELD_PVS)))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_ENABLED, QFieldType.BOOLEAN).withLabel("Enabled").withDefaultValue(true)));
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Disable or Remove Search Index, with the documents choice.
+    *******************************************************************************/
+   static QProcessMetaData buildStopIndexingProcess(QuickSearchQBitConfig config, String name, String label, String icon, String mode, QuickSearchDocumentsChoice defaultChoice)
+   {
+      String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(config.applyPrefix(name))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(indexTable)
+         .withSourceTable(indexTable)
+         .withDestinationTable(indexTable)
+         .withExtractStepClass(ExtractViaQueryStep.class)
+         .withTransformStepClass(StopIndexingSteps.Transform.class)
+         .withLoadStepClass(StopIndexingSteps.Load.class)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withTransactionLevelAutoCommit()
+         .withMinInputRecords(1)
+         .withFields(List.of(new QFieldMetaData(StopIndexingSteps.FIELD_MODE, QFieldType.STRING).withDefaultValue(mode)))
+         .withReviewStepRecordFields(indexReviewFields(config))
+         .getProcessMetaData();
+      process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+
+      process.withStep(0, new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Search documents")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text",
+            StopIndexingSteps.MODE_REMOVE.equals(mode)
+               ? "Removing stops indexing the table and deletes its index and field rows. Code defaults never add it back (see Quick Search Removed Config)."
+               : "Disabling stops indexing the table and hides it from search; enabling it again rebuilds it."))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withFormField(new QFieldMetaData(StopIndexingSteps.FIELD_DOCUMENTS, QFieldType.STRING).withLabel("Search Documents").withIsRequired(true)
+            .withDefaultValue(defaultChoice.name())
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.DOCUMENTS_CHOICE_PVS))));
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Apply Search Index Changes: preview field changes, then rebuild.
+    *******************************************************************************/
+   static QProcessMetaData buildApplyChangesProcess(QuickSearchQBitConfig config)
+   {
+      String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
+      return (base(config, APPLY_CHANGES_PROCESS_NAME, "Apply Search Index Changes", "published_with_changes", indexTable,
+         ExtractViaQueryStep.class, ApplyIndexChangesSteps.Transform.class, ApplyIndexChangesSteps.Load.class, 1, indexReviewFields(config)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Cancel Index Run: the cooperative cancel flag.
+    *******************************************************************************/
+   static QProcessMetaData buildCancelRunProcess(QuickSearchQBitConfig config)
+   {
+      String runTable = config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME);
+      return (base(config, CANCEL_RUN_PROCESS_NAME, "Cancel Index Run", "cancel", runTable,
+         ExtractViaQueryStep.class, CancelRunSteps.Transform.class, CancelRunSteps.Load.class, 1, List.of(
+            new QFieldMetaData("id", QFieldType.INTEGER),
+            new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)),
+            new QFieldMetaData("runType", QFieldType.STRING),
+            new QFieldMetaData("status", QFieldType.STRING),
+            new QFieldMetaData("recordsProcessed", QFieldType.INTEGER))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Pause or Resume Indexing.
+    *******************************************************************************/
+   static QProcessMetaData buildSetPausedProcess(QuickSearchQBitConfig config, String name, String label, String icon, boolean pause)
+   {
+      String settingsTable = config.applyTablePrefix(QuickSearchSettings.TABLE_NAME);
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(config.applyPrefix(name))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(settingsTable)
+         .withSourceTable(settingsTable)
+         .withDestinationTable(settingsTable)
+         .withExtractStepClass(ExtractViaQueryStep.class)
+         .withTransformStepClass(SetIndexingPausedSteps.Transform.class)
+         .withLoadStepClass(SetIndexingPausedSteps.Load.class)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withTransactionLevelAutoCommit()
+         .withMinInputRecords(1)
+         .withFields(List.of(new QFieldMetaData(SetIndexingPausedSteps.FIELD_PAUSE, QFieldType.BOOLEAN).withDefaultValue(pause)))
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("indexingPaused", QFieldType.BOOLEAN).withLabel("Indexing Paused"),
+            new QFieldMetaData("pausedAt", QFieldType.DATE_TIME).withLabel("Paused Since"),
+            new QFieldMetaData("pausedBy", QFieldType.STRING).withLabel("Paused By")))
+         .getProcessMetaData();
+      process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+
+      if(pause)
+      {
+         process.withStep(0, new QFrontendStepMetaData()
+            .withName("input")
+            .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+            .withLabel("Pause indexing")
+            .withFormField(new QFieldMetaData(SetIndexingPausedSteps.FIELD_REASON, QFieldType.STRING).withLabel("Reason (optional)")));
+      }
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** The index fields shown on the review step.
+    *******************************************************************************/
+   private static List<QFieldMetaData> indexReviewFields(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         new QFieldMetaData("id", QFieldType.INTEGER),
+         new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table")
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)),
+         new QFieldMetaData("enabled", QFieldType.BOOLEAN),
+         new QFieldMetaData("status", QFieldType.STRING),
+         new QFieldMetaData("documentCount", QFieldType.INTEGER)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A streamed-ETL process with a validated review step, gated by the admin
+    ** permission.
+    *******************************************************************************/
+   private static QProcessMetaData base(QuickSearchQBitConfig config, String name, String label, String icon, String tableName, Class<? extends AbstractExtractStep> extract,
+      Class<? extends AbstractTransformStep> transform, Class<? extends AbstractLoadStep> load, Integer minInputRecords, List<QFieldMetaData> reviewFields)
+   {
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(config.applyPrefix(name))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(tableName)
+         .withSourceTable(tableName)
+         .withDestinationTable(tableName)
+         .withExtractStepClass(extract)
+         .withTransformStepClass(transform)
+         .withLoadStepClass(load)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withTransactionLevelAutoCommit()
+         .withMinInputRecords(minInputRecords)
+         .withReviewStepRecordFields(reviewFields)
+         .getProcessMetaData();
+      process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+      return (process);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigProcessesMetaDataProducer.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the config processes (see QuickSearchConfigProcessMetaDataHelper).
+ *******************************************************************************/
+public class QuickSearchConfigProcessesMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+      for(QProcessMetaData process : QuickSearchConfigProcessMetaDataHelper.buildProcesses(getQBitConfig()))
+      {
+         output.add(process);
+      }
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigTombstoneTableMetaDataProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces quick_search_config_tombstone, what an admin removed.
+ *******************************************************************************/
+public class QuickSearchConfigTombstoneTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchConfigMetaDataHelper.buildTombstoneTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDeveloperPanelsMetaDataProducer.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelsMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDeveloperPanelProvider;
+
+
+/*******************************************************************************
+ ** Registers the Quick Search record developer panel
+ ** ({@link QuickSearchDeveloperPanelProvider}) in the instance's developer
+ ** panels. The provider decides per table and user whether the panel shows
+ ** (indexed tables, users who may read Quick Search Index). Registration is
+ ** idempotent, so producing the QBit again adds nothing twice.
+ *******************************************************************************/
+public class QuickSearchDeveloperPanelsMetaDataProducer extends QBitComponentMetaDataProducer<DeveloperPanelsMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DeveloperPanelsMetaData produce(QInstance qInstance) throws QException
+   {
+      return (DeveloperPanelsMetaData.ofOrWithNew(qInstance).withProvider(new QCodeReference(QuickSearchDeveloperPanelProvider.class)));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticEventTableMetaDataProducer.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quick_search_diagnostic_event table when enableDiagnosticTools is true.
+ *******************************************************************************/
+public class QuickSearchDiagnosticEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildDiagnosticEventTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticsProcessMetaDataHelper.java

@@ -0,0 +1,279 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.AbstractDiagnosticToolStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.AnalyzeTextStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.InspectRecordStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.ReindexRecordsStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.SearchPlaygroundStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.ValidateIndexStep;
+
+
+/*******************************************************************************
+ ** Builds the diagnostic tool processes. Every one is a table or record
+ ** action (it has a tableName and is never added to an app's nav):
+ ** - on Quick Search Index: Search Playground, Validate Search Index (also
+ **   schedulable), Test Search Analyzer;
+ ** - on every indexed table and on Quick Search Failed Event: Inspect Search
+ **   Document and Reindex in Search.
+ ** Read-only tools use debugPermissionRules; Reindex in Search uses
+ ** adminPermissionRules.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsProcessMetaDataHelper
+{
+   public static final String PLAYGROUND_PROCESS_NAME     = "quickSearchPlayground";
+   public static final String VALIDATE_PROCESS_NAME       = "quickSearchValidateIndex";
+   public static final String ANALYZE_PROCESS_NAME        = "quickSearchAnalyzeText";
+   public static final String INSPECT_PROCESS_PREFIX      = "quickSearchInspect";
+   public static final String REINDEX_RECORDS_PROCESS_PREFIX = "quickSearchReindexRecords";
+
+
+
+   /*******************************************************************************
+    ** Name of the Inspect Search Document process of a table.
+    *******************************************************************************/
+   public static String inspectProcessName(QuickSearchQBitConfig config, String tableName)
+   {
+      return (config.applyPrefix(INSPECT_PROCESS_PREFIX) + "_" + tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Name of the Reindex in Search process of a table.
+    *******************************************************************************/
+   public static String reindexRecordsProcessName(QuickSearchQBitConfig config, String tableName)
+   {
+      return (config.applyPrefix(REINDEX_RECORDS_PROCESS_PREFIX) + "_" + tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Search Playground, a table action on Quick Search Index.
+    *******************************************************************************/
+   public static QProcessMetaData buildPlaygroundProcess(QuickSearchQBitConfig config)
+   {
+      return (new QProcessMetaData()
+         .withName(config.applyPrefix(PLAYGROUND_PROCESS_NAME))
+         .withLabel("Search Playground")
+         .withIcon(new QIcon("travel_explore"))
+         .withTableName(config.getQuickSearchIndexTableName())
+         .withMinInputRecords(0)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(SearchPlaygroundStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
+         .withStep(form("Search", List.of(
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_TERM, QFieldType.STRING).withLabel("Search term").withIsRequired(true).withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH),
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_TABLES, QFieldType.STRING).withLabel("Tables (optional; names separated by commas)"),
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_LIMIT_PER_TABLE, QFieldType.INTEGER).withLabel("Results per table").withDefaultValue(RecordSearchAction.DEFAULT_LIMIT_PER_TABLE),
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_EXPLAIN_HIT, QFieldType.STRING).withLabel("Explain hit (optional document id, e.g. blogPost:42)"))))
+         .withStep(backend(SearchPlaygroundStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Test Search Analyzer, a table action on Quick Search Index.
+    *******************************************************************************/
+   public static QProcessMetaData buildAnalyzeProcess(QuickSearchQBitConfig config)
+   {
+      return (new QProcessMetaData()
+         .withName(config.applyPrefix(ANALYZE_PROCESS_NAME))
+         .withLabel("Test Search Analyzer")
+         .withIcon(new QIcon("text_fields"))
+         .withTableName(config.getQuickSearchIndexTableName())
+         .withMinInputRecords(0)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(AnalyzeTextStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
+         .withStep(form("Analyze", List.of(
+            new QFieldMetaData(AnalyzeTextStep.FIELD_TEXT, QFieldType.TEXT).withLabel("Text (as a field value)").withIsRequired(true),
+            new QFieldMetaData(AnalyzeTextStep.FIELD_QUERY, QFieldType.STRING).withLabel("Search term (optional)").withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH))))
+         .withStep(backend(AnalyzeTextStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate Search Index, a table action on Quick Search Index; scheduled
+    ** by cron when validationCronExpression and a schedulerName are set.
+    *******************************************************************************/
+   public static QProcessMetaData buildValidateProcess(QuickSearchQBitConfig config)
+   {
+      QProcessMetaData process = new QProcessMetaData()
+         .withName(config.applyPrefix(VALIDATE_PROCESS_NAME))
+         .withLabel("Validate Search Index")
+         .withIcon(new QIcon("fact_check"))
+         .withTableName(config.getQuickSearchIndexTableName())
+         .withMinInputRecords(0)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(ValidateIndexStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
+         .withStep(form("Validate", List.of(
+            new QFieldMetaData(ValidateIndexStep.FIELD_SAMPLE_SIZE, QFieldType.INTEGER).withLabel("Records and documents to sample per table")
+               .withDefaultValue(config.getValidationSampleSize()))))
+         .withStep(backend(ValidateIndexStep.class, AbstractDiagnosticToolStep.STEP_RUN, null))
+         .withStep(result(AbstractDiagnosticToolStep.STEP_RESULT, "Validation report"));
+
+      if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getValidationCronExpression()))
+      {
+         process.withSchedule(new QScheduleMetaData()
+            .withSchedulerName(config.getSchedulerName())
+            .withCronExpression(config.getValidationCronExpression())
+            .withCronTimeZoneId(config.getValidationCronTimeZoneId())
+            .withDescription("Quick Search index validation"));
+      }
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect Search Document, a record action on a table (an indexed table,
+    ** or Quick Search Failed Event for the event's record).
+    *******************************************************************************/
+   public static QProcessMetaData buildInspectProcess(QuickSearchQBitConfig config, String tableName)
+   {
+      return (new QProcessMetaData()
+         .withName(inspectProcessName(config, tableName))
+         .withLabel("Inspect Search Document")
+         .withIcon(new QIcon("manage_search"))
+         .withTableName(tableName)
+         .withMinInputRecords(1)
+         .withMaxInputRecords(InspectRecordStep.MAX_RECORDS)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(InspectRecordStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, tableName))
+         .withStep(form("Inspect", List.of(
+            new QFieldMetaData(InspectRecordStep.FIELD_TERM, QFieldType.STRING).withLabel("Why doesn't it match... (optional search term)")
+               .withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH))))
+         .withStep(backend(InspectRecordStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Reindex in Search, a record action on a table (an indexed table, or
+    ** Quick Search Failed Event for the event's record). Writes to the index,
+    ** so it uses the admin permission rules.
+    *******************************************************************************/
+   public static QProcessMetaData buildReindexRecordsProcess(QuickSearchQBitConfig config, String tableName)
+   {
+      return (new QProcessMetaData()
+         .withName(reindexRecordsProcessName(config, tableName))
+         .withLabel("Reindex in Search")
+         .withIcon(new QIcon("sync"))
+         .withTableName(tableName)
+         .withMinInputRecords(1)
+         .withMaxInputRecords(ReindexRecordsStep.MAX_RECORDS)
+         .withPermissionRules(clone(config.getAdminPermissionRules()))
+         .withStep(backend(ReindexRecordsStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, tableName))
+         .withStep(result(AbstractDiagnosticToolStep.STEP_CONFIRM, "Confirm"))
+         .withStep(backend(ReindexRecordsStep.class, AbstractDiagnosticToolStep.STEP_RUN, null))
+         .withStep(result(AbstractDiagnosticToolStep.STEP_RESULT, "Reindexed")));
+   }
+
+
+
+   /*******************************************************************************
+    ** The fixed processes (on Quick Search Index and Quick Search Failed
+    ** Event).
+    *******************************************************************************/
+   public static List<QProcessMetaData> buildFixedProcesses(QuickSearchQBitConfig config)
+   {
+      List<QProcessMetaData> processes = new ArrayList<>();
+      processes.add(buildPlaygroundProcess(config));
+      processes.add(buildValidateProcess(config));
+      processes.add(buildAnalyzeProcess(config));
+      processes.add(buildInspectProcess(config, config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)));
+      processes.add(buildReindexRecordsProcess(config, config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)));
+      return (processes);
+   }
+
+
+
+   /*******************************************************************************
+    ** A backend step of a tool; with recordsTableName, the step receives the
+    ** selected records of that table.
+    *******************************************************************************/
+   private static QBackendStepMetaData backend(Class<?> stepClass, String name, String recordsTableName)
+   {
+      QBackendStepMetaData step = new QBackendStepMetaData().withName(name).withCode(new QCodeReference(stepClass));
+      if(recordsTableName != null)
+      {
+         step.withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName(recordsTableName)));
+      }
+      return (step);
+   }
+
+
+
+   /*******************************************************************************
+    ** The FORM step: fields above, the tool's HTML below.
+    *******************************************************************************/
+   private static QFrontendStepMetaData form(String label, List<QFieldMetaData> fields)
+   {
+      QFrontendStepMetaData step = new QFrontendStepMetaData()
+         .withName(AbstractDiagnosticToolStep.STEP_FORM)
+         .withLabel(label)
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML));
+      fields.forEach(step::withFormField);
+      return (step);
+   }
+
+
+
+   /*******************************************************************************
+    ** An HTML-only frontend step.
+    *******************************************************************************/
+   private static QFrontendStepMetaData result(String name, String label)
+   {
+      return (new QFrontendStepMetaData()
+         .withName(name)
+         .withLabel(label)
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QPermissionRules clone(QPermissionRules rules)
+   {
+      return (rules == null ? null : rules.clone());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticsProcessesMetaDataProducer.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the fixed diagnostic tool processes (Search Playground, Validate
+ ** Search Index, Test Search Analyzer, and Inspect / Reindex on Quick Search
+ ** Failed Event) when enableDiagnosticTools is true. The per-table record
+ ** actions are added by QuickSearchQBitProducer once the indexed tables are
+ ** known.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsProcessesMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+      for(QProcessMetaData process : QuickSearchDiagnosticsProcessMetaDataHelper.buildFixedProcesses(getQBitConfig()))
+      {
+         output.add(process);
+      }
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticsWidgetMetaDataProducer.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDiagnosticsWidgetRenderer;
+
+
+/*******************************************************************************
+ ** Produces the Search Diagnostics widget (quickSearchDiagnostics) when
+ ** enableDiagnosticTools is true. The admin app shows it on its home; a host
+ ** with its own Quick Search home adds it there by name.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsWidgetMetaDataProducer extends QBitComponentMetaDataProducer<QWidgetMetaData, QuickSearchQBitConfig>
+{
+   public static final String WIDGET_NAME = "quickSearchDiagnostics";
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public QWidgetMetaData produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig config = getQBitConfig();
+      return (new QWidgetMetaData()
+         .withName(config.applyPrefix(WIDGET_NAME))
+         .withLabel("Search Diagnostics")
+         .withIcon("troubleshoot")
+         .withType(WidgetType.HTML.getType())
+         .withGridColumns(12)
+         .withIsCard(true)
+         .withCodeReference(new QCodeReference(QuickSearchDiagnosticsWidgetRenderer.class))
+         .withPermissionRules(config.getDebugPermissionRules() == null ? null : config.getDebugPermissionRules().clone()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (300);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchFailedEventTableMetaDataProducer.java

@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 
 
 /*******************************************************************************
- ** Produces the quickSearchFailedEvent table.
+ ** Produces the quick_search_failed_event table.
  *******************************************************************************/
 public class QuickSearchFailedEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
 {

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexFieldTableMetaDataProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces quick_search_index_field, the searchable fields of each index.
+ *******************************************************************************/
+public class QuickSearchIndexFieldTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchConfigMetaDataHelper.buildIndexFieldTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexPossibleValueSourceProducer.java

@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 
 
 /*******************************************************************************
- ** Produces the possible value source over the quickSearchIndex table.
+ ** Produces the possible value source over the quick_search_index table.
  *******************************************************************************/
 public class QuickSearchIndexPossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
 {

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexRunTableMetaDataProducer.java

@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 
 
 /*******************************************************************************
- ** Produces the quickSearchIndexRun table.
+ ** Produces the quick_search_index_run table.
  *******************************************************************************/
 public class QuickSearchIndexRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
 {

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexTableMetaDataProducer.java

@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 
 
 /*******************************************************************************
- ** Produces the quickSearchIndex table.
+ ** Produces the quick_search_index table.
  *******************************************************************************/
 public class QuickSearchIndexTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
 {

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchProcessMetaDataHelper.java

@@ -22,6 +22,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
@@ -51,7 +53,7 @@ public class QuickSearchProcessMetaDataHelper
    public static QProcessMetaData buildBasepullProcess(QuickSearchQBitConfig config)
    {
       QProcessMetaData process = buildProcess(config, BASEPULL_PROCESS_NAME, "Quick Search Basepull Index", "basepull", BasepullIndexStep.class, "update");
-      process.withStep(resultStep("Basepull complete", "tablesIndexed", "failedEventsReplayed"));
+      process.withStep(resultStep("Basepull complete", "tablesIndexed", "tablesBuilt", "failedEventsReplayed", "paused"));
 
       if(StringUtils.hasContent(config.getSchedulerName()))
       {
@@ -73,7 +75,7 @@ public class QuickSearchProcessMetaDataHelper
    {
       QProcessMetaData process = buildProcess(config, FULL_REINDEX_PROCESS_NAME, "Quick Search Full Reindex", "fullReindex", FullReindexStep.class, "refresh");
       process.withStep(0, inputStep(config, "Leave the table blank to rebuild every table into a fresh index and swap the alias; pick a table to rebuild just that table in place."));
-      process.withStep(resultStep("Full reindex complete", "recordsIndexed", "aliasSwapped"));
+      process.withStep(resultStep("Full reindex complete", "recordsIndexed", "aliasSwapped", "message"));
       return (process);
    }
 
@@ -87,7 +89,7 @@ public class QuickSearchProcessMetaDataHelper
    {
       QProcessMetaData process = buildProcess(config, RECONCILE_PROCESS_NAME, "Quick Search Reconcile Index", "reconcile", ReconcileIndexStep.class, "sync");
       process.withStep(0, inputStep(config, "Leave the table blank to reconcile every table and remove documents of unconfigured tables; pick a table to reconcile just that one."));
-      process.withStep(resultStep("Reconcile complete", "recordsIndexed", "documentsRemoved", "orphanDocumentsRemoved", "tablesSkipped"));
+      process.withStep(resultStep("Reconcile complete", "recordsIndexed", "documentsRemoved", "orphanDocumentsRemoved", "tablesSkipped", "message"));
 
       if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getReconcileCronExpression()))
       {
@@ -104,18 +106,21 @@ public class QuickSearchProcessMetaDataHelper
 
 
    /*******************************************************************************
-    ** Optional table picker shown before the backend step.
+    ** Optional table picker shown before the backend step. Its field is
+    ** searchTableName, not tableName, which a process run from a table gets
+    ** from the frontend (the table it runs from).
     *******************************************************************************/
    private static QFrontendStepMetaData inputStep(QuickSearchQBitConfig config, String help)
    {
       return (new QFrontendStepMetaData()
          .withName("input")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text", help))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
          .withLabel("Choose table")
-         .withFormField(new QFieldMetaData("tableName", QFieldType.STRING)
+         .withFormField(new QFieldMetaData("searchTableName", QFieldType.STRING)
             .withLabel("Table (optional)")
-            .withPossibleValueSourceName(config.applyPrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
-            .withIsRequired(false))
-         .withViewField(new QFieldMetaData("help", QFieldType.STRING).withLabel(" ").withDefaultValue(help)));
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
+            .withIsRequired(false)));
    }
 
 
@@ -140,10 +145,15 @@ public class QuickSearchProcessMetaDataHelper
     *******************************************************************************/
    private static QProcessMetaData buildProcess(QuickSearchQBitConfig config, String baseName, String label, String stepName, Class<?> stepClass, String icon)
    {
+      ///////////////////////////////////////////////////////////////////////
+      // run from Quick Search Index's Actions (processes are never in the //
+      // nav); the input screen's table picker is searchTableName          //
+      ///////////////////////////////////////////////////////////////////////
       return (new QProcessMetaData()
          .withName(config.applyPrefix(baseName))
          .withLabel(label)
          .withIcon(new QIcon(icon))
+         .withTableName(config.getQuickSearchIndexTableName())
          .withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone())
          .withStep(new QBackendStepMetaData()
             .withName(stepName)

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchSearchableTablePossibleValueSourceProducer.java

@@ -17,70 +17,45 @@
 package com.kingsrook.qbits.quicksearch.metadata;
 
 
-import java.util.ArrayList;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
-import com.kingsrook.qbits.quicksearch.SearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchTablePossibleValueProviders;
 
 
 /*******************************************************************************
- ** Enum possible value source listing the searchable tables (from annotated
- ** entity classes and config-driven tables), used by the process input screens
- ** so an operator can pick one table for a full reindex or reconcile.
+ ** Possible value source listing the indexed tables, used by the process input
+ ** screens (pick one table for a full reindex or reconcile) and to show stored
+ ** table names by label.
  *******************************************************************************/
 public class QuickSearchSearchableTablePossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
 {
-   public static final String NAME = "quickSearchSearchableTable";
+   public static final String NAME = "quick_search_searchable_table";
+
+   /** Possible-value source name before 1.1.0-RC.3 (camelCase). */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_NAME = "quickSearchSearchableTable";
 
 
 
    /*******************************************************************************
-    **
+    ** A custom source over the effective indexed tables (since 1.1.0-RC.3; it
+    ** was an enum of the code-declared tables), so tables added at runtime
+    ** appear at once, by label.
     *******************************************************************************/
    @Override
    public QPossibleValueSource produce(QInstance qInstance) throws QException
    {
-      QuickSearchQBitConfig config = getQBitConfig();
-
-      Set<String> tableNames = new LinkedHashSet<>();
-      for(Class<?> entityClass : config.getSearchableEntityClasses() == null ? List.<Class<?>>of() : config.getSearchableEntityClasses())
-      {
-         QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class);
-         if(annotation != null)
-         {
-            tableNames.add(annotation.tableName());
-         }
-      }
-      for(SearchableTableConfig tableConfig : config.getSearchableTables() == null ? List.<SearchableTableConfig>of() : config.getSearchableTables())
-      {
-         if(tableConfig.getTableName() != null)
-         {
-            tableNames.add(tableConfig.getTableName());
-         }
-      }
-
-      List<QPossibleValue<?>> values = new ArrayList<>();
-      for(String tableName : tableNames)
-      {
-         QTableMetaData table = qInstance.getTable(tableName);
-         String         label = table != null && table.getLabel() != null ? table.getLabel() : tableName;
-         values.add(new QPossibleValue<>(tableName, label));
-      }
-
       return (new QPossibleValueSource()
-         .withName(config.applyPrefix(NAME))
-         .withType(QPossibleValueSourceType.ENUM)
-         .withEnumValues(values));
+         .withName(getQBitConfig().applyTablePrefix(NAME))
+         .withType(QPossibleValueSourceType.CUSTOM)
+         .withIdType(QFieldType.STRING)
+         .withCustomCodeReference(new QCodeReference(QuickSearchTablePossibleValueProviders.SearchableTables.class)));
    }
 
 

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchSettingsTableMetaDataProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces quick_search_settings, the system settings row (global pause).
+ *******************************************************************************/
+public class QuickSearchSettingsTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchConfigMetaDataHelper.buildSettingsTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchTableMetaDataHelper.java

@@ -17,16 +17,32 @@
 package com.kingsrook.qbits.quicksearch.metadata;
 
 
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchIndexTableCustomizer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
 
 
 /*******************************************************************************
@@ -37,25 +53,46 @@ public class QuickSearchTableMetaDataHelper
 {
 
    /*******************************************************************************
-    ** quickSearchIndex: one row per indexed table.
+    ** quick_search_index: one row per indexed table.
     *******************************************************************************/
    public static QTableMetaData buildIndexTable(QuickSearchQBitConfig config)
    {
+      ////////////////////////////////////////////////////////////////////////////
+      // rows are added by Add Search Index (and by self-priming) and removed  //
+      // by Remove Search Index, which validate, preview and build or clean up //
+      // the index; the generic create and delete screens are turned off      //
+      ////////////////////////////////////////////////////////////////////////////
       return (new QTableMetaData()
-         .withName(config.applyPrefix(QuickSearchIndex.TABLE_NAME))
+         .withName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
          .withLabel("Quick Search Index")
          .withIcon(new QIcon("manage_search"))
          .withBackendName(config.getBackendName())
          .withPrimaryKeyField("id")
          .withRecordLabelFields("tableName")
          .withUniqueKey(new UniqueKey("tableName"))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
          .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
+         .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
+         .withAssociation(new Association()
+            .withName(QuickSearchConfigMetaDataHelper.FIELDS_ASSOCIATION)
+            .withAssociatedTableName(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
+            .withJoinName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.INDEX_FIELDS_JOIN)))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withIsEditable(false))
+         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table").withIsRequired(true).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
          .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
-         .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withIsEditable(false))
-         .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER).withLabel("Update Interval (Minutes)"))
+         .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withLabel("Timestamp Field")
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.TIMESTAMP_FIELD_PVS)))
+         .withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.CONFIG_SOURCE_PVS)))
+         .withField(new QFieldMetaData("ownerEdited", QFieldType.BOOLEAN).withLabel("Edited by Owner").withIsEditable(false))
+         .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.TEXT).withLabel("Fields as Last Built").withIsEditable(false))
          .withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("lastReconcileTime", QFieldType.DATE_TIME).withIsEditable(false))
@@ -63,76 +100,195 @@ public class QuickSearchTableMetaDataHelper
          .withField(new QFieldMetaData("lastErrorMessage", QFieldType.TEXT).withIsEditable(false))
          .withField(new QFieldMetaData("recordCount", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("documentCount", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("realTimeErrorCount", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("realTimeErrorCount", QFieldType.INTEGER).withIsEditable(false)
+            .withHelpContent(new QHelpContent("Real-time indexing failures since the last full reindex or basepull")))
          .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
-         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Search Index", new QIcon("manage_search"), Tier.T1,
+            List.of("id", "tableName", "enabled", "status", "basepullIntervalMinutes", "basepullTimestampField")))
+         .withSection(new QFieldSection("fields", "Searchable Fields", new QIcon("text_fields"), Tier.T2, null)
+            .withWidgetName(config.applyPrefix(QuickSearchConfigMetaDataHelper.INDEX_FIELDS_WIDGET)))
+         .withSection(new QFieldSection("runs", "Recent Runs", new QIcon("history"), Tier.T2, null)
+            .withWidgetName(config.applyPrefix(QuickSearchConfigMetaDataHelper.INDEX_RUNS_WIDGET)))
+         .withSection(new QFieldSection("activity", "Activity", new QIcon("insights"), Tier.T2,
+            List.of("lastBasepullTime", "lastFullReindexTime", "lastReconcileTime", "lastRunStatus", "lastErrorMessage", "recordCount", "documentCount", "realTimeErrorCount")))
+         .withSection(new QFieldSection("origin", "Origin", new QIcon("info"), Tier.T3,
+            List.of("source", "ownerEdited", "searchableFieldsJson", "createDate", "modifyDate"))));
    }
 
 
 
    /*******************************************************************************
-    ** quickSearchIndexRun: history of basepull, reindex and reconcile runs.
+    ** quick_search_index_run: history of basepull, reindex and reconcile runs.
     *******************************************************************************/
    public static QTableMetaData buildIndexRunTable(QuickSearchQBitConfig config)
    {
+      /////////////////////////////////////////////////////////////////////////
+      // run history is written by the QBit only: no create or edit screens //
+      // (Delete stays, for clearing); Cancel Index Run sets the cancel flag //
+      /////////////////////////////////////////////////////////////////////////
       return (new QTableMetaData()
-         .withName(config.applyPrefix(QuickSearchIndexRun.TABLE_NAME))
+         .withName(config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME))
          .withLabel("Quick Search Index Run")
          .withIcon(new QIcon("history"))
          .withBackendName(config.getBackendName())
          .withPrimaryKeyField("id")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("runType", "startTime")
          .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withIsEditable(false)
-            .withPossibleValueSourceName(config.applyPrefix(QuickSearchIndex.TABLE_NAME)))
-         .withField(new QFieldMetaData("runType", QFieldType.STRING).withIsEditable(false))
-         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)))
+         .withField(new QFieldMetaData("runType", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.RUN_TYPE_PVS)))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.RUN_STATUS_PVS)))
          .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("errorCount", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("cancelRequested", QFieldType.BOOLEAN).withIsEditable(false))
+         .withField(new QFieldMetaData("cancelRequestedBy", QFieldType.STRING).withIsEditable(false))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
-         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Last Heartbeat").withIsEditable(false))
+         .withSection(new QFieldSection("run", "Run", new QIcon("history"), Tier.T1, List.of("id", "quickSearchIndexId", "runType", "status", "startTime", "endTime")))
+         .withSection(new QFieldSection("counts", "Counts", new QIcon("functions"), Tier.T2, List.of("recordsProcessed", "recordsIndexed", "errorCount", "errorMessage")))
+         .withSection(new QFieldSection("cancel", "Cancel", new QIcon("cancel"), Tier.T2, List.of("cancelRequested", "cancelRequestedBy")))
+         .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
    }
 
 
 
    /*******************************************************************************
-    ** quickSearchFailedEvent: real-time index or delete events that could not
+    ** quick_search_failed_event: real-time index or delete events that could not
     ** reach OpenSearch, replayed by the scheduled basepull.
     *******************************************************************************/
    public static QTableMetaData buildFailedEventTable(QuickSearchQBitConfig config)
    {
+      /////////////////////////////////////////////////////////////////////////
+      // system events, written by the QBit only: no create or edit screens //
+      // (Delete stays, for clearing)                                        //
+      /////////////////////////////////////////////////////////////////////////
       return (new QTableMetaData()
-         .withName(config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME))
+         .withName(config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME))
          .withLabel("Quick Search Failed Event")
          .withIcon(new QIcon("error_outline"))
          .withBackendName(config.getBackendName())
          .withPrimaryKeyField("id")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("tableName", "recordId")
          .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table").withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
          .withField(new QFieldMetaData("recordId", QFieldType.STRING).withIsEditable(false))
-         .withField(new QFieldMetaData("action", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("action", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.EVENT_ACTION_PVS)))
          .withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
          .withField(new QFieldMetaData("attempts", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.EVENT_STATUS_PVS)))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
-         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("event", "Event", new QIcon("error_outline"), Tier.T1, List.of("id", "tableName", "recordId", "action", "status")))
+         .withSection(new QFieldSection("error", "Error and retries", new QIcon("replay"), Tier.T2, List.of("errorMessage", "attempts")))
+         .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
    }
 
 
 
    /*******************************************************************************
-    ** Possible value source over quickSearchIndex, so run rows show the table
+    ** quick_search_diagnostic_event: failed searches, diagnostic tool runs (the
+    ** audit trail) and refused runs. Written by the QBit only: insert, update
+    ** and delete are disabled for users and the API (core actions, which the
+    ** QBit uses, do not check capabilities).
+    *******************************************************************************/
+   public static QTableMetaData buildDiagnosticEventTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME))
+         .withLabel("Quick Search Diagnostic Event")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
+         .withIcon(new QIcon("bug_report"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("eventType", "source")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("eventType", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("outcome", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("userName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("tableNames", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("searchTerm", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("durationMillis", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("errorClass", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("detail", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("hostName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Event", new QIcon("bug_report"), Tier.T1, List.of("id", "createDate", "eventType", "source", "outcome")))
+         .withSection(new QFieldSection("request", "Request", new QIcon("search"), Tier.T2, List.of("userName", "tableNames", "searchTerm", "durationMillis", "hostName")))
+         .withSection(new QFieldSection("error", "Error and detail", new QIcon("error_outline"), Tier.T2, List.of("errorClass", "errorMessage", "detail"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_validation_run: index validation reports (history). The
+    ** report is an HTML field, shown in full on the record view; only the
+    ** QBit writes rows (insert, update and delete are disabled for users and
+    ** the API), so the HTML is always the QBit's escaped output.
+    *******************************************************************************/
+   public static QTableMetaData buildValidationRunTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME))
+         .withLabel("Quick Search Validation Run")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
+         .withIcon(new QIcon("fact_check"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("status", "startTime")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("triggerType", QFieldType.STRING).withLabel("Trigger").withIsEditable(false))
+         .withField(new QFieldMetaData("userName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("tableNames", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("sampleSize", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("tablesChecked", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("failCount", QFieldType.INTEGER).withLabel("Failed Checks").withIsEditable(false))
+         .withField(new QFieldMetaData("warnCount", QFieldType.INTEGER).withLabel("Warnings").withIsEditable(false))
+         .withField(new QFieldMetaData("summary", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("reportHtml", QFieldType.HTML).withLabel("Report").withIsEditable(false))
+         .withField(new QFieldMetaData("reportJson", QFieldType.TEXT).withLabel("Report JSON").withIsEditable(false)
+            .withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json"))))
+         .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Run", new QIcon("fact_check"), Tier.T1, List.of("id", "status", "summary", "triggerType", "startTime", "endTime")))
+         .withSection(new QFieldSection("scope", "Scope", new QIcon("table_chart"), Tier.T2, List.of("userName", "tableNames", "sampleSize", "tablesChecked", "failCount", "warnCount", "createDate")))
+         .withSection(new QFieldSection("report", "Report", new QIcon("description"), Tier.T2, List.of("reportHtml")))
+         .withSection(new QFieldSection("data", "Report data (JSON)", new QIcon("data_object"), Tier.T3, List.of("reportJson"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Possible value source over quick_search_index, so run rows show the table
     ** name instead of an id.
     *******************************************************************************/
    public static QPossibleValueSource buildIndexPossibleValueSource(QuickSearchQBitConfig config)
    {
-      return (QPossibleValueSource.newForTable(config.applyPrefix(QuickSearchIndex.TABLE_NAME)));
+      return (QPossibleValueSource.newForTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)));
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchValidationRunTableMetaDataProducer.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quick_search_validation_run table when enableDiagnosticTools is true.
+ *******************************************************************************/
+public class QuickSearchValidationRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildValidationRunTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchConfigTombstone.java

@@ -0,0 +1,227 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for the quick_search_config_tombstone table: a table (fieldName
+ ** null) or one field the owner removed from Quick Search. Startup priming
+ ** never re-adds what a tombstone names, even when code still declares it;
+ ** deleting the tombstone lets code defaults seed it again.
+ *******************************************************************************/
+public class QuickSearchConfigTombstone extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quick_search_config_tombstone";
+
+   @QField()
+   private Integer id;
+   @QField()
+   private String tableName;
+   @QField()
+   private String fieldName;
+   @QField()
+   private String removedBy;
+   @QField()
+   private String reason;
+   @QField()
+   private Instant createDate;
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchConfigTombstone withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return (this.tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    *******************************************************************************/
+   public QuickSearchConfigTombstone withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldName
+    *******************************************************************************/
+   public String getFieldName()
+   {
+      return (this.fieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldName
+    *******************************************************************************/
+   public void setFieldName(String fieldName)
+   {
+      this.fieldName = fieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldName
+    *******************************************************************************/
+   public QuickSearchConfigTombstone withFieldName(String fieldName)
+   {
+      this.fieldName = fieldName;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for removedBy
+    *******************************************************************************/
+   public String getRemovedBy()
+   {
+      return (this.removedBy);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for removedBy
+    *******************************************************************************/
+   public void setRemovedBy(String removedBy)
+   {
+      this.removedBy = removedBy;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for removedBy
+    *******************************************************************************/
+   public QuickSearchConfigTombstone withRemovedBy(String removedBy)
+   {
+      this.removedBy = removedBy;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for reason
+    *******************************************************************************/
+   public String getReason()
+   {
+      return (this.reason);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for reason
+    *******************************************************************************/
+   public void setReason(String reason)
+   {
+      this.reason = reason;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reason
+    *******************************************************************************/
+   public QuickSearchConfigTombstone withReason(String reason)
+   {
+      this.reason = reason;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchConfigTombstone withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchDiagnosticEvent.java

@@ -0,0 +1,512 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ ** A Quick Search diagnostic event: a failed search (OpenSearch error or timeout,
+ ** after which core record search falls back to its own search), a diagnostic
+ ** tool run (the audit trail of the admin tools), or a refused run (rate limit).
+ ** Written by DiagnosticEventRecorder; read-only for operators.
+ *******************************************************************************/
+public class QuickSearchDiagnosticEvent extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quick_search_diagnostic_event";
+
+   /** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_TABLE_NAME = "quickSearchDiagnosticEvent";
+
+   public static final String TYPE_SEARCH_FAILED = "SEARCH_FAILED";
+   public static final String TYPE_TOOL_RUN = "TOOL_RUN";
+   public static final String TYPE_REINDEX = "REINDEX";
+   public static final String TYPE_RATE_LIMITED = "RATE_LIMITED";
+   public static final String SOURCE_CORE_RECORD_SEARCH = "CORE_RECORD_SEARCH";
+   public static final String SOURCE_QUICK_SEARCH_ACTION = "QUICK_SEARCH_ACTION";
+   public static final String OUTCOME_OK = "OK";
+   public static final String OUTCOME_ERROR = "ERROR";
+   public static final String OUTCOME_REFUSED = "REFUSED";
+
+   @QField()
+   private Integer id;
+
+   @QField()
+   private String eventType;
+
+   @QField()
+   private String source;
+
+   @QField()
+   private String outcome;
+
+   @QField()
+   private String userName;
+
+   @QField()
+   private String tableNames;
+
+   @QField()
+   private String searchTerm;
+
+   @QField()
+   private Integer durationMillis;
+
+   @QField()
+   private String errorClass;
+
+   @QField()
+   private String errorMessage;
+
+   @QField()
+   private String detail;
+
+   @QField()
+   private String hostName;
+
+   @QField()
+   private Instant createDate;
+
+
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor from a QRecord
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent(QRecord record) throws QException
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for eventType
+    *******************************************************************************/
+   public String getEventType()
+   {
+      return (this.eventType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for eventType
+    *******************************************************************************/
+   public void setEventType(String eventType)
+   {
+      this.eventType = eventType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for eventType
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withEventType(String eventType)
+   {
+      this.eventType = eventType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for source
+    *******************************************************************************/
+   public String getSource()
+   {
+      return (this.source);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for source
+    *******************************************************************************/
+   public void setSource(String source)
+   {
+      this.source = source;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for source
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withSource(String source)
+   {
+      this.source = source;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outcome
+    *******************************************************************************/
+   public String getOutcome()
+   {
+      return (this.outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outcome
+    *******************************************************************************/
+   public void setOutcome(String outcome)
+   {
+      this.outcome = outcome;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outcome
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withOutcome(String outcome)
+   {
+      this.outcome = outcome;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userName
+    *******************************************************************************/
+   public String getUserName()
+   {
+      return (this.userName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userName
+    *******************************************************************************/
+   public void setUserName(String userName)
+   {
+      this.userName = userName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userName
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withUserName(String userName)
+   {
+      this.userName = userName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableNames
+    *******************************************************************************/
+   public String getTableNames()
+   {
+      return (this.tableNames);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableNames
+    *******************************************************************************/
+   public void setTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableNames
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for searchTerm
+    *******************************************************************************/
+   public String getSearchTerm()
+   {
+      return (this.searchTerm);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for searchTerm
+    *******************************************************************************/
+   public void setSearchTerm(String searchTerm)
+   {
+      this.searchTerm = searchTerm;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for searchTerm
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withSearchTerm(String searchTerm)
+   {
+      this.searchTerm = searchTerm;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for durationMillis
+    *******************************************************************************/
+   public Integer getDurationMillis()
+   {
+      return (this.durationMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for durationMillis
+    *******************************************************************************/
+   public void setDurationMillis(Integer durationMillis)
+   {
+      this.durationMillis = durationMillis;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for durationMillis
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withDurationMillis(Integer durationMillis)
+   {
+      this.durationMillis = durationMillis;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorClass
+    *******************************************************************************/
+   public String getErrorClass()
+   {
+      return (this.errorClass);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorClass
+    *******************************************************************************/
+   public void setErrorClass(String errorClass)
+   {
+      this.errorClass = errorClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for errorClass
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withErrorClass(String errorClass)
+   {
+      this.errorClass = errorClass;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorMessage
+    *******************************************************************************/
+   public String getErrorMessage()
+   {
+      return (this.errorMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorMessage
+    *******************************************************************************/
+   public void setErrorMessage(String errorMessage)
+   {
+      this.errorMessage = errorMessage;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for errorMessage
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withErrorMessage(String errorMessage)
+   {
+      this.errorMessage = errorMessage;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for detail
+    *******************************************************************************/
+   public String getDetail()
+   {
+      return (this.detail);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for detail
+    *******************************************************************************/
+   public void setDetail(String detail)
+   {
+      this.detail = detail;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for detail
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withDetail(String detail)
+   {
+      this.detail = detail;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hostName
+    *******************************************************************************/
+   public String getHostName()
+   {
+      return (this.hostName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hostName
+    *******************************************************************************/
+   public void setHostName(String hostName)
+   {
+      this.hostName = hostName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hostName
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withHostName(String hostName)
+   {
+      this.hostName = hostName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchDocumentsChoice.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** What happens to a table's search documents when it is disabled or
+ ** removed: DELETE takes them out of the index now (they stop matching at
+ ** once); KEEP leaves them in the index. A disabled table's kept documents
+ ** are hidden from search and rebuilt when it is enabled again; a removed
+ ** table's are swept by the next all-tables reconcile.
+ *******************************************************************************/
+public enum QuickSearchDocumentsChoice implements PossibleValueEnum<String>
+{
+   DELETE("Delete the documents from the search index"),
+   KEEP("Keep the documents");
+
+   private final String label;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   QuickSearchDocumentsChoice(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** The value stored in the database (the constant's name).
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueId()
+   {
+      return (name());
+   }
+
+
+
+   /*******************************************************************************
+    ** The label shown in the UI.
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (label);
+   }
+
+
+
+   /*******************************************************************************
+    ** The constant with the given name, or the fallback when it is null or
+    ** unknown (a value written by a newer version, or by hand).
+    *******************************************************************************/
+   public static QuickSearchDocumentsChoice fromName(String value, QuickSearchDocumentsChoice fallback)
+   {
+      if(value != null)
+      {
+         for(QuickSearchDocumentsChoice candidate : values())
+         {
+            if(candidate.name().equalsIgnoreCase(value.trim()))
+            {
+               return (candidate);
+            }
+         }
+      }
+      return (fallback);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchFailedEvent.java

@@ -29,11 +29,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
  *******************************************************************************/
 public class QuickSearchFailedEvent extends QRecordEntity
 {
-   public static final String TABLE_NAME = "quickSearchFailedEvent";
+   public static final String TABLE_NAME = "quick_search_failed_event";
+
+   /** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_TABLE_NAME = "quickSearchFailedEvent";
 
    public static final String STATUS_PENDING   = "PENDING";
    public static final String STATUS_EXHAUSTED = "EXHAUSTED";
 
+   /** A change made while indexing was paused; becomes PENDING (replayed by basepull) when indexing resumes. */
+   public static final String STATUS_DEFERRED = "DEFERRED";
+
    ///////////////////////////////////////////////////////////////////////////
    // a delete made while a full reindex rebuilds into a new physical index; //
    // the reindex applies it after the swap, and basepull does not replay it //

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchFieldAnalyzer.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** How a searchable field's text is analyzed. PREFIX (the default) matches
+ ** word prefixes as you type; WORD matches whole words; ENGLISH matches English
+ ** word stems (run, running); EXACT matches the whole value, ignoring case.
+ *******************************************************************************/
+public enum QuickSearchFieldAnalyzer implements PossibleValueEnum<String>
+{
+   PREFIX("Prefix (as you type)"),
+   WORD("Whole words"),
+   ENGLISH("English (stemmed)"),
+   EXACT("Exact value");
+
+   private final String label;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   QuickSearchFieldAnalyzer(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** The value stored in the database (the constant's name).
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueId()
+   {
+      return (name());
+   }
+
+
+
+   /*******************************************************************************
+    ** The label shown in the UI.
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (label);
+   }
+
+
+
+   /*******************************************************************************
+    ** The constant with the given name, or the fallback when it is null or
+    ** unknown (a value written by a newer version, or by hand).
+    *******************************************************************************/
+   public static QuickSearchFieldAnalyzer fromName(String value, QuickSearchFieldAnalyzer fallback)
+   {
+      if(value != null)
+      {
+         for(QuickSearchFieldAnalyzer candidate : values())
+         {
+            if(candidate.name().equalsIgnoreCase(value.trim()))
+            {
+               return (candidate);
+            }
+         }
+      }
+      return (fallback);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchFieldRole.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** What a searchable field is for. TITLE fields make up the hit's label,
+ ** SUBTITLE fields its subtitle and DISPLAY (T1) fields the short details shown
+ ** with it; BODY is ordinary searchable text; KEYWORD is searchable but kept
+ ** out of the highlighted snippet; EXACT matches the whole value only.
+ *******************************************************************************/
+public enum QuickSearchFieldRole implements PossibleValueEnum<String>
+{
+   DISPLAY("T1 display"),
+   TITLE("Title"),
+   SUBTITLE("Subtitle"),
+   BODY("Body"),
+   KEYWORD("Keyword"),
+   EXACT("Exact");
+
+   private final String label;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   QuickSearchFieldRole(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** The value stored in the database (the constant's name).
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueId()
+   {
+      return (name());
+   }
+
+
+
+   /*******************************************************************************
+    ** The label shown in the UI.
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (label);
+   }
+
+
+
+   /*******************************************************************************
+    ** The constant with the given name, or the fallback when it is null or
+    ** unknown (a value written by a newer version, or by hand).
+    *******************************************************************************/
+   public static QuickSearchFieldRole fromName(String value, QuickSearchFieldRole fallback)
+   {
+      if(value != null)
+      {
+         for(QuickSearchFieldRole candidate : values())
+         {
+            if(candidate.name().equalsIgnoreCase(value.trim()))
+            {
+               return (candidate);
+            }
+         }
+      }
+      return (fallback);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndex.java

@@ -23,7 +23,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 
 
 /*******************************************************************************
- ** Entity bean for the quickSearchIndex table.
+ ** Entity bean for the quick_search_index table.
  **
  ** Tracks configuration for each table that participates in the quick search
  ** index, including basepull scheduling, searchable field definitions, and
@@ -31,7 +31,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
  *******************************************************************************/
 public class QuickSearchIndex extends QRecordEntity
 {
-   public static final String TABLE_NAME = "quickSearchIndex";
+   public static final String TABLE_NAME = "quick_search_index";
+
+   /** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_TABLE_NAME = "quickSearchIndex";
+
+   ///////////////////////////////////////////////////////////////////////
+   // source: who created the row. CODE rows were seeded from code      //
+   // defaults; OWNER rows were added by an admin. ownerEdited marks a  //
+   // row whose config an admin changed: priming never overwrites it.   //
+   ///////////////////////////////////////////////////////////////////////
+   public static final String SOURCE_CODE  = "CODE";
+   public static final String SOURCE_OWNER = "OWNER";
 
    @QField()
    private Integer id;
@@ -69,6 +81,12 @@ public class QuickSearchIndex extends QRecordEntity
    @QField()
    private Instant modifyDate;
 
+   @QField()
+   private String source;
+
+   @QField()
+   private Boolean ownerEdited;
+
 
 
    /*******************************************************************************
@@ -321,4 +339,65 @@ public class QuickSearchIndex extends QRecordEntity
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for source
+    *******************************************************************************/
+   public String getSource()
+   {
+      return (this.source);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for source
+    *******************************************************************************/
+   public void setSource(String source)
+   {
+      this.source = source;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for source
+    *******************************************************************************/
+   public QuickSearchIndex withSource(String source)
+   {
+      this.source = source;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for ownerEdited
+    *******************************************************************************/
+   public Boolean getOwnerEdited()
+   {
+      return (this.ownerEdited);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for ownerEdited
+    *******************************************************************************/
+   public void setOwnerEdited(Boolean ownerEdited)
+   {
+      this.ownerEdited = ownerEdited;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for ownerEdited
+    *******************************************************************************/
+   public QuickSearchIndex withOwnerEdited(Boolean ownerEdited)
+   {
+      this.ownerEdited = ownerEdited;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexField.java

@@ -0,0 +1,454 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for the quick_search_index_field table: one searchable field of
+ ** one quick_search_index row. These rows are the source of truth for what is
+ ** indexed and how it is boosted; code (@QuickSearchField, searchableTables)
+ ** only seeds them (see QuickSearchConfigPrimer for the precedence rules).
+ *******************************************************************************/
+public class QuickSearchIndexField extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quick_search_index_field";
+
+   public static final String SOURCE_CODE  = "CODE";
+   public static final String SOURCE_OWNER = "OWNER";
+
+   @QField()
+   private Integer id;
+   @QField()
+   private Integer quickSearchIndexId;
+   @QField()
+   private String fieldName;
+   @QField()
+   private Integer weight;
+   @QField()
+   private String role;
+   @QField()
+   private String analyzer;
+   @QField()
+   private Boolean includeInSnippet;
+   @QField()
+   private Boolean includeLabel;
+   @QField()
+   private Integer sortOrder;
+   @QField()
+   private String source;
+   @QField()
+   private Boolean ownerEdited;
+   @QField()
+   private Instant createDate;
+   @QField()
+   private Instant modifyDate;
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchIndexField withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for quickSearchIndexId
+    *******************************************************************************/
+   public Integer getQuickSearchIndexId()
+   {
+      return (this.quickSearchIndexId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for quickSearchIndexId
+    *******************************************************************************/
+   public void setQuickSearchIndexId(Integer quickSearchIndexId)
+   {
+      this.quickSearchIndexId = quickSearchIndexId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for quickSearchIndexId
+    *******************************************************************************/
+   public QuickSearchIndexField withQuickSearchIndexId(Integer quickSearchIndexId)
+   {
+      this.quickSearchIndexId = quickSearchIndexId;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldName
+    *******************************************************************************/
+   public String getFieldName()
+   {
+      return (this.fieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldName
+    *******************************************************************************/
+   public void setFieldName(String fieldName)
+   {
+      this.fieldName = fieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldName
+    *******************************************************************************/
+   public QuickSearchIndexField withFieldName(String fieldName)
+   {
+      this.fieldName = fieldName;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for weight
+    *******************************************************************************/
+   public Integer getWeight()
+   {
+      return (this.weight);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for weight
+    *******************************************************************************/
+   public void setWeight(Integer weight)
+   {
+      this.weight = weight;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for weight
+    *******************************************************************************/
+   public QuickSearchIndexField withWeight(Integer weight)
+   {
+      this.weight = weight;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for role
+    *******************************************************************************/
+   public String getRole()
+   {
+      return (this.role);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for role
+    *******************************************************************************/
+   public void setRole(String role)
+   {
+      this.role = role;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for role
+    *******************************************************************************/
+   public QuickSearchIndexField withRole(String role)
+   {
+      this.role = role;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for analyzer
+    *******************************************************************************/
+   public String getAnalyzer()
+   {
+      return (this.analyzer);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for analyzer
+    *******************************************************************************/
+   public void setAnalyzer(String analyzer)
+   {
+      this.analyzer = analyzer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for analyzer
+    *******************************************************************************/
+   public QuickSearchIndexField withAnalyzer(String analyzer)
+   {
+      this.analyzer = analyzer;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for includeInSnippet
+    *******************************************************************************/
+   public Boolean getIncludeInSnippet()
+   {
+      return (this.includeInSnippet);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for includeInSnippet
+    *******************************************************************************/
+   public void setIncludeInSnippet(Boolean includeInSnippet)
+   {
+      this.includeInSnippet = includeInSnippet;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for includeInSnippet
+    *******************************************************************************/
+   public QuickSearchIndexField withIncludeInSnippet(Boolean includeInSnippet)
+   {
+      this.includeInSnippet = includeInSnippet;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for includeLabel
+    *******************************************************************************/
+   public Boolean getIncludeLabel()
+   {
+      return (this.includeLabel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for includeLabel
+    *******************************************************************************/
+   public void setIncludeLabel(Boolean includeLabel)
+   {
+      this.includeLabel = includeLabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for includeLabel
+    *******************************************************************************/
+   public QuickSearchIndexField withIncludeLabel(Boolean includeLabel)
+   {
+      this.includeLabel = includeLabel;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for sortOrder
+    *******************************************************************************/
+   public Integer getSortOrder()
+   {
+      return (this.sortOrder);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sortOrder
+    *******************************************************************************/
+   public void setSortOrder(Integer sortOrder)
+   {
+      this.sortOrder = sortOrder;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sortOrder
+    *******************************************************************************/
+   public QuickSearchIndexField withSortOrder(Integer sortOrder)
+   {
+      this.sortOrder = sortOrder;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for source
+    *******************************************************************************/
+   public String getSource()
+   {
+      return (this.source);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for source
+    *******************************************************************************/
+   public void setSource(String source)
+   {
+      this.source = source;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for source
+    *******************************************************************************/
+   public QuickSearchIndexField withSource(String source)
+   {
+      this.source = source;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for ownerEdited
+    *******************************************************************************/
+   public Boolean getOwnerEdited()
+   {
+      return (this.ownerEdited);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for ownerEdited
+    *******************************************************************************/
+   public void setOwnerEdited(Boolean ownerEdited)
+   {
+      this.ownerEdited = ownerEdited;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for ownerEdited
+    *******************************************************************************/
+   public QuickSearchIndexField withOwnerEdited(Boolean ownerEdited)
+   {
+      this.ownerEdited = ownerEdited;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchIndexField withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDate
+    *******************************************************************************/
+   public void setModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDate
+    *******************************************************************************/
+   public QuickSearchIndexField withModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRun.java

@@ -23,7 +23,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 
 
 /*******************************************************************************
- ** Entity bean for the quickSearchIndexRun table.
+ ** Entity bean for the quick_search_index_run table.
  **
  ** Records a single execution of the indexing process for a given
  ** QuickSearchIndex, capturing run type, timing, record counts, and any errors
@@ -31,7 +31,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
  *******************************************************************************/
 public class QuickSearchIndexRun extends QRecordEntity
 {
-   public static final String TABLE_NAME = "quickSearchIndexRun";
+   public static final String TABLE_NAME = "quick_search_index_run";
+
+   /** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_TABLE_NAME = "quickSearchIndexRun";
 
    @QField()
    private Integer id;
@@ -69,6 +73,12 @@ public class QuickSearchIndexRun extends QRecordEntity
    @QField()
    private Instant modifyDate;
 
+   @QField()
+   private Boolean cancelRequested;
+
+   @QField()
+   private String cancelRequestedBy;
+
 
 
    /*******************************************************************************
@@ -321,4 +331,65 @@ public class QuickSearchIndexRun extends QRecordEntity
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for cancelRequested
+    *******************************************************************************/
+   public Boolean getCancelRequested()
+   {
+      return (this.cancelRequested);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for cancelRequested
+    *******************************************************************************/
+   public void setCancelRequested(Boolean cancelRequested)
+   {
+      this.cancelRequested = cancelRequested;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for cancelRequested
+    *******************************************************************************/
+   public QuickSearchIndexRun withCancelRequested(Boolean cancelRequested)
+   {
+      this.cancelRequested = cancelRequested;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for cancelRequestedBy
+    *******************************************************************************/
+   public String getCancelRequestedBy()
+   {
+      return (this.cancelRequestedBy);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for cancelRequestedBy
+    *******************************************************************************/
+   public void setCancelRequestedBy(String cancelRequestedBy)
+   {
+      this.cancelRequestedBy = cancelRequestedBy;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for cancelRequestedBy
+    *******************************************************************************/
+   public QuickSearchIndexRun withCancelRequestedBy(String cancelRequestedBy)
+   {
+      this.cancelRequestedBy = cancelRequestedBy;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchSettings.java

@@ -0,0 +1,356 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for the quick_search_settings table: the one row (id 1) of
+ ** system-level Quick Search settings. indexingPaused stops all indexing
+ ** (real-time and scheduled) while search keeps serving the existing index;
+ ** the priming fields report what startup self-priming last added.
+ *******************************************************************************/
+public class QuickSearchSettings extends QRecordEntity
+{
+   public static final String  TABLE_NAME  = "quick_search_settings";
+   public static final Integer SETTINGS_ID = 1;
+
+   @QField()
+   private Integer id;
+   @QField()
+   private Boolean indexingPaused;
+   @QField()
+   private Instant pausedAt;
+   @QField()
+   private String pausedBy;
+   @QField()
+   private String pauseReason;
+   @QField()
+   private Instant lastPrimedAt;
+   @QField()
+   private String lastPrimingSummary;
+   @QField()
+   private Integer lastPrimingNewTables;
+   @QField()
+   private Instant createDate;
+   @QField()
+   private Instant modifyDate;
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchSettings withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for indexingPaused
+    *******************************************************************************/
+   public Boolean getIndexingPaused()
+   {
+      return (this.indexingPaused);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for indexingPaused
+    *******************************************************************************/
+   public void setIndexingPaused(Boolean indexingPaused)
+   {
+      this.indexingPaused = indexingPaused;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for indexingPaused
+    *******************************************************************************/
+   public QuickSearchSettings withIndexingPaused(Boolean indexingPaused)
+   {
+      this.indexingPaused = indexingPaused;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for pausedAt
+    *******************************************************************************/
+   public Instant getPausedAt()
+   {
+      return (this.pausedAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for pausedAt
+    *******************************************************************************/
+   public void setPausedAt(Instant pausedAt)
+   {
+      this.pausedAt = pausedAt;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for pausedAt
+    *******************************************************************************/
+   public QuickSearchSettings withPausedAt(Instant pausedAt)
+   {
+      this.pausedAt = pausedAt;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for pausedBy
+    *******************************************************************************/
+   public String getPausedBy()
+   {
+      return (this.pausedBy);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for pausedBy
+    *******************************************************************************/
+   public void setPausedBy(String pausedBy)
+   {
+      this.pausedBy = pausedBy;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for pausedBy
+    *******************************************************************************/
+   public QuickSearchSettings withPausedBy(String pausedBy)
+   {
+      this.pausedBy = pausedBy;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for pauseReason
+    *******************************************************************************/
+   public String getPauseReason()
+   {
+      return (this.pauseReason);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for pauseReason
+    *******************************************************************************/
+   public void setPauseReason(String pauseReason)
+   {
+      this.pauseReason = pauseReason;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for pauseReason
+    *******************************************************************************/
+   public QuickSearchSettings withPauseReason(String pauseReason)
+   {
+      this.pauseReason = pauseReason;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for lastPrimedAt
+    *******************************************************************************/
+   public Instant getLastPrimedAt()
+   {
+      return (this.lastPrimedAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for lastPrimedAt
+    *******************************************************************************/
+   public void setLastPrimedAt(Instant lastPrimedAt)
+   {
+      this.lastPrimedAt = lastPrimedAt;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for lastPrimedAt
+    *******************************************************************************/
+   public QuickSearchSettings withLastPrimedAt(Instant lastPrimedAt)
+   {
+      this.lastPrimedAt = lastPrimedAt;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for lastPrimingSummary
+    *******************************************************************************/
+   public String getLastPrimingSummary()
+   {
+      return (this.lastPrimingSummary);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for lastPrimingSummary
+    *******************************************************************************/
+   public void setLastPrimingSummary(String lastPrimingSummary)
+   {
+      this.lastPrimingSummary = lastPrimingSummary;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for lastPrimingSummary
+    *******************************************************************************/
+   public QuickSearchSettings withLastPrimingSummary(String lastPrimingSummary)
+   {
+      this.lastPrimingSummary = lastPrimingSummary;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for lastPrimingNewTables
+    *******************************************************************************/
+   public Integer getLastPrimingNewTables()
+   {
+      return (this.lastPrimingNewTables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for lastPrimingNewTables
+    *******************************************************************************/
+   public void setLastPrimingNewTables(Integer lastPrimingNewTables)
+   {
+      this.lastPrimingNewTables = lastPrimingNewTables;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for lastPrimingNewTables
+    *******************************************************************************/
+   public QuickSearchSettings withLastPrimingNewTables(Integer lastPrimingNewTables)
+   {
+      this.lastPrimingNewTables = lastPrimingNewTables;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchSettings withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDate
+    *******************************************************************************/
+   public void setModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDate
+    *******************************************************************************/
+   public QuickSearchSettings withModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchValidationRun.java

@@ -0,0 +1,576 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ ** One run of the index validation: overall status, counts of failed and
+ ** warning checks, and the full report (HTML for the record view, JSON for
+ ** tools). Kept as history; purged after runHistoryRetentionDays.
+ *******************************************************************************/
+public class QuickSearchValidationRun extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quick_search_validation_run";
+
+   /** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
+   @Deprecated(since = "1.1.0-RC.3")
+   public static final String LEGACY_TABLE_NAME = "quickSearchValidationRun";
+
+   public static final String STATUS_PASS = "PASS";
+   public static final String STATUS_WARN = "WARN";
+   public static final String STATUS_FAIL = "FAIL";
+   public static final String STATUS_ERROR = "ERROR";
+   public static final String TRIGGER_MANUAL = "MANUAL";
+   public static final String TRIGGER_SCHEDULED = "SCHEDULED";
+
+   @QField()
+   private Integer id;
+
+   @QField()
+   private String status;
+
+   @QField()
+   private String triggerType;
+
+   @QField()
+   private String userName;
+
+   @QField()
+   private String tableNames;
+
+   @QField()
+   private Integer sampleSize;
+
+   @QField()
+   private Integer tablesChecked;
+
+   @QField()
+   private Integer failCount;
+
+   @QField()
+   private Integer warnCount;
+
+   @QField()
+   private String summary;
+
+   @QField()
+   private String reportHtml;
+
+   @QField()
+   private String reportJson;
+
+   @QField()
+   private Instant startTime;
+
+   @QField()
+   private Instant endTime;
+
+   @QField()
+   private Instant createDate;
+
+
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public QuickSearchValidationRun()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor from a QRecord
+    *******************************************************************************/
+   public QuickSearchValidationRun(QRecord record) throws QException
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchValidationRun withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for status
+    *******************************************************************************/
+   public String getStatus()
+   {
+      return (this.status);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for status
+    *******************************************************************************/
+   public void setStatus(String status)
+   {
+      this.status = status;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for status
+    *******************************************************************************/
+   public QuickSearchValidationRun withStatus(String status)
+   {
+      this.status = status;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for triggerType
+    *******************************************************************************/
+   public String getTriggerType()
+   {
+      return (this.triggerType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for triggerType
+    *******************************************************************************/
+   public void setTriggerType(String triggerType)
+   {
+      this.triggerType = triggerType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for triggerType
+    *******************************************************************************/
+   public QuickSearchValidationRun withTriggerType(String triggerType)
+   {
+      this.triggerType = triggerType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userName
+    *******************************************************************************/
+   public String getUserName()
+   {
+      return (this.userName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userName
+    *******************************************************************************/
+   public void setUserName(String userName)
+   {
+      this.userName = userName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userName
+    *******************************************************************************/
+   public QuickSearchValidationRun withUserName(String userName)
+   {
+      this.userName = userName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableNames
+    *******************************************************************************/
+   public String getTableNames()
+   {
+      return (this.tableNames);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableNames
+    *******************************************************************************/
+   public void setTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableNames
+    *******************************************************************************/
+   public QuickSearchValidationRun withTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sampleSize
+    *******************************************************************************/
+   public Integer getSampleSize()
+   {
+      return (this.sampleSize);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sampleSize
+    *******************************************************************************/
+   public void setSampleSize(Integer sampleSize)
+   {
+      this.sampleSize = sampleSize;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sampleSize
+    *******************************************************************************/
+   public QuickSearchValidationRun withSampleSize(Integer sampleSize)
+   {
+      this.sampleSize = sampleSize;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tablesChecked
+    *******************************************************************************/
+   public Integer getTablesChecked()
+   {
+      return (this.tablesChecked);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tablesChecked
+    *******************************************************************************/
+   public void setTablesChecked(Integer tablesChecked)
+   {
+      this.tablesChecked = tablesChecked;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tablesChecked
+    *******************************************************************************/
+   public QuickSearchValidationRun withTablesChecked(Integer tablesChecked)
+   {
+      this.tablesChecked = tablesChecked;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for failCount
+    *******************************************************************************/
+   public Integer getFailCount()
+   {
+      return (this.failCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for failCount
+    *******************************************************************************/
+   public void setFailCount(Integer failCount)
+   {
+      this.failCount = failCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for failCount
+    *******************************************************************************/
+   public QuickSearchValidationRun withFailCount(Integer failCount)
+   {
+      this.failCount = failCount;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for warnCount
+    *******************************************************************************/
+   public Integer getWarnCount()
+   {
+      return (this.warnCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for warnCount
+    *******************************************************************************/
+   public void setWarnCount(Integer warnCount)
+   {
+      this.warnCount = warnCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for warnCount
+    *******************************************************************************/
+   public QuickSearchValidationRun withWarnCount(Integer warnCount)
+   {
+      this.warnCount = warnCount;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for summary
+    *******************************************************************************/
+   public String getSummary()
+   {
+      return (this.summary);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for summary
+    *******************************************************************************/
+   public void setSummary(String summary)
+   {
+      this.summary = summary;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for summary
+    *******************************************************************************/
+   public QuickSearchValidationRun withSummary(String summary)
+   {
+      this.summary = summary;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reportHtml
+    *******************************************************************************/
+   public String getReportHtml()
+   {
+      return (this.reportHtml);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for reportHtml
+    *******************************************************************************/
+   public void setReportHtml(String reportHtml)
+   {
+      this.reportHtml = reportHtml;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reportHtml
+    *******************************************************************************/
+   public QuickSearchValidationRun withReportHtml(String reportHtml)
+   {
+      this.reportHtml = reportHtml;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reportJson
+    *******************************************************************************/
+   public String getReportJson()
+   {
+      return (this.reportJson);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for reportJson
+    *******************************************************************************/
+   public void setReportJson(String reportJson)
+   {
+      this.reportJson = reportJson;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reportJson
+    *******************************************************************************/
+   public QuickSearchValidationRun withReportJson(String reportJson)
+   {
+      this.reportJson = reportJson;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for startTime
+    *******************************************************************************/
+   public Instant getStartTime()
+   {
+      return (this.startTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for startTime
+    *******************************************************************************/
+   public void setStartTime(Instant startTime)
+   {
+      this.startTime = startTime;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for startTime
+    *******************************************************************************/
+   public QuickSearchValidationRun withStartTime(Instant startTime)
+   {
+      this.startTime = startTime;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for endTime
+    *******************************************************************************/
+   public Instant getEndTime()
+   {
+      return (this.endTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for endTime
+    *******************************************************************************/
+   public void setEndTime(Instant endTime)
+   {
+      this.endTime = endTime;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for endTime
+    *******************************************************************************/
+   public QuickSearchValidationRun withEndTime(Instant endTime)
+   {
+      this.endTime = endTime;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchValidationRun withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocument.java

@@ -20,6 +20,7 @@ package com.kingsrook.qbits.quicksearch.opensearch;
 import java.time.Instant;
 import java.util.Map;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 
 
 /*******************************************************************************
@@ -38,6 +39,21 @@ public class OpenSearchDocument
    private Instant indexedAt;
    private Map<String, Object> fieldValues;
 
+   ////////////////////////////////////////////////////////////////////////
+   // set only for tables whose fields have roles or non-default         //
+   // analyzers, so other documents are written exactly as before        //
+   ////////////////////////////////////////////////////////////////////////
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   private String subtitle;
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   private Map<String, String> displayValues;
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   private Map<String, Object> fieldValuesWord;
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   private Map<String, Object> fieldValuesEnglish;
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   private Map<String, Object> fieldValuesExact;
+
    private transient Long version;
 
 
@@ -212,4 +228,155 @@ public class OpenSearchDocument
       return (sourceTable + ":" + recordId);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for subtitle
+    *******************************************************************************/
+   public String getSubtitle()
+   {
+      return (this.subtitle);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for subtitle
+    *******************************************************************************/
+   public void setSubtitle(String subtitle)
+   {
+      this.subtitle = subtitle;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for subtitle
+    *******************************************************************************/
+   public OpenSearchDocument withSubtitle(String subtitle)
+   {
+      this.subtitle = subtitle;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for displayValues
+    *******************************************************************************/
+   public Map<String, String> getDisplayValues()
+   {
+      return (this.displayValues);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for displayValues
+    *******************************************************************************/
+   public void setDisplayValues(Map<String, String> displayValues)
+   {
+      this.displayValues = displayValues;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for displayValues
+    *******************************************************************************/
+   public OpenSearchDocument withDisplayValues(Map<String, String> displayValues)
+   {
+      this.displayValues = displayValues;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldValuesWord
+    *******************************************************************************/
+   public Map<String, Object> getFieldValuesWord()
+   {
+      return (this.fieldValuesWord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldValuesWord
+    *******************************************************************************/
+   public void setFieldValuesWord(Map<String, Object> fieldValuesWord)
+   {
+      this.fieldValuesWord = fieldValuesWord;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldValuesWord
+    *******************************************************************************/
+   public OpenSearchDocument withFieldValuesWord(Map<String, Object> fieldValuesWord)
+   {
+      this.fieldValuesWord = fieldValuesWord;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldValuesEnglish
+    *******************************************************************************/
+   public Map<String, Object> getFieldValuesEnglish()
+   {
+      return (this.fieldValuesEnglish);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldValuesEnglish
+    *******************************************************************************/
+   public void setFieldValuesEnglish(Map<String, Object> fieldValuesEnglish)
+   {
+      this.fieldValuesEnglish = fieldValuesEnglish;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldValuesEnglish
+    *******************************************************************************/
+   public OpenSearchDocument withFieldValuesEnglish(Map<String, Object> fieldValuesEnglish)
+   {
+      this.fieldValuesEnglish = fieldValuesEnglish;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldValuesExact
+    *******************************************************************************/
+   public Map<String, Object> getFieldValuesExact()
+   {
+      return (this.fieldValuesExact);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldValuesExact
+    *******************************************************************************/
+   public void setFieldValuesExact(Map<String, Object> fieldValuesExact)
+   {
+      this.fieldValuesExact = fieldValuesExact;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldValuesExact
+    *******************************************************************************/
+   public OpenSearchDocument withFieldValuesExact(Map<String, Object> fieldValuesExact)
+   {
+      this.fieldValuesExact = fieldValuesExact;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchDiagnosticsOperations.java

@@ -0,0 +1,566 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.opensearch;
+
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import jakarta.json.stream.JsonGenerator;
+import org.opensearch.client.json.JsonpMapper;
+import org.opensearch.client.json.JsonpSerializable;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch._types.OpenSearchException;
+import org.opensearch.client.opensearch._types.query_dsl.Query;
+import org.opensearch.client.opensearch.core.ExplainResponse;
+import org.opensearch.client.opensearch.core.SearchRequest;
+import org.opensearch.client.opensearch.core.SearchResponse;
+import org.opensearch.client.opensearch.core.search.Hit;
+import org.opensearch.client.opensearch.core.search.HighlightField;
+import org.opensearch.client.opensearch.core.search.MatchedQueries;
+import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
+import org.opensearch.client.opensearch.generic.Body;
+import org.opensearch.client.opensearch.generic.Requests;
+import org.opensearch.client.opensearch.generic.Response;
+
+
+/*******************************************************************************
+ ** Read-only OpenSearch operations behind the Quick Search admin diagnostics:
+ ** a probe search that reports scores, highlights and matched clauses with
+ ** its request and response JSON, explain for one document, document reads,
+ ** _analyze, random document sampling, and raw cluster / index / mapping /
+ ** settings reads. Nothing here writes to the cluster.
+ ** The probe uses exactly the query production search builds (same clauses,
+ ** operator, boosts and table filter); it only adds query names and
+ ** highlighting on the field values, which change neither matching nor
+ ** scoring.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsOperations
+{
+   public static final int MAX_ANALYZE_TEXT_LENGTH = 10_000;
+   public static final int MAX_PROBE_SIZE          = 100;
+   public static final int MAX_SAMPLE_SIZE         = 1_000;
+
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+   private final QuickSearchOpenSearchClient quickSearchClient;
+   private final OpenSearchClient            client;
+   private final String                      indexName;
+
+
+
+   /*******************************************************************************
+    ** One hit of a probe search.
+    *******************************************************************************/
+   public record ProbeHit(String recordId, String recordLabel, Double score, List<String> matchedClauses, Map<String, List<String>> highlights)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A probe search of one table: request and response JSON, server and
+    ** wall-clock time, total hits and the hits.
+    *******************************************************************************/
+   public record SearchProbe(String tableName, String requestJson, String responseJson, Long tookMillis, Long wallMillis, Long totalHits, Boolean totalIsLowerBound, List<ProbeHit> hits)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The result of explain for one document and the search query.
+    *******************************************************************************/
+   public record ExplainResult(Boolean documentFound, Boolean matched, Double score, String explanationText, String json)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One indexed document as stored: whether it exists, its version, the
+    ** physical index holding it and its source.
+    *******************************************************************************/
+   public record DocumentView(Boolean found, Long version, String physicalIndex, JsonNode source)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One token produced by _analyze.
+    *******************************************************************************/
+   public record AnalyzedToken(String token, Integer position, Integer startOffset, Integer endOffset, String type)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A raw JSON response: HTTP status and parsed body (null when empty).
+    *******************************************************************************/
+   public record RawResponse(Integer status, JsonNode body)
+   {
+      /*******************************************************************************
+       ** Whether the status is 2xx.
+       *******************************************************************************/
+      public boolean isOk()
+      {
+         return (status != null && status >= 200 && status < 300);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchDiagnosticsOperations(QuickSearchOpenSearchClient quickSearchClient)
+   {
+      this.quickSearchClient = quickSearchClient;
+      this.client            = quickSearchClient.getOpenSearchClient();
+      this.indexName         = quickSearchClient.getIndexName();
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for the alias name the operations read.
+    *******************************************************************************/
+   public String getIndexName()
+   {
+      return (indexName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the production search query against one table with query names and
+    ** field highlighting, returning up to size hits (capped at MAX_PROBE_SIZE).
+    *******************************************************************************/
+   public SearchProbe probeSearch(String term, String tableName, int size, List<QuickSearchableTableConfig> tableConfigs) throws QException
+   {
+      int           boundedSize = Math.max(1, Math.min(size, MAX_PROBE_SIZE));
+      Query         query       = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, true);
+      SearchRequest request     = SearchRequest.of(r -> r
+         .index(indexName)
+         .query(query)
+         .highlight(h -> h
+            .requireFieldMatch(true)
+            .fields("searchableText", HighlightField.of(hf -> hf))
+            .fields("fieldValues.*", HighlightField.of(hf -> hf)))
+         .trackTotalHits(t -> t.count(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW))
+         .size(boundedSize));
+
+      long start = System.nanoTime();
+      try
+      {
+         SearchResponse<OpenSearchDocument> response = client.search(request, OpenSearchDocument.class);
+         long                               wall     = (System.nanoTime() - start) / 1_000_000;
+
+         List<ProbeHit> hits = new ArrayList<>();
+         for(Hit<OpenSearchDocument> hit : response.hits().hits())
+         {
+            OpenSearchDocument source = hit.source();
+            hits.add(new ProbeHit(
+               source == null ? idFromDocumentId(hit.id()) : source.getRecordId(),
+               source == null ? null : source.getRecordLabel(),
+               hit.score(),
+               matchedClauses(hit.matchedQueries()),
+               hit.highlight() == null ? Map.of() : new LinkedHashMap<>(hit.highlight())));
+         }
+
+         Long    total      = response.hits().total() == null ? (long) hits.size() : response.hits().total().value();
+         Boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
+         return (new SearchProbe(tableName, toJson(request), toJson(response), response.took(), wall, total, lowerBound, hits));
+      }
+      catch(OpenSearchException | IOException e)
+      {
+         throw (wrap("Probe search failed for table [" + tableName + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Names of the clauses a hit matched (query names or, in newer servers,
+    ** the keys of the named-query scores).
+    *******************************************************************************/
+   static List<String> matchedClauses(MatchedQueries matchedQueries)
+   {
+      if(matchedQueries == null)
+      {
+         return (List.of());
+      }
+      if(matchedQueries.isScores())
+      {
+         return (new ArrayList<>(matchedQueries.scores().keySet()));
+      }
+      return (new ArrayList<>(matchedQueries.names()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Explain how the production query scores one document of a table. A
+    ** missing document gives documentFound false (not an error).
+    *******************************************************************************/
+   public ExplainResult explain(String term, String tableName, String recordId, List<QuickSearchableTableConfig> tableConfigs) throws QException
+   {
+      Query  query      = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, false);
+      String documentId = OpenSearchDocument.buildDocumentId(tableName, recordId);
+      try
+      {
+         ExplainResponse<OpenSearchDocument> response = client.explain(e -> e.index(indexName).id(documentId).query(query), OpenSearchDocument.class);
+         String   json        = toJson(response);
+         JsonNode tree        = MAPPER.readTree(json);
+         JsonNode explanation = tree.path("explanation");
+         Double   score       = explanation.isMissingNode() ? null : explanation.path("value").asDouble();
+         return (new ExplainResult(true, response.matched(), score, formatExplanation(explanation), json));
+      }
+      catch(OpenSearchException e)
+      {
+         if(e.status() == 404)
+         {
+            return (new ExplainResult(false, false, null, null, null));
+         }
+         throw (wrap("Explain failed for document [" + documentId + "]", e));
+      }
+      catch(IOException e)
+      {
+         throw (wrap("Explain failed for document [" + documentId + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An explanation tree as indented text ("value  description"), bounded to
+    ** 400 lines.
+    *******************************************************************************/
+   public static String formatExplanation(JsonNode explanation)
+   {
+      if(explanation == null || explanation.isMissingNode() || explanation.isNull())
+      {
+         return ("");
+      }
+      List<String> lines = new ArrayList<>();
+      appendExplanation(explanation, 0, lines);
+      return (String.join("\n", lines));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void appendExplanation(JsonNode node, int depth, List<String> lines)
+   {
+      if(lines.size() >= 400)
+      {
+         return;
+      }
+      lines.add("  ".repeat(depth) + String.format("%.4f", node.path("value").asDouble()) + "  " + node.path("description").asText(""));
+      for(JsonNode detail : node.path("details"))
+      {
+         appendExplanation(detail, depth + 1, lines);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Read one document of a table by record id.
+    *******************************************************************************/
+   public DocumentView getDocument(String tableName, String recordId) throws QException
+   {
+      RawResponse response = raw("GET", "/" + encode(indexName) + "/_doc/" + encode(OpenSearchDocument.buildDocumentId(tableName, recordId)), null);
+      if(response.status() == 404)
+      {
+         return (new DocumentView(false, null, null, null));
+      }
+      requireOk(response, "Document read failed");
+      JsonNode body = response.body();
+      return (new DocumentView(body.path("found").asBoolean(false), body.has("_version") ? body.get("_version").asLong() : null,
+         body.path("_index").asText(null), body.get("_source")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Read many documents of a table by record id: record id to source, for
+    ** the documents that exist.
+    *******************************************************************************/
+   public Map<String, JsonNode> getDocuments(String tableName, Collection<String> recordIds) throws QException
+   {
+      Map<String, JsonNode> sources = new LinkedHashMap<>();
+      if(recordIds == null || recordIds.isEmpty())
+      {
+         return (sources);
+      }
+
+      ObjectNode body = JsonNodeFactory.instance.objectNode();
+      ArrayNode  ids  = body.putArray("ids");
+      for(String recordId : recordIds)
+      {
+         ids.add(OpenSearchDocument.buildDocumentId(tableName, recordId));
+      }
+
+      RawResponse response = raw("POST", "/" + encode(indexName) + "/_mget", body.toString());
+      requireOk(response, "Document read failed");
+      for(JsonNode doc : response.body().path("docs"))
+      {
+         if(doc.path("found").asBoolean(false) && doc.has("_source"))
+         {
+            sources.put(idFromDocumentId(doc.path("_id").asText()), doc.get("_source"));
+         }
+      }
+      return (sources);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run _analyze on the index with a named analyzer (an index analyzer or a
+    ** built-in one). Text is truncated to MAX_ANALYZE_TEXT_LENGTH.
+    *******************************************************************************/
+   public List<AnalyzedToken> analyze(String analyzer, String text) throws QException
+   {
+      String     bounded = text == null ? "" : (text.length() > MAX_ANALYZE_TEXT_LENGTH ? text.substring(0, MAX_ANALYZE_TEXT_LENGTH) : text);
+      ObjectNode body    = JsonNodeFactory.instance.objectNode().put("analyzer", analyzer).put("text", bounded);
+
+      RawResponse response = raw("POST", "/" + encode(indexName) + "/_analyze", body.toString());
+      requireOk(response, "Analyze failed for analyzer [" + analyzer + "]");
+
+      List<AnalyzedToken> tokens = new ArrayList<>();
+      for(JsonNode token : response.body().path("tokens"))
+      {
+         tokens.add(new AnalyzedToken(token.path("token").asText(), token.path("position").asInt(), token.path("start_offset").asInt(),
+            token.path("end_offset").asInt(), token.path("type").asText(null)));
+      }
+      return (tokens);
+   }
+
+
+
+   /*******************************************************************************
+    ** Up to n record ids of a table's documents, chosen at random.
+    *******************************************************************************/
+   public List<String> sampleRecordIds(String tableName, int n) throws QException
+   {
+      int        size  = Math.max(1, Math.min(n, MAX_SAMPLE_SIZE));
+      ObjectNode body  = JsonNodeFactory.instance.objectNode();
+      body.put("size", size);
+      body.putArray("_source").add("recordId");
+      ObjectNode functionScore = body.putObject("query").putObject("function_score");
+      functionScore.putObject("query").putObject("term").put("sourceTable", tableName);
+      functionScore.putObject("random_score");
+
+      RawResponse response = raw("POST", "/" + encode(indexName) + "/_search", body.toString());
+      requireOk(response, "Sampling documents failed for table [" + tableName + "]");
+
+      List<String> recordIds = new ArrayList<>();
+      for(JsonNode hit : response.body().path("hits").path("hits"))
+      {
+         String recordId = hit.path("_source").path("recordId").asText(null);
+         recordIds.add(recordId != null ? recordId : idFromDocumentId(hit.path("_id").asText()));
+      }
+      return (recordIds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health, with the alias's indexes (level=indices).
+    *******************************************************************************/
+   public RawResponse clusterHealth() throws QException
+   {
+      return (raw("GET", "/_cluster/health/" + encode(indexName) + "?level=indices", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The live mapping of the alias's index(es).
+    *******************************************************************************/
+   public RawResponse mapping() throws QException
+   {
+      return (raw("GET", "/" + encode(indexName) + "/_mapping", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The live settings of the alias's index(es).
+    *******************************************************************************/
+   public RawResponse settings() throws QException
+   {
+      return (raw("GET", "/" + encode(indexName) + "/_settings", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The physical indexes named like this alias's ({alias}-v*), with health,
+    ** status, document counts and size, from _cat/indices.
+    *******************************************************************************/
+   public RawResponse physicalIndexes() throws QException
+   {
+      return (raw("GET", "/_cat/indices/" + encode(indexName) + "-v*?format=json&h=index,health,status,docs.count,docs.deleted,store.size&expand_wildcards=all", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The record id part of a document id ("table:id").
+    *******************************************************************************/
+   static String idFromDocumentId(String documentId)
+   {
+      if(documentId == null)
+      {
+         return (null);
+      }
+      int colon = documentId.indexOf(':');
+      return (colon < 0 ? documentId : documentId.substring(colon + 1));
+   }
+
+
+
+   /*******************************************************************************
+    ** Send a raw request and parse the JSON body. HTTP errors are returned
+    ** (status and body), not thrown; transport errors throw.
+    *******************************************************************************/
+   public RawResponse raw(String method, String endpoint, String jsonBody) throws QException
+   {
+      Requests.JsonBodyBuilder builder = Requests.builder().method(method).endpoint(endpoint);
+      if(jsonBody != null)
+      {
+         builder.json(jsonBody);
+      }
+
+      try(Response response = client.generic().execute(builder.build()))
+      {
+         JsonNode body = null;
+         if(response.getBody().isPresent())
+         {
+            try(Body responseBody = response.getBody().get())
+            {
+               String text = responseBody.bodyAsString();
+               body = text == null || text.isBlank() ? null : MAPPER.readTree(text);
+            }
+         }
+         return (new RawResponse(response.getStatus(), body));
+      }
+      catch(OpenSearchException | IOException e)
+      {
+         throw (wrap(method + " " + endpoint.replaceAll("\\?.*", "") + " failed", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Throw a QException with OpenSearch's reason when a raw response is not
+    ** 2xx.
+    *******************************************************************************/
+   public static void requireOk(RawResponse response, String message) throws QException
+   {
+      if(!response.isOk())
+      {
+         throw (new QException(message + ": OpenSearch returned status " + response.status() + ": " + errorReason(response.body())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** OpenSearch's error reason from an error body, or a short form of it.
+    *******************************************************************************/
+   public static String errorReason(JsonNode body)
+   {
+      if(body == null)
+      {
+         return ("no response body");
+      }
+      JsonNode error = body.path("error");
+      if(error.isTextual())
+      {
+         return (error.asText());
+      }
+      String reason = error.path("reason").asText(null);
+      if(reason != null)
+      {
+         return (reason);
+      }
+      String text = body.toString();
+      return (text.length() > 300 ? text.substring(0, 300) + "..." : text);
+   }
+
+
+
+   /*******************************************************************************
+    ** Serialize a request or response with the transport's mapper (which knows
+    ** OpenSearchDocument; the default mapper of toJsonString does not).
+    *******************************************************************************/
+   private String toJson(JsonpSerializable value)
+   {
+      JsonpMapper  mapper = client._transport().jsonpMapper();
+      StringWriter writer = new StringWriter();
+      try(JsonGenerator generator = mapper.jsonProvider().createGenerator(writer))
+      {
+         value.serialize(generator, mapper);
+      }
+      return (writer.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** URL-encode one path segment.
+    *******************************************************************************/
+   static String encode(String segment)
+   {
+      return (URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20"));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QException wrap(String message, Exception e)
+   {
+      if(e instanceof OpenSearchException openSearchException)
+      {
+         String reason = openSearchException.error() == null ? openSearchException.getMessage() : openSearchException.error().reason();
+         return (new QException(message + ": OpenSearch returned status " + openSearchException.status() + ": " + reason, e));
+      }
+      return (new QException(message + ": " + e.getMessage(), e));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java

@@ -21,9 +21,14 @@ import java.io.Closeable;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
@@ -31,7 +36,10 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
 import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch.OpenSearchAsyncClient;
 import org.opensearch.client.opensearch.OpenSearchClient;
 import org.opensearch.client.opensearch._types.Conflicts;
 import org.opensearch.client.opensearch._types.FieldValue;
@@ -51,12 +59,17 @@ import org.opensearch.client.opensearch.core.DeleteByQueryRequest;
 import org.opensearch.client.opensearch.core.DeleteByQueryResponse;
 import org.opensearch.client.opensearch.core.DeleteRequest;
 import org.opensearch.client.opensearch.core.IndexRequest;
+import org.opensearch.client.opensearch.core.MsearchRequest;
+import org.opensearch.client.opensearch.core.MsearchResponse;
 import org.opensearch.client.opensearch.core.SearchRequest;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.opensearch.client.opensearch.core.bulk.BulkOperation;
 import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
 import org.opensearch.client.opensearch.core.bulk.DeleteOperation;
 import org.opensearch.client.opensearch.core.bulk.IndexOperation;
+import org.opensearch.client.opensearch.core.msearch.MultiSearchResponseItem;
+import org.opensearch.client.opensearch.core.msearch.RequestItem;
+import org.opensearch.client.opensearch.core.search.Hit;
 import org.opensearch.client.opensearch.core.search.Highlight;
 import org.opensearch.client.opensearch.core.search.HighlightField;
 import org.opensearch.client.opensearch.indices.CreateIndexRequest;
@@ -91,14 +104,23 @@ public class QuickSearchOpenSearchClient implements Closeable
    public static final String INDEX_ANALYZER       = "quick_search_analyzer";
    public static final String SEARCH_ANALYZER      = "quick_search_search_analyzer";
    public static final int    MAX_RESULT_WINDOW    = 10_000;
+   public static final String MATCH_CLAUSE_NAME    = "searchableText";
+   public static final String FIELD_CLAUSE_PREFIX  = "field:";
+   public static final String PATH_PREFIX          = "fieldValues";
+   public static final String PATH_WORD            = "fieldValuesWord";
+   public static final String PATH_ENGLISH         = "fieldValuesEnglish";
+   public static final String PATH_EXACT           = "fieldValuesExact";
 
-   private final OpenSearchClient    client;
-   private final OpenSearchTransport transport;
-   private final String              indexName;
-   private final int                 maxBulkRequestBytes;
-   private final ObjectMapper        sizingMapper;
+   private final OpenSearchClient      client;
+   private final OpenSearchAsyncClient asyncClient;
+   private final OpenSearchTransport   transport;
+   private final String                indexName;
+   private final int                   maxBulkRequestBytes;
+   private final ObjectMapper          sizingMapper;
 
    private volatile Boolean mappingOutdated = false;
+   private volatile Boolean indexCreated    = false;
+   private volatile Instant indexCreatedAt;
    private          Boolean closed          = false;
 
 
@@ -110,6 +132,7 @@ public class QuickSearchOpenSearchClient implements Closeable
    {
       this.transport           = OpenSearchTransportFactory.build(config);
       this.client              = new OpenSearchClient(transport);
+      this.asyncClient         = new OpenSearchAsyncClient(transport);
       this.indexName           = config.getOpensearchIndexName();
       this.maxBulkRequestBytes = config.getMaxBulkRequestBytes() == null ? 5 * 1024 * 1024 : config.getMaxBulkRequestBytes();
       this.sizingMapper        = new ObjectMapper().registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
@@ -137,6 +160,7 @@ public class QuickSearchOpenSearchClient implements Closeable
             else
             {
                LOG.info("OpenSearch index already exists", logPair("indexName", indexName));
+               extendMapping();
             }
             return;
          }
@@ -145,6 +169,8 @@ public class QuickSearchOpenSearchClient implements Closeable
          createPhysicalIndex(physicalIndex);
          client.indices().updateAliases(u -> u.actions(a -> a.add(ad -> ad.index(physicalIndex).alias(indexName))));
          mappingOutdated = false;
+         indexCreated    = true;
+         indexCreatedAt  = Instant.now();
          LOG.info("Created OpenSearch index", logPair("indexName", indexName), logPair("physicalIndex", physicalIndex));
       }
       catch(OpenSearchException | java.io.IOException e)
@@ -155,6 +181,101 @@ public class QuickSearchOpenSearchClient implements Closeable
 
 
 
+   /*******************************************************************************
+    ** Whether ensureIndexExists found no index and created an empty one (a new
+    ** install, or the index was deleted). Self-priming then queues a build of
+    ** every enabled table. Cleared by clearIndexCreated().
+    *******************************************************************************/
+   public boolean wasIndexCreated()
+   {
+      return (Boolean.TRUE.equals(indexCreated));
+   }
+
+
+
+   /*******************************************************************************
+    ** When ensureIndexExists last created the index (null when it has not).
+    *******************************************************************************/
+   public Instant getIndexCreatedAt()
+   {
+      return (indexCreatedAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget that the index was created (its builds have been queued).
+    *******************************************************************************/
+   public void clearIndexCreated()
+   {
+      indexCreated = false;
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the alias (or a legacy concrete index) exists. Throws when the
+    ** cluster cannot be asked.
+    *******************************************************************************/
+   public boolean indexExists() throws QException
+   {
+      try
+      {
+         return (client.indices().exists(ExistsRequest.of(r -> r.index(indexName))).value());
+      }
+      catch(OpenSearchException | java.io.IOException e)
+      {
+         throw wrap("Failed to check whether index [" + indexName + "] exists", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Number of documents in the index, all tables.
+    *******************************************************************************/
+   public Long countAllDocuments() throws QException
+   {
+      try
+      {
+         return (client.count(c -> c.index(indexName)).count());
+      }
+      catch(OpenSearchException | java.io.IOException e)
+      {
+         throw wrap("Failed to count documents in index [" + indexName + "]", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Add the field-analyzer templates and the role properties (added in
+    ** 1.1.0-RC.3) to an index created before them. Additive, so the mapping
+    ** version stays 2 and no rebuild is needed; a failure is logged and only
+    ** means those fields fall back to dynamic mapping.
+    *******************************************************************************/
+   void extendMapping()
+   {
+      try
+      {
+         client.indices().putMapping(p -> p
+            .index(indexName)
+            .dynamicTemplates(dynamicTemplates())
+            .properties("subtitle", pr -> pr.text(t -> t.index(false)))
+            .properties("displayValues", pr -> pr.object(o -> o.enabled(false)))
+            .properties(PATH_WORD, pr -> pr.object(o -> o))
+            .properties(PATH_ENGLISH, pr -> pr.object(o -> o))
+            .properties(PATH_EXACT, pr -> pr.object(o -> o)));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not add the field-analyzer mapping to the existing index; WORD, ENGLISH and EXACT fields use dynamic mapping until a full reindex",
+            e, logPair("indexName", indexName));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Whether the live index was created with an older mapping than this
     ** client writes. A full reindex of all tables rebuilds it.
@@ -205,15 +326,16 @@ public class QuickSearchOpenSearchClient implements Closeable
     *******************************************************************************/
    private TypeMapping buildMapping()
    {
-      DynamicTemplate fieldValuesAsText = DynamicTemplate.of(t -> t
-         .pathMatch("fieldValues.*")
-         .mapping(p -> p.text(tx -> tx.analyzer(INDEX_ANALYZER).searchAnalyzer(SEARCH_ANALYZER))));
-
       return (TypeMapping.of(m -> m
          .meta(META_MAPPING_VERSION, JsonData.of(MAPPING_VERSION))
          .dateDetection(false)
          .numericDetection(false)
-         .dynamicTemplates(Map.of("fieldValuesAsText", fieldValuesAsText))
+         .dynamicTemplates(dynamicTemplates())
+         .properties("subtitle", p -> p.text(t -> t.index(false)))
+         .properties("displayValues", p -> p.object(o -> o.enabled(false)))
+         .properties(PATH_WORD, p -> p.object(o -> o))
+         .properties(PATH_ENGLISH, p -> p.object(o -> o))
+         .properties(PATH_EXACT, p -> p.object(o -> o))
          .properties("sourceTable", p -> p.keyword(k -> k))
          .properties("recordId", p -> p.keyword(k -> k))
          .properties("recordLabel", p -> p.text(t -> t.analyzer(INDEX_ANALYZER).searchAnalyzer(SEARCH_ANALYZER)))
@@ -224,6 +346,27 @@ public class QuickSearchOpenSearchClient implements Closeable
 
 
 
+   /*******************************************************************************
+    ** The dynamic templates: fieldValues.* with the prefix analyzers (as since
+    ** mapping version 2), and the field-analyzer copies: fieldValuesWord.*
+    ** (standard), fieldValuesEnglish.* (english) and fieldValuesExact.*
+    ** (keyword; values are lower-cased before indexing).
+    *******************************************************************************/
+   static List<Map<String, DynamicTemplate>> dynamicTemplates()
+   {
+      return (List.of(
+         Map.of("fieldValuesAsText", DynamicTemplate.of(t -> t.pathMatch(PATH_PREFIX + ".*")
+            .mapping(p -> p.text(tx -> tx.analyzer(INDEX_ANALYZER).searchAnalyzer(SEARCH_ANALYZER))))),
+         Map.of("fieldValuesWordAsText", DynamicTemplate.of(t -> t.pathMatch(PATH_WORD + ".*")
+            .mapping(p -> p.text(tx -> tx.analyzer("standard"))))),
+         Map.of("fieldValuesEnglishAsText", DynamicTemplate.of(t -> t.pathMatch(PATH_ENGLISH + ".*")
+            .mapping(p -> p.text(tx -> tx.analyzer("english"))))),
+         Map.of("fieldValuesExactAsKeyword", DynamicTemplate.of(t -> t.pathMatch(PATH_EXACT + ".*")
+            .mapping(p -> p.keyword(k -> k.ignoreAbove(1024)))))));
+   }
+
+
+
    /*******************************************************************************
     ** Read quickSearchMappingVersion from the live mapping's _meta; 1 when
     ** absent (a pre-1.0 index).
@@ -652,14 +795,7 @@ public class QuickSearchOpenSearchClient implements Closeable
    {
       try
       {
-         List<FieldValue> allowed = allowedTables.stream().map(FieldValue::of).toList();
-
-         BoolQuery.Builder boolBuilder = new BoolQuery.Builder()
-            .must(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm -> mm.query(searchTerm).fields("searchableText").operator(Operator.And)))))
-            .filter(Query.of(q -> q.terms(t -> t.field("sourceTable").terms(tv -> tv.value(allowed)))))
-            .should(buildFieldBoostQueries(searchTerm, tableConfigs, allowedTables));
-
-         Query     finalQuery = Query.of(q -> q.bool(boolBuilder.build()));
+         Query     finalQuery = buildSearchQuery(searchTerm, allowedTables, tableConfigs);
          Highlight highlight  = Highlight.of(h -> h.fields("searchableText", HighlightField.of(hf -> hf)));
 
          SearchRequest request = SearchRequest.of(r -> r
@@ -681,10 +817,199 @@ public class QuickSearchOpenSearchClient implements Closeable
 
 
 
+   /*******************************************************************************
+    ** Record ids of the best matches in each table, best first, for core record
+    ** search. One _msearch request with one search per table (the same query and
+    ** ranking as {@link #search}, restricted to that table), so every table gets
+    ** its own top idsPerTable hits. Only sourceTable and recordId are fetched.
+    **
+    ** The whole request is bounded by timeoutMillis of wall-clock time (and the
+    ** same server-side search timeout): when it runs out, the request is
+    ** cancelled and a QException is thrown. A failed search for any table fails
+    ** the call. Returns every requested table as a key, in the given order.
+    *******************************************************************************/
+   public Map<String, List<String>> searchRecordIdsPerTable(String searchTerm, List<String> tableNames, int idsPerTable, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
+   {
+      Map<String, List<String>> idsByTable = new LinkedHashMap<>();
+      if(tableNames == null || tableNames.isEmpty())
+      {
+         return (idsByTable);
+      }
+
+      List<RequestItem> searches = new ArrayList<>();
+      for(String tableName : tableNames)
+      {
+         Query query = buildSearchQuery(searchTerm, List.of(tableName), tableConfigs);
+         searches.add(RequestItem.of(ri -> ri
+            .header(h -> h.index(indexName))
+            .body(b -> b
+               .query(query)
+               .size(idsPerTable)
+               .trackTotalHits(t -> t.enabled(false))
+               .source(sc -> sc.filter(f -> f.includes("sourceTable", "recordId")))
+               .timeout(timeoutMillis + "ms"))));
+      }
+
+      CompletableFuture<MsearchResponse<OpenSearchDocument>> future = null;
+      try
+      {
+         future = asyncClient.msearch(MsearchRequest.of(r -> r.searches(searches)), OpenSearchDocument.class);
+         MsearchResponse<OpenSearchDocument> response = future.get(timeoutMillis, TimeUnit.MILLISECONDS);
+
+         List<MultiSearchResponseItem<OpenSearchDocument>> items = response.responses();
+         if(items == null || items.size() != tableNames.size())
+         {
+            throw (new QException("Record search failed: OpenSearch returned " + (items == null ? 0 : items.size()) + " responses for " + tableNames.size() + " searches"));
+         }
+
+         for(int i = 0; i < tableNames.size(); i++)
+         {
+            MultiSearchResponseItem<OpenSearchDocument> item = items.get(i);
+            if(item.isFailure())
+            {
+               String reason = item.failure() != null && item.failure().error() != null ? item.failure().error().reason() : "unknown error";
+               throw (new QException("Record search failed for table [" + tableNames.get(i) + "]: " + reason));
+            }
+
+            List<String> ids = new ArrayList<>();
+            for(Hit<OpenSearchDocument> hit : item.result().hits().hits())
+            {
+               if(hit.source() != null && hit.source().getRecordId() != null)
+               {
+                  ids.add(hit.source().getRecordId());
+               }
+            }
+            idsByTable.put(tableNames.get(i), ids);
+         }
+         return (idsByTable);
+      }
+      catch(TimeoutException e)
+      {
+         future.cancel(true);
+         throw (new QException("Record search timed out after " + timeoutMillis + " ms", e));
+      }
+      catch(InterruptedException e)
+      {
+         if(future != null)
+         {
+            future.cancel(true);
+         }
+         Thread.currentThread().interrupt();
+         throw (new QException("Record search was interrupted", e));
+      }
+      catch(ExecutionException e)
+      {
+         throw wrap("Record search failed", e.getCause() instanceof Exception cause ? cause : e);
+      }
+      catch(OpenSearchException | java.io.IOException e)
+      {
+         throw wrap("Record search failed", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The search query: every term of the query (operator AND) must match
+    ** searchableText, only the allowed tables match (an empty set matches
+    ** nothing), and per-field should clauses boost by configured weight.
+    *******************************************************************************/
+   private Query buildSearchQuery(String searchTerm, Collection<String> allowedTables, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      return (buildSearchQuery(searchTerm, allowedTables, tableConfigs, false));
+   }
+
+
+
+   /*******************************************************************************
+    ** The search query; with named true each clause carries a query name
+    ** (MATCH_CLAUSE_NAME for the required searchableText match, the field name
+    ** for each boost clause) so a hit reports which clauses matched. Names do
+    ** not change matching or scoring.
+    *******************************************************************************/
+   Query buildSearchQuery(String searchTerm, Collection<String> allowedTables, List<QuickSearchableTableConfig> tableConfigs, boolean named)
+   {
+      List<FieldValue> allowed = allowedTables.stream().map(FieldValue::of).toList();
+
+      Query snippetMatch = Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
+      {
+         mm.query(searchTerm).fields("searchableText").operator(Operator.And);
+         return (named ? mm.queryName(MATCH_CLAUSE_NAME) : mm);
+      })));
+
+      ////////////////////////////////////////////////////////////////////////////
+      // fields kept out of the snippet (KEYWORD, EXACT, or includeInSnippet   //
+      // off) match on their own; with none, the query is exactly as before    //
+      ////////////////////////////////////////////////////////////////////////////
+      List<Query> ownMatches = buildOwnFieldMatchQueries(searchTerm, tableConfigs, allowedTables);
+      Query       match      = ownMatches.isEmpty() ? snippetMatch
+         : Query.of(q -> q.bool(b -> b.should(snippetMatch).should(ownMatches).minimumShouldMatch("1")));
+
+      BoolQuery.Builder boolBuilder = new BoolQuery.Builder()
+         .must(match)
+         .filter(Query.of(q -> q.terms(t -> t.field("sourceTable").terms(tv -> tv.value(allowed)))))
+         .should(buildFieldBoostQueries(searchTerm, tableConfigs, allowedTables, named));
+
+      return (Query.of(q -> q.bool(boolBuilder.build())));
+   }
+
+
+
+   /*******************************************************************************
+    ** Matches for the fields that are not in searchableText, each limited to
+    ** its own table: an EXACT field matches the whole (lower-cased) term, any
+    ** other matches every term with its analyzer.
+    *******************************************************************************/
+   private List<Query> buildOwnFieldMatchQueries(String searchTerm, List<QuickSearchableTableConfig> tableConfigs, Collection<String> allowedTables)
+   {
+      List<Query> queries = new ArrayList<>();
+      for(QuickSearchableTableConfig tableConfig : tableConfigs == null ? List.<QuickSearchableTableConfig>of() : tableConfigs)
+      {
+         if(!allowedTables.contains(tableConfig.getTableName()) || tableConfig.isAllPlainFields())
+         {
+            continue;
+         }
+
+         Query inTable = Query.of(q -> q.term(t -> t.field("sourceTable").value(FieldValue.of(tableConfig.getTableName()))));
+         for(String field : tableConfig.getSearchableFields() == null ? List.<String>of() : tableConfig.getSearchableFields())
+         {
+            if(tableConfig.isIncludedInSnippet(field))
+            {
+               continue;
+            }
+
+            QuickSearchFieldAnalyzer analyzer = tableConfig.getAnalyzer(field);
+            Query fieldMatch = analyzer == QuickSearchFieldAnalyzer.EXACT
+               ? Query.of(q -> q.term(t -> t.field(PATH_EXACT + "." + field).value(FieldValue.of(IndexingUtils.normalizeExact(searchTerm)))))
+               : Query.of(q -> q.multiMatch(mm -> mm.query(searchTerm).fields(fieldPath(analyzer) + "." + field).operator(Operator.And).lenient(true)));
+            queries.add(Query.of(q -> q.bool(b -> b.filter(inTable).must(fieldMatch))));
+         }
+      }
+      return (queries);
+   }
+
+
+
+   /*******************************************************************************
+    ** The document object a field's analyzer writes it to.
+    *******************************************************************************/
+   static String fieldPath(QuickSearchFieldAnalyzer analyzer)
+   {
+      return (switch(analyzer == null ? QuickSearchFieldAnalyzer.PREFIX : analyzer)
+      {
+         case WORD -> PATH_WORD;
+         case ENGLISH -> PATH_ENGLISH;
+         case EXACT -> PATH_EXACT;
+         default -> PATH_PREFIX;
+      });
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<Query> buildFieldBoostQueries(String searchTerm, List<QuickSearchableTableConfig> tableConfigs, Collection<String> allowedTables)
+   private List<Query> buildFieldBoostQueries(String searchTerm, List<QuickSearchableTableConfig> tableConfigs, Collection<String> allowedTables, boolean named)
    {
       List<Query> queries = new ArrayList<>();
       if(tableConfigs == null)
@@ -703,13 +1028,29 @@ public class QuickSearchOpenSearchClient implements Closeable
          List<String>         fields  = tableConfig.getSearchableFields() == null ? List.of() : tableConfig.getSearchableFields();
          for(String field : fields)
          {
-            float boost = (weights == null || weights.get(field) == null) ? 1f : weights.get(field).floatValue();
-            queries.add(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm -> mm
-               .query(searchTerm)
-               .fields("fieldValues." + field)
-               .operator(Operator.And)
-               .lenient(true)
-               .boost(boost)))));
+            float                    boost    = (weights == null || weights.get(field) == null) ? 1f : weights.get(field).floatValue();
+            QuickSearchFieldAnalyzer analyzer = tableConfig.getAnalyzer(field);
+            if(analyzer == QuickSearchFieldAnalyzer.EXACT)
+            {
+               String exact = IndexingUtils.normalizeExact(searchTerm);
+               queries.add(Query.of(q -> q.term(t ->
+               {
+                  t.field(PATH_EXACT + "." + field).value(FieldValue.of(exact)).boost(boost);
+                  return (named ? t.queryName(FIELD_CLAUSE_PREFIX + field) : t);
+               })));
+               continue;
+            }
+
+            String path = fieldPath(analyzer) + "." + field;
+            queries.add(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
+            {
+               mm.query(searchTerm)
+                  .fields(path)
+                  .operator(Operator.And)
+                  .lenient(true)
+                  .boost(boost);
+               return (named ? mm.queryName(FIELD_CLAUSE_PREFIX + field) : mm);
+            }))));
          }
       }
 
@@ -752,6 +1093,17 @@ public class QuickSearchOpenSearchClient implements Closeable
 
 
 
+   /*******************************************************************************
+    ** The underlying OpenSearch client, for the read-only diagnostics
+    ** (QuickSearchDiagnosticsOperations).
+    *******************************************************************************/
+   OpenSearchClient getOpenSearchClient()
+   {
+      return (client);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for the configured alias name.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java

@@ -23,13 +23,18 @@ import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
@@ -45,8 +50,13 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
 import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
@@ -66,9 +76,14 @@ public abstract class AbstractIndexingStep implements BackendStep
    public static final String STATUS_ACTIVE        = "ACTIVE";
    public static final String STATUS_NEEDS_REINDEX = "NEEDS_REINDEX";
    public static final String STATUS_REBUILDING    = "REBUILDING";
+   public static final String STATUS_PENDING_BUILD = "PENDING_BUILD";
+   public static final String STATUS_BUILDING      = "BUILDING";
    public static final String RUN_COMPLETED        = "COMPLETED";
    public static final String RUN_FAILED           = "FAILED";
    public static final String RUN_RUNNING          = "RUNNING";
+   public static final String RUN_CANCELLED        = "CANCELLED";
+
+   private String cancelledBy = "an admin";
 
 
 
@@ -133,7 +148,8 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Make sure a quickSearchIndex row exists for the table. The table's unique
+    ** Make sure a quick_search_index row exists for the table (hosts without
+    ** the config tables only; see below). The table's unique
     ** key on tableName makes a concurrent first insert fail, which is handled by
     ** re-querying.
     *******************************************************************************/
@@ -144,6 +160,17 @@ public abstract class AbstractIndexingStep implements BackendStep
          return;
       }
 
+      ///////////////////////////////////////////////////////////////////////////
+      // with the config tables, rows come only from priming (which honors    //
+      // tombstones) and Add Search Index: never re-create a removed table    //
+      // here, even while priming has not succeeded and code is in use        //
+      ///////////////////////////////////////////////////////////////////////////
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime != null && runtime.getConfigStore().hasConfigTables())
+      {
+         return;
+      }
+
       try
       {
          QRecord record = new QRecord()
@@ -158,7 +185,7 @@ public abstract class AbstractIndexingStep implements BackendStep
          InsertInput insertInput = new InsertInput();
          insertInput.setTableName(getConfig().getQuickSearchIndexTableName());
          insertInput.setRecords(List.of(record));
-         InsertOutput insertOutput = new InsertAction().execute(insertInput);
+         InsertOutput insertOutput = QuickSearchConfigWrites.asSystem(() -> new InsertAction().execute(insertInput));
 
          if(insertOutput.getRecords().isEmpty() || CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
          {
@@ -178,7 +205,7 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** The quickSearchIndex row for a table, or null.
+    ** The quick_search_index row for a table, or null.
     *******************************************************************************/
    protected QRecord queryIndexRow(String tableName) throws QException
    {
@@ -192,7 +219,7 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** The id of the quickSearchIndex row for a table.
+    ** The id of the quick_search_index row for a table.
     *******************************************************************************/
    protected Integer queryIndexId(String tableName) throws QException
    {
@@ -280,7 +307,7 @@ public abstract class AbstractIndexingStep implements BackendStep
    /*******************************************************************************
     ** JSON array describing a table's searchable fields, weights and labels.
     *******************************************************************************/
-   protected static String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
+   public static String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
    {
       if(tableConfig.getSearchableFields() == null || tableConfig.getSearchableFields().isEmpty())
       {
@@ -301,13 +328,44 @@ public abstract class AbstractIndexingStep implements BackendStep
          Boolean includeLabel = tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName) ? tableConfig.getFieldIncludeLabels().get(fieldName) : false;
          String  escapedName  = fieldName.replace("\\", "\\\\").replace("\"", "\\\"");
 
-         sb.append("{\"fieldName\":\"").append(escapedName).append("\",\"weight\":").append(weight).append(",\"includeLabel\":").append(includeLabel).append("}");
+         sb.append("{\"fieldName\":\"").append(escapedName).append("\",\"weight\":").append(weight).append(",\"includeLabel\":").append(includeLabel);
+
+         ///////////////////////////////////////////////////////////////////
+         // roles, analyzers and snippet flags only when set, so a table  //
+         // of plain fields keeps the JSON it had before they existed     //
+         ///////////////////////////////////////////////////////////////////
+         appendIfSet(sb, "role", tableConfig.getFieldRoles(), fieldName);
+         appendIfSet(sb, "analyzer", tableConfig.getFieldAnalyzers(), fieldName);
+         appendIfSet(sb, "includeInSnippet", tableConfig.getFieldIncludeInSnippet(), fieldName);
+         sb.append("}");
       }
       return (sb.append("]").toString());
    }
 
 
 
+   /*******************************************************************************
+    ** Append ,"key":value for a field's entry in the map, when it has one.
+    *******************************************************************************/
+   private static void appendIfSet(StringBuilder sb, String key, Map<String, ?> values, String fieldName)
+   {
+      Object value = values == null ? null : values.get(fieldName);
+      if(value != null)
+      {
+         sb.append(",\"").append(key).append("\":");
+         if(value instanceof Boolean)
+         {
+            sb.append(value);
+         }
+         else
+         {
+            sb.append("\"").append(String.valueOf(value).replace("\\", "\\\\").replace("\"", "\\\"")).append("\"");
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Insert a RUNNING run record and return it with its id.
     *******************************************************************************/
@@ -324,7 +382,7 @@ public abstract class AbstractIndexingStep implements BackendStep
       InsertOutput insertOutput = new InsertAction().execute(insertInput);
       if(insertOutput.getRecords().isEmpty())
       {
-         throw new QException("Failed to create quickSearchIndexRun record: no record returned from insert");
+         throw new QException("Failed to create quick_search_index_run record: no record returned from insert");
       }
 
       QRecord inserted = insertOutput.getRecords().get(0);
@@ -377,7 +435,65 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Update fields on a quickSearchIndex row.
+    ** Refresh a FULL_REINDEX run record's modifyDate: the heartbeat by which
+    ** other nodes tell a full reindex that is still going from one that
+    ** stopped (see FullReindexStep.recoverDeadRun). Nothing reads it for other
+    ** run types. Best effort: a failure is logged and never stops the run (one
+    ** judged stopped as a result fails safely before its swap).
+    *******************************************************************************/
+   protected void heartbeat(QuickSearchIndexRun run)
+   {
+      if(run == null || run.getId() == null || !FullReindexStep.RUN_TYPE.equals(run.getRunType()))
+      {
+         return;
+      }
+
+      try
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+         updateInput.setRecords(List.of(new QRecord().withValue("id", run.getId()).withValue("modifyDate", Instant.now())));
+         new UpdateAction().execute(updateInput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not refresh the full reindex heartbeat", e, logPair("runId", run.getId()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The record IDs, among the rows' recordIds, that exist in the source table.
+    *******************************************************************************/
+   Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         return (Set.of());
+      }
+
+      String             primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
+      List<Serializable> recordIds       = new ArrayList<>();
+      rows.forEach(row -> recordIds.add(row.getValueString("recordId")));
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, recordIds)));
+
+      Set<String> existingIds = new HashSet<>();
+      for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+      {
+         existingIds.add(record.getValueString(primaryKeyField));
+      }
+      return (existingIds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update fields on a quick_search_index row.
     *******************************************************************************/
    protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
    {
@@ -390,7 +506,7 @@ public abstract class AbstractIndexingStep implements BackendStep
       UpdateInput updateInput = new UpdateInput();
       updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
       updateInput.setRecords(List.of(record));
-      new UpdateAction().execute(updateInput);
+      QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput));
    }
 
 
@@ -429,9 +545,10 @@ public abstract class AbstractIndexingStep implements BackendStep
    /*******************************************************************************
     ** Index every source record matching the criteria into targetIndex (null
     ** for the alias), paging by primary key so each record is read once even
-    ** while rows are inserted or deleted.
+    ** while rows are inserted or deleted. The run's heartbeat is refreshed
+    ** after every page.
     *******************************************************************************/
-   protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
+   protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
    {
       QuickSearchQBitConfig config          = getConfig();
       String                tableName       = tableConfig.getTableName();
@@ -491,6 +608,9 @@ public abstract class AbstractIndexingStep implements BackendStep
          {
             throw new QException("Source record has no value in primary key field [" + primaryKeyField + "]");
          }
+
+         heartbeat(run);
+         stopIfCancelledOrPaused(run, new IndexCounts(processed, indexed, errors, skipped, firstError));
       }
 
       return (new IndexCounts(processed, indexed, errors, skipped, firstError));
@@ -498,23 +618,157 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
 
+   /*******************************************************************************
+    ** The table a reindex or reconcile is limited to, or null for all:
+    ** searchTableName (the input screen's field), else tableName (API
+    ** callers). A Quick Search table is never meant: a process run from the
+    ** Quick Search Index table gets that table as its tableName.
+    *******************************************************************************/
+   protected String tableNameInput(RunBackendStepInput input)
+   {
+      String tableName = input.getValueString("searchTableName");
+      if(tableName == null || tableName.isBlank())
+      {
+         tableName = input.getValueString("tableName");
+      }
+      QuickSearchQBitConfig config = QuickSearchQBitContext.getConfig();
+      if(tableName == null || tableName.isBlank() || (config != null && QuickSearchFieldRules.isQuickSearchTable(config, tableName)))
+      {
+         return (null);
+      }
+      return (tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** When an admin has paused all indexing: say so in the output (the result
+    ** screen's message) and return true, so the step does nothing.
+    *******************************************************************************/
+   protected boolean isPausedSkip(RunBackendStepOutput output)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || !runtime.isIndexingPaused())
+      {
+         return (false);
+      }
+      LOG.info("Quick Search indexing is paused; " + getClass().getSimpleName() + " skipped");
+      output.addValue("message", "Quick Search indexing is paused; nothing was indexed. Resume indexing from Quick Search Settings first.");
+      output.addValue("paused", true);
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Between batches: stop the run, with its counts so far, when an admin
+    ** asked to cancel it or paused all indexing.
+    *******************************************************************************/
+   protected void stopIfCancelledOrPaused(QuickSearchIndexRun run, IndexCounts counts) throws QuickSearchRunCancelledException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime != null && runtime.isIndexingPaused())
+      {
+         throw (new QuickSearchRunCancelledException("Stopped: Quick Search indexing was paused", counts));
+      }
+      if(isCancelRequested(run))
+      {
+         throw (new QuickSearchRunCancelledException("Cancelled by " + cancelledBy, counts));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the run's cancel flag is set (read from its record, so a cancel
+    ** made on another server is seen). A read failure is not a cancel.
+    *******************************************************************************/
+   protected boolean isCancelRequested(QuickSearchIndexRun run)
+   {
+      if(run == null || run.getId() == null)
+      {
+         return (false);
+      }
+      return (isAnyCancelRequested(List.of(run.getId())));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether any of the runs' cancel flags is set; remembers who asked.
+    *******************************************************************************/
+   protected boolean isAnyCancelRequested(List<Integer> runIds)
+   {
+      try
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, new ArrayList<>(runIds)))
+            .withCriteria(new QFilterCriteria("cancelRequested", QCriteriaOperator.EQUALS, true)));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            return (false);
+         }
+         cancelledBy = rows.get(0).getValueString("cancelRequestedBy") == null ? "an admin" : rows.get(0).getValueString("cancelRequestedBy");
+         return (true);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not read the cancel flag of a run", e);
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** End a run that was cancelled (or stopped by a pause): CANCELLED with its
+    ** counts so far. Never throws.
+    *******************************************************************************/
+   protected void completeCancelledRun(QuickSearchIndexRun run, QuickSearchRunCancelledException e)
+   {
+      IndexCounts counts = e.getCounts();
+      completeRunRecord(run, RUN_CANCELLED, counts == null ? 0 : counts.processed(), counts == null ? 0 : counts.indexed(), counts == null ? 0 : counts.errors(), e.getMessage());
+   }
+
+
+
    /*******************************************************************************
     ** Delete run records older than runHistoryRetentionDays. Never throws.
     *******************************************************************************/
    protected void purgeOldRuns()
+   {
+      Integer retentionDays = getConfigOrNull() == null ? null : getConfigOrNull().getRunHistoryRetentionDays();
+      if(retentionDays == null || retentionDays <= 0)
+      {
+         return;
+      }
+
+      Instant cutoff = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
+      purgeOlderThan(getConfigOrNull().getQuickSearchIndexRunTableName(), "startTime", cutoff, retentionDays);
+      purgeOlderThan(getConfigOrNull().applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME), "createDate", cutoff, retentionDays);
+      purgeOlderThan(getConfigOrNull().applyTablePrefix(QuickSearchValidationRun.TABLE_NAME), "createDate", cutoff, retentionDays);
+   }
+
+
+
+   /*******************************************************************************
+    ** Delete up to 5000 rows of a history table older than the cutoff. A table
+    ** not in the instance is skipped; failures are logged.
+    *******************************************************************************/
+   private void purgeOlderThan(String tableName, String dateField, Instant cutoff, Integer retentionDays)
    {
       try
       {
-         Integer retentionDays = getConfig().getRunHistoryRetentionDays();
-         if(retentionDays == null || retentionDays <= 0)
+         if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
          {
             return;
          }
 
-         Instant    cutoff     = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
          QueryInput queryInput = new QueryInput();
-         queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
-         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("startTime", QCriteriaOperator.LESS_THAN, cutoff)).withLimit(5000));
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(dateField, QCriteriaOperator.LESS_THAN, cutoff)).withLimit(5000));
          List<QRecord> old = new QueryAction().execute(queryInput).getRecords();
          if(CollectionUtils.nullSafeIsEmpty(old))
          {
@@ -528,14 +782,31 @@ public abstract class AbstractIndexingStep implements BackendStep
          }
 
          DeleteInput deleteInput = new DeleteInput();
-         deleteInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+         deleteInput.setTableName(tableName);
          deleteInput.setPrimaryKeys(ids);
          new DeleteAction().execute(deleteInput);
-         LOG.info("Purged old Quick Search run records", logPair("count", ids.size()), logPair("olderThanDays", retentionDays));
+         LOG.info("Purged old Quick Search history records", logPair("tableName", tableName), logPair("count", ids.size()), logPair("olderThanDays", retentionDays));
       }
       catch(Exception e)
       {
-         LOG.warn("Failed to purge old run records", e);
+         LOG.warn("Failed to purge old Quick Search history records", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The config, or null when it cannot be resolved (purging is best effort).
+    *******************************************************************************/
+   private QuickSearchQBitConfig getConfigOrNull()
+   {
+      try
+      {
+         return (getConfig());
+      }
+      catch(Exception e)
+      {
+         return (null);
       }
    }
 

src/main/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStep.java

@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
@@ -42,7 +43,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
@@ -55,10 +58,12 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 /*******************************************************************************
  ** Scheduled catch-up. For each enabled table that is due, re-indexes records
  ** whose basepull timestamp is at or after the previous run's start minus the
- ** configured overlap, paging by primary key. First replays failed real-time
- ** events, and afterwards purges old run history.
+ ** configured overlap, paging by primary key. First recovers what a full
+ ** reindex that stopped without finishing left behind (so its captured deletes
+ ** are replayed in the same run) and replays failed real-time events, and
+ ** afterwards purges old run history.
  **
- ** The watermark stored on the quickSearchIndex row is the run's start time,
+ ** The watermark stored on the quick_search_index row is the run's start time,
  ** captured before the first query, so rows modified during the run are read
  ** again next time (re-indexing is idempotent). A table without a timestamp
  ** field is left to the reconcile process.
@@ -81,7 +86,41 @@ public class BasepullIndexStep extends AbstractIndexingStep
    @Override
    public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
    {
-      List<String> failures = new ArrayList<>();
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime != null)
+      {
+         runtime.getConfigStore().tryPrime();
+         if(runtime.isIndexingPaused())
+         {
+            LOG.info("Quick Search indexing is paused; basepull skipped");
+            output.addValue("tablesIndexed", 0);
+            output.addValue("paused", true);
+            return;
+         }
+      }
+
+      List<String>  failures = new ArrayList<>();
+      List<QRecord> liveRuns = List.of();
+
+      try
+      {
+         liveRuns = new FullReindexStep().recoverDeadRun(true);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Recovering from a stopped full reindex did not complete", e);
+         failures.add("full reindex recovery: " + e.getMessage());
+      }
+
+      try
+      {
+         queueBuildsIfIndexMissingOrEmpty();
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not check whether the search index is missing or empty", e);
+         failures.add("index check: " + e.getMessage());
+      }
 
       try
       {
@@ -105,14 +144,38 @@ public class BasepullIndexStep extends AbstractIndexingStep
 
       if(CollectionUtils.nullSafeIsEmpty(indexRecords))
       {
-         LOG.info("No enabled quickSearchIndex rows found; nothing to basepull");
+         LOG.info("No enabled quick_search_index rows found; nothing to basepull");
          purgeOldRuns();
          return;
       }
 
       Integer tablesIndexed = 0;
+      Integer tablesBuilt   = 0;
       for(QRecord indexRecord : indexRecords)
       {
+         ////////////////////////////////////////////////////////////////////////
+         // self-priming: a table queued for a build (new, changed by code, or //
+         // the index was missing) is built in place instead of basepulled; a  //
+         // BUILDING row with no live run was left by a stopped server         //
+         ////////////////////////////////////////////////////////////////////////
+         String status = indexRecord.getValueString("status");
+         if(STATUS_PENDING_BUILD.equals(status) || (STATUS_BUILDING.equals(status) && CollectionUtils.nullSafeIsEmpty(liveRuns)))
+         {
+            try
+            {
+               if(buildTable(indexRecord, runtime))
+               {
+                  tablesBuilt++;
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.warn("Building a queued search index failed; it is retried after its update interval", e, logPair("tableName", indexRecord.getValueString("tableName")));
+               failures.add(indexRecord.getValueString("tableName") + " build: " + e.getMessage());
+            }
+            continue;
+         }
+
          QuickSearchIndex index = new QuickSearchIndex()
             .withId(indexRecord.getValueInteger("id"))
             .withTableName(indexRecord.getValueString("tableName"))
@@ -144,6 +207,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
 
       purgeOldRuns();
       output.addValue("tablesIndexed", tablesIndexed);
+      output.addValue("tablesBuilt", tablesBuilt);
 
       if(!failures.isEmpty())
       {
@@ -153,6 +217,148 @@ public class BasepullIndexStep extends AbstractIndexingStep
 
 
 
+   /*******************************************************************************
+    ** Build one queued table in place (the reconcile algorithm, recorded as a
+    ** FULL_REINDEX run): BUILDING while it runs, ACTIVE after. A failure
+    ** queues it again, retried once its update interval has passed since the
+    ** failure; a cancel leaves it NEEDS_REINDEX (Apply Search Index Changes
+    ** starts it again); a pause leaves it queued for when indexing resumes.
+    ** Returns false when it was skipped (backing off after a failure).
+    *******************************************************************************/
+   public boolean buildTable(QRecord row, QuickSearchRuntime runtime) throws QException
+   {
+      String                     tableName   = row.getValueString("tableName");
+      Integer                    indexId     = row.getValueInteger("id");
+      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Table is no longer configured for Quick Search"));
+      }
+
+      Integer intervalMinutes = row.getValueInteger("basepullIntervalMinutes");
+      long    backoffMillis   = Math.max(1, intervalMinutes == null ? 5 : intervalMinutes) * 60_000L;
+      if(runtime != null && System.currentTimeMillis() - runtime.getLastBuildFailureMillis(tableName) < backoffMillis)
+      {
+         return (false);
+      }
+
+      updateIndexRow(indexId, Map.of("status", STATUS_BUILDING));
+      try
+      {
+         new ReconcileIndexStep().reconcileTable(tableConfig, FullReindexStep.RUN_TYPE);
+         if(runtime != null)
+         {
+            runtime.clearBuildFailure(tableName);
+         }
+         LOG.info("Built queued search index", logPair("tableName", tableName));
+         return (true);
+      }
+      catch(QuickSearchRunCancelledException e)
+      {
+         boolean paused = runtime != null && runtime.isIndexingPaused();
+         updateIndexRow(indexId, Map.of("status", paused ? STATUS_PENDING_BUILD : STATUS_NEEDS_REINDEX));
+         LOG.info("Queued search index build stopped: " + e.getMessage(), logPair("tableName", tableName));
+         return (false);
+      }
+      catch(Exception e)
+      {
+         updateIndexRow(indexId, Map.of("status", STATUS_PENDING_BUILD));
+         if(runtime != null)
+         {
+            runtime.recordBuildFailure(tableName);
+         }
+         throw (e instanceof QException qe ? qe : new QException("Build failed: " + e.getMessage(), e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Self-priming of the index itself: when the alias is gone (a new cluster,
+    ** or the index was deleted) it is created again. When the index was just
+    ** created, every enabled table that had documents in the lost index (and
+    ** has not been fully rebuilt since) is queued for a build; when it is
+    ** empty while a table last counted documents in it, every enabled table is.
+    *******************************************************************************/
+   void queueBuildsIfIndexMissingOrEmpty() throws QException
+   {
+      QuickSearchOpenSearchClient client = getClient();
+      String                      reason = null;
+      if(!client.indexExists())
+      {
+         client.ensureIndexExists();
+      }
+
+      List<QRecord> rows = queryEnabledRows();
+      if(client.wasIndexCreated())
+      {
+         ///////////////////////////////////////////////////////////////////////
+         // only tables that had documents in the lost index and have not been //
+         // fully rebuilt since: a table never basepulled reads everything on  //
+         // its first basepull anyway                                          //
+         ///////////////////////////////////////////////////////////////////////
+         Instant createdAt = client.getIndexCreatedAt() == null ? Instant.EPOCH : client.getIndexCreatedAt();
+         rows = rows.stream().filter(r -> r.getValue("lastBasepullTime") != null
+            && !isAfter(r.getValueInstant("lastFullReindexTime"), createdAt) && !isAfter(r.getValueInstant("lastReconcileTime"), createdAt)).toList();
+         reason = "the search index was missing and has been created";
+      }
+      else
+      {
+         Long documents = client.countAllDocuments();
+         if(documents != null && documents == 0 && rows.stream().anyMatch(r -> r.getValueInteger("documentCount") != null && r.getValueInteger("documentCount") > 0))
+         {
+            reason = "the search index is empty";
+         }
+      }
+
+      if(reason == null)
+      {
+         return;
+      }
+
+      int queued = 0;
+      for(QRecord row : rows)
+      {
+         String status = row.getValueString("status");
+         if(!STATUS_PENDING_BUILD.equals(status) && !STATUS_BUILDING.equals(status) && !STATUS_REBUILDING.equals(status))
+         {
+            QuickSearchConfigWrites.asSystem(() ->
+            {
+               updateIndexRow(row.getValueInteger("id"), Map.of("status", STATUS_PENDING_BUILD));
+               return (null);
+            });
+            queued++;
+         }
+      }
+      client.clearIndexCreated();
+      LOG.info("Queued a build of every enabled table: " + reason, logPair("tablesQueued", queued));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a time is set and after another.
+    *******************************************************************************/
+   private static boolean isAfter(Instant time, Instant other)
+   {
+      return (time != null && time.isAfter(other));
+   }
+
+
+
+   /*******************************************************************************
+    ** The enabled quick_search_index rows.
+    *******************************************************************************/
+   private List<QRecord> queryEnabledRows() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)));
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
    /*******************************************************************************
     ** Due when never run, or when the interval has elapsed since the last run.
     *******************************************************************************/
@@ -183,7 +389,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
 
       if(tableConfig == null)
       {
-         String message = "Table is no longer configured for Quick Search; disable or remove its quickSearchIndex row";
+         String message = "Table is no longer configured for Quick Search; disable or remove its quick_search_index row";
          completeRunRecord(run, RUN_FAILED, 0, 0, 1, message);
          throw (new QException(tableName + ": " + message));
       }
@@ -211,12 +417,13 @@ public class BasepullIndexStep extends AbstractIndexingStep
             criteria.add(new QFilterCriteria(timestampField, QCriteriaOperator.GREATER_THAN_OR_EQUALS, since));
          }
 
-         IndexCounts counts = indexAllRecords(client, tableConfig, criteria, null);
+         IndexCounts counts = indexAllRecords(client, tableConfig, criteria, null, run);
 
          if(counts.errors() == 0)
          {
             Map<String, Serializable> values = new HashMap<>();
             values.put("lastBasepullTime", runStart);
+            values.put("realTimeErrorCount", 0);
             Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
             if(documentCount != null)
             {
@@ -236,6 +443,11 @@ public class BasepullIndexStep extends AbstractIndexingStep
             throw (new QException("Basepull indexed " + counts.indexed() + " of " + counts.processed() + " records for table [" + tableName + "]; first error: " + counts.firstError()));
          }
       }
+      catch(QuickSearchRunCancelledException e)
+      {
+         completeCancelledRun(run, e);
+         throw e;
+      }
       catch(QException e)
       {
          if(!RUN_FAILED.equals(run.getStatus()))
@@ -256,13 +468,15 @@ public class BasepullIndexStep extends AbstractIndexingStep
 
    /*******************************************************************************
     ** Replay PENDING failed real-time events: re-read and index records for
-    ** INDEX events, delete documents for DELETE events. Successes are removed;
-    ** failures bump attempts and become EXHAUSTED at the limit.
+    ** INDEX events, delete documents for DELETE events. A DELETE whose record
+    ** exists again (re-created with the same key since) keeps its document and
+    ** counts as replayed. Successes are removed; failures bump attempts and
+    ** become EXHAUSTED at the limit.
     *******************************************************************************/
    void replayFailedEvents(RunBackendStepOutput output) throws QException
    {
       QuickSearchQBitConfig config          = getConfig();
-      String                failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
+      String                failedEventTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(failedEventTable);
@@ -292,19 +506,37 @@ public class BasepullIndexStep extends AbstractIndexingStep
 
       for(Map.Entry<String, List<QRecord>> entry : deleteByTable.entrySet())
       {
-         List<String> documentIds = new ArrayList<>();
-         for(QRecord event : entry.getValue())
-         {
-            documentIds.add(OpenSearchDocument.buildDocumentId(entry.getKey(), event.getValueString("recordId")));
-         }
+         List<QRecord> toDelete = entry.getValue();
          try
          {
-            BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
-            markReplayOutcome(entry.getValue(), result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
+            Set<String> existingIds = queryExistingRecordIds(entry.getKey(), entry.getValue());
+            toDelete = new ArrayList<>();
+            for(QRecord event : entry.getValue())
+            {
+               if(existingIds.contains(event.getValueString("recordId")))
+               {
+                  succeeded.add(event.getValue("id"));
+               }
+               else
+               {
+                  toDelete.add(event);
+               }
+            }
+
+            if(!toDelete.isEmpty())
+            {
+               List<String> documentIds = new ArrayList<>();
+               for(QRecord event : toDelete)
+               {
+                  documentIds.add(OpenSearchDocument.buildDocumentId(entry.getKey(), event.getValueString("recordId")));
+               }
+               BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
+               markReplayOutcome(toDelete, result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
+            }
          }
          catch(Exception e)
          {
-            markReplayOutcome(entry.getValue(), false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
+            markReplayOutcome(toDelete, false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
          }
       }
 

src/main/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStep.java

@@ -19,12 +19,15 @@ package com.kingsrook.qbits.quicksearch.processes;
 
 import java.io.Serializable;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -56,7 +59,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 /*******************************************************************************
  ** Rebuild the index from source.
  **
- ** Without a "tableName" input, every enabled table is indexed into a fresh
+ ** Without a "searchTableName" (or "tableName") input, every enabled table is indexed into a fresh
  ** physical index with the current mapping; when all of them succeed the alias
  ** is swapped to it and the old index is deleted, so search keeps serving the
  ** old documents until the new set is complete. This is also how an index
@@ -66,12 +69,20 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** While the new index is built, real-time writes still go through the alias
  ** to the old one. Updates are caught by the next basepull (lastBasepullTime
  ** is set to each table's start). Deletes are caught here: each table's
- ** quickSearchIndex row is REBUILDING for the whole rebuild, which makes the
+ ** quick_search_index row is REBUILDING for the whole rebuild, which makes the
  ** record change listener (on any node) also record each delete as an
  ** AWAITING_REINDEX failed event. Once the rebuild's own writes are done and
  ** the alias is swapped, those deletes are applied through the alias.
  **
- ** With a "tableName" input, that table is rebuilt in place with the reconcile
+ ** Only one full reindex runs at a time. Each table's run record is RUNNING
+ ** before any table is marked REBUILDING, and its modifyDate is refreshed
+ ** after every page; a run whose records show no sign of life for
+ ** fullReindexStaleMinutes has stopped (killed JVM, lost node). Every full
+ ** reindex, and every basepull, first recovers what such a run left behind
+ ** (see recoverDeadRun); a full reindex then refuses to start while another
+ ** one is alive.
+ **
+ ** With a "searchTableName" (or "tableName") input, that table is rebuilt in place with the reconcile
  ** algorithm (index everything, then delete what was not re-indexed).
  *******************************************************************************/
 public class FullReindexStep extends AbstractIndexingStep
@@ -80,6 +91,9 @@ public class FullReindexStep extends AbstractIndexingStep
 
    public static final String RUN_TYPE = "FULL_REINDEX";
 
+   private List<Integer>       cancelGroup;
+   private QuickSearchIndexRun runOfCurrentTable;
+
 
 
    /*******************************************************************************
@@ -88,12 +102,25 @@ public class FullReindexStep extends AbstractIndexingStep
    @Override
    public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
    {
-      String tableNameFilter = input.getValueString("tableName");
+      if(isPausedSkip(output))
+      {
+         output.addValue("recordsIndexed", 0);
+         output.addValue("aliasSwapped", false);
+         return;
+      }
+
+      String  tableNameFilter = tableNameInput(input);
+      boolean allTables       = tableNameFilter == null || tableNameFilter.isBlank();
+
+      //////////////////////////////////////////////////////////////////////////////
+      // an all-tables rebuild applies the captured deletes itself after its swap //
+      //////////////////////////////////////////////////////////////////////////////
+      refuseIfAlive(recoverDeadRun(!allTables));
 
       List<QuickSearchableTableConfig> tables = new ArrayList<>();
       for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
       {
-         if(tableNameFilter != null && !tableNameFilter.isBlank() && !tableNameFilter.equals(tableConfig.getTableName()))
+         if(!allTables && !tableNameFilter.equals(tableConfig.getTableName()))
          {
             continue;
          }
@@ -108,7 +135,7 @@ public class FullReindexStep extends AbstractIndexingStep
          }
       }
 
-      if(tableNameFilter != null && !tableNameFilter.isBlank())
+      if(!allTables)
       {
          Integer recordsIndexed = 0;
          for(QuickSearchableTableConfig tableConfig : tables)
@@ -126,6 +153,22 @@ public class FullReindexStep extends AbstractIndexingStep
 
 
 
+   /*******************************************************************************
+    ** An all-tables rebuild is one run per table; cancelling any of them
+    ** cancels the rebuild (the old index stays in service).
+    *******************************************************************************/
+   @Override
+   protected boolean isCancelRequested(QuickSearchIndexRun run)
+   {
+      if(cancelGroup != null && !cancelGroup.isEmpty())
+      {
+         return (isAnyCancelRequested(cancelGroup));
+      }
+      return (super.isCancelRequested(run));
+   }
+
+
+
    /*******************************************************************************
     ** Index every table into a new physical index, then swap the alias.
     *******************************************************************************/
@@ -134,29 +177,45 @@ public class FullReindexStep extends AbstractIndexingStep
       QuickSearchOpenSearchClient client        = getClient();
       String                      physicalIndex = client.newPhysicalIndexName();
 
-      client.createPhysicalIndex(physicalIndex);
-      LOG.info("Rebuilding Quick Search index", logPair("physicalIndex", physicalIndex), logPair("tables", tables.size()));
-
       Map<QuickSearchableTableConfig, QuickSearchIndexRun> runs        = new LinkedHashMap<>();
       Map<String, IndexCounts>                              counts      = new LinkedHashMap<>();
       Map<String, Instant>                                  startTimes  = new LinkedHashMap<>();
       Map<String, String>                                   priorStatus = new LinkedHashMap<>();
       Integer                                               indexed     = 0;
 
+      boolean created = false;
+      boolean marked  = false;
       boolean swapped = false;
 
       try
       {
-         markRebuilding(tables, priorStatus);
-
+         /////////////////////////////////////////////////////////////////////////
+         // the run records exist before any table is REBUILDING, so whoever    //
+         // sees a table REBUILDING also sees this run alive; then make sure no //
+         // other run started at the same moment                                //
+         /////////////////////////////////////////////////////////////////////////
          for(QuickSearchableTableConfig tableConfig : tables)
          {
-            String              tableName = tableConfig.getTableName();
-            QuickSearchIndexRun run       = createRunRecord(queryIndexId(tableName), RUN_TYPE);
-            runs.put(tableConfig, run);
+            runs.put(tableConfig, createRunRecord(queryIndexId(tableConfig.getTableName()), RUN_TYPE));
+         }
+         refuseIfAlive(queryLiveRunsOtherThan(runs.values()));
+
+         client.createPhysicalIndex(physicalIndex);
+         created = true;
+         LOG.info("Rebuilding Quick Search index", logPair("physicalIndex", physicalIndex), logPair("tables", tables.size()));
+
+         marked = true;
+         markRebuilding(tables, priorStatus);
+
+         cancelGroup = new ArrayList<>();
+         runs.values().forEach(r -> cancelGroup.add(r.getId()));
+         for(QuickSearchableTableConfig tableConfig : tables)
+         {
+            String tableName = tableConfig.getTableName();
             startTimes.put(tableName, Instant.now());
 
-            IndexCounts tableCounts = indexAllRecords(client, tableConfig, List.of(), physicalIndex);
+            runOfCurrentTable = runs.get(tableConfig);
+            IndexCounts tableCounts = indexAllRecords(client, tableConfig, List.of(), physicalIndex, runs.get(tableConfig));
             counts.put(tableName, tableCounts);
             indexed = indexed + tableCounts.indexed();
 
@@ -167,6 +226,8 @@ public class FullReindexStep extends AbstractIndexingStep
          }
 
          client.refreshIndex(physicalIndex);
+         keepAlive(runs.values());
+         verifyStillOwned(tables, runs.values());
          client.swapAliasTo(physicalIndex);
          swapped = true;
 
@@ -174,9 +235,11 @@ public class FullReindexStep extends AbstractIndexingStep
          {
             String      tableName   = tableConfig.getTableName();
             IndexCounts tableCounts = counts.get(tableName);
+            keepAlive(runs.values());
 
             Map<String, Serializable> values = new HashMap<>();
             values.put("lastFullReindexTime", Instant.now());
+            values.put("realTimeErrorCount", 0);
             values.put("lastBasepullTime", startTimes.get(tableName));
             values.put("recordCount", tableCounts.processed());
             Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
@@ -188,30 +251,31 @@ public class FullReindexStep extends AbstractIndexingStep
             values.put("status", STATUS_ACTIVE);
             updateIndexRow(runs.get(tableConfig).getQuickSearchIndexId(), values);
             priorStatus.remove(tableName);
-
-            completeRunRecord(runs.get(tableConfig), RUN_COMPLETED, tableCounts.processed(), tableCounts.indexed(), 0, null);
          }
 
-         applyCapturedDeletes(client, tables);
+         /////////////////////////////////////////////////////////////////////////////
+         // the run records stay RUNNING, refreshed per batch, until the captured   //
+         // deletes are applied; if a recovery takes the run over anyway, the drain //
+         // stops, and it never reads rows captured after it started, which belong  //
+         // to a newer run                                                          //
+         /////////////////////////////////////////////////////////////////////////////
+         applyCapturedDeletes(client, tables, runs.values());
+
+         for(QuickSearchableTableConfig tableConfig : tables)
+         {
+            IndexCounts tableCounts = counts.get(tableConfig.getTableName());
+            completeRunRecord(runs.get(tableConfig), RUN_COMPLETED, tableCounts.processed(), tableCounts.indexed(), 0, null);
+         }
 
          LOG.info("Full reindex complete", logPair("physicalIndex", physicalIndex), logPair("recordsIndexed", indexed));
          return (indexed);
       }
       catch(Exception e)
       {
+         boolean takenOver = marked && isTakenOverOrUnknown(runs.values());
          LOG.warn(swapped ? "Full reindex failed after the alias swap; the new index stays in service" : "Full reindex failed; the previous index stays in service", e, logPair("physicalIndex", physicalIndex));
 
-         for(Map.Entry<QuickSearchableTableConfig, QuickSearchIndexRun> entry : runs.entrySet())
-         {
-            IndexCounts tableCounts = counts.get(entry.getKey().getTableName());
-            completeRunRecord(entry.getValue(), RUN_FAILED,
-               tableCounts == null ? 0 : tableCounts.processed(),
-               tableCounts == null ? 0 : tableCounts.indexed(),
-               tableCounts == null ? 1 : Math.max(1, tableCounts.errors()),
-               e.getMessage());
-         }
-
-         if(!swapped)
+         if(created && !swapped)
          {
             try
             {
@@ -223,8 +287,36 @@ public class FullReindexStep extends AbstractIndexingStep
             }
          }
 
-         restoreStatus(priorStatus);
-         applyCapturedDeletes(client, tables);
+         ///////////////////////////////////////////////////////////////////////////
+         // a recovery that took this run over owns its tables, captured deletes  //
+         // and run records (a newer run may own the first two by now); when that //
+         // cannot be read, the run records stay RUNNING for the next recovery    //
+         ///////////////////////////////////////////////////////////////////////////
+         if(!takenOver)
+         {
+            if(marked)
+            {
+               restoreStatus(priorStatus);
+               applyCapturedDeletes(client, tables, runs.values());
+            }
+
+            for(Map.Entry<QuickSearchableTableConfig, QuickSearchIndexRun> entry : runs.entrySet())
+            {
+               IndexCounts tableCounts = counts.get(entry.getKey().getTableName());
+               if(e instanceof QuickSearchRunCancelledException cancelled)
+               {
+                  IndexCounts stopped = tableCounts != null ? tableCounts : (entry.getValue() == runOfCurrentTable ? cancelled.getCounts() : null);
+                  completeRunRecord(entry.getValue(), RUN_CANCELLED, stopped == null ? 0 : stopped.processed(), stopped == null ? 0 : stopped.indexed(),
+                     stopped == null ? 0 : stopped.errors(), e.getMessage() + "; the previous index stays in service");
+                  continue;
+               }
+               completeRunRecord(entry.getValue(), RUN_FAILED,
+                  tableCounts == null ? 0 : tableCounts.processed(),
+                  tableCounts == null ? 0 : tableCounts.indexed(),
+                  tableCounts == null ? 1 : Math.max(1, tableCounts.errors()),
+                  e.getMessage());
+            }
+         }
 
          if(e instanceof QException qException)
          {
@@ -275,20 +367,421 @@ public class FullReindexStep extends AbstractIndexingStep
 
 
 
+   /*******************************************************************************
+    ** Recover what a full reindex that stopped without finishing (killed JVM,
+    ** lost node) left behind, unless a full reindex is alive. Returns the live
+    ** RUNNING FULL_REINDEX run records, and changes nothing when there are any.
+    **
+    ** A run is alive while the startTime or modifyDate (its heartbeat) of one of
+    ** its RUNNING records is within fullReindexStaleMinutes. When none is
+    ** alive: each REBUILDING table goes back to ACTIVE (the status it had
+    ** before is not stored; drift detection marks NEEDS_REINDEX again where it
+    ** applies), with its lastBasepullTime moved back to the stopped run's
+    ** start, so basepull re-reads updates that a swap may have left out; with
+    ** handCapturedDeletesToBasepull, every AWAITING_REINDEX row up to the
+    ** newest one read here becomes PENDING; and only then is each stale RUNNING
+    ** record marked FAILED, so a recovery that fails partway leaves them for
+    ** the next one, run start included. Unlike completeRunRecord, that write
+    ** fails the recovery (and so the full reindex calling it) when any record
+    ** is not updated. Replaying those deletes is harmless
+    ** whether or not the stopped run swapped: before a swap the alias points at
+    ** the old index, which the listener already deleted from (a missing
+    ** document counts as success); after one, it removes the copy the run read
+    ** before the delete.
+    **
+    ** The REBUILDING rows and captured rows are read before the run records. A
+    ** run creates its records before it marks a table, and the listener
+    ** captures a delete only once it sees REBUILDING, so anything read here
+    ** that belongs to a live run comes with a live record, and nothing of a
+    ** live run is touched. A run that starts after the records are read, or a
+    ** live run misjudged as stopped, can have a table put back to ACTIVE or its
+    ** records marked FAILED; it checks both right after a heartbeat, before its
+    ** swap, and fails without swapping. A misjudged run that already swapped
+    ** stops applying its captured deletes once its records are FAILED, and
+    ** never reads rows captured after its drain started.
+    *******************************************************************************/
+   List<QRecord> recoverDeadRun(boolean handCapturedDeletesToBasepull) throws QException
+   {
+      List<QRecord> rebuildingRows = queryRebuildingIndexRows();
+      Serializable  lastCapturedId = handCapturedDeletesToBasepull ? queryLastCapturedDeleteId() : null;
+      List<QRecord> runningRuns    = queryRunningRuns();
+
+      Instant       cutoff = staleCutoff();
+      List<QRecord> alive  = runningRuns.stream().filter(run -> isAlive(run, cutoff)).toList();
+      if(!alive.isEmpty() || (rebuildingRows.isEmpty() && runningRuns.isEmpty() && lastCapturedId == null))
+      {
+         return (alive);
+      }
+
+      LOG.warn("Recovering what a full reindex that stopped without finishing left behind",
+         logPair("staleRunIds", runningRuns.stream().map(run -> run.getValue("id")).toList()),
+         logPair("tableNames", rebuildingRows.stream().map(row -> row.getValueString("tableName")).toList()));
+
+      for(QRecord row : rebuildingRows)
+      {
+         Map<String, Serializable> values = new HashMap<>();
+         values.put("status", STATUS_ACTIVE);
+
+         Instant runStart         = earliestStart(runningRuns, row.getValueInteger("id"));
+         Instant lastBasepullTime = row.getValueInstant("lastBasepullTime");
+         if(runStart != null && lastBasepullTime != null && runStart.isBefore(lastBasepullTime))
+         {
+            values.put("lastBasepullTime", runStart);
+         }
+         updateIndexRow(row.getValueInteger("id"), values);
+      }
+
+      if(lastCapturedId != null)
+      {
+         handCapturedDeletesToBasepull(lastCapturedId);
+      }
+
+      failDeadRuns(runningRuns);
+      return (alive);
+   }
+
+
+
+   /*******************************************************************************
+    ** Mark the stale run records FAILED, failing when any of them is not
+    ** updated: a live run misjudged as stopped learns it was taken over only
+    ** from these records, so recovery must not finish, and no new full reindex
+    ** may start, until they are written. The status is mirrored onto each
+    ** index row, as completeRunRecord does.
+    *******************************************************************************/
+   private void failDeadRuns(List<QRecord> runningRuns) throws QException
+   {
+      if(runningRuns.isEmpty())
+      {
+         return;
+      }
+
+      Instant       endTime = Instant.now();
+      List<QRecord> updates = new ArrayList<>();
+      for(QRecord runRecord : runningRuns)
+      {
+         updates.add(new QRecord()
+            .withValue("id", runRecord.getValue("id"))
+            .withValue("status", RUN_FAILED)
+            .withValue("endTime", endTime)
+            .withValue("recordsProcessed", 0)
+            .withValue("recordsIndexed", 0)
+            .withValue("errorCount", 1)
+            .withValue("errorMessage", truncate(staleRunMessage(runRecord), 4000)));
+      }
+
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+      updateInput.setRecords(updates);
+      List<QRecord> results = CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords());
+      List<QRecord> written = results.stream().filter(result -> CollectionUtils.nullSafeIsEmpty(result.getErrors())).toList();
+      if(written.size() < updates.size())
+      {
+         List<QRecord> notWritten = results.stream().filter(result -> CollectionUtils.nullSafeHasContents(result.getErrors())).toList();
+         throw (new QException("Could not mark " + (updates.size() - written.size()) + " stopped full reindex run record(s) FAILED"
+            + (notWritten.isEmpty() ? "" : " (first error: " + notWritten.get(0).getErrors() + ")")
+            + "; recovery did not finish, and no full reindex starts until it does"));
+      }
+
+      for(QRecord runRecord : runningRuns)
+      {
+         Integer indexId = runRecord.getValueInteger("quickSearchIndexId");
+         if(indexId != null)
+         {
+            Map<String, Serializable> values = new HashMap<>();
+            values.put("lastRunStatus", RUN_TYPE + " " + RUN_FAILED);
+            values.put("lastErrorMessage", truncate(staleRunMessage(runRecord), 4000));
+            updateIndexRow(indexId, values);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The error message recorded on a run that stopped without finishing.
+    *******************************************************************************/
+   private static String staleRunMessage(QRecord runRecord)
+   {
+      return ("Full reindex stopped without finishing (no heartbeat since " + lastSeen(runRecord) + "); recovered automatically");
+   }
+
+
+
+   /*******************************************************************************
+    ** Make the AWAITING_REINDEX rows up to lastCapturedId PENDING, for basepull
+    ** to replay. Rows are read in id order past the previous batch, so a row
+    ** that could not be updated is not read again.
+    *******************************************************************************/
+   private void handCapturedDeletesToBasepull(Serializable lastCapturedId) throws QException
+   {
+      QuickSearchQBitConfig config           = getConfig();
+      String                failedEventTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+      Serializable          afterId          = null;
+      Integer               handedOver       = 0;
+
+      while(true)
+      {
+         QQueryFilter filter = new QQueryFilter(
+            new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX),
+            new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, lastCapturedId))
+            .withOrderBy(new QFilterOrderBy("id", true))
+            .withLimit(config.getSourceBatchSize());
+         if(afterId != null)
+         {
+            filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, afterId));
+         }
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(failedEventTable);
+         queryInput.setFilter(filter);
+         List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+         if(CollectionUtils.nullSafeIsEmpty(rows))
+         {
+            break;
+         }
+
+         afterId = rows.get(rows.size() - 1).getValue("id");
+         List<QRecord> updates = new ArrayList<>();
+         rows.forEach(row -> updates.add(new QRecord().withValue("id", row.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING)));
+
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(failedEventTable);
+         updateInput.setRecords(updates);
+         new UpdateAction().execute(updateInput);
+         handedOver = handedOver + rows.size();
+      }
+
+      LOG.info("Handed deletes captured during a stopped full reindex to basepull", logPair("count", handedOver));
+   }
+
+
+
+   /*******************************************************************************
+    ** Fail when another full reindex is alive.
+    *******************************************************************************/
+   private void refuseIfAlive(List<QRecord> alive) throws QException
+   {
+      if(alive.isEmpty())
+      {
+         return;
+      }
+
+      QRecord run = alive.get(0);
+      throw (new QException("A full reindex is already running (run " + run.getValue("id") + ", last heartbeat " + lastSeen(run) + "); try again once it finishes. "
+         + "A run with no heartbeat for " + getConfig().getEffectiveFullReindexStaleMinutes() + " minutes is treated as stopped and recovered automatically."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Just before the swap, fail unless this run still owns its tables: its
+    ** run records are still RUNNING (a recovery that judged it stopped marks
+    ** them FAILED) and every table is still REBUILDING (otherwise a delete
+    ** since then may have reached only the old index, uncaptured).
+    *******************************************************************************/
+   private void verifyStillOwned(List<QuickSearchableTableConfig> tables, Collection<QuickSearchIndexRun> runs) throws QException
+   {
+      if(isTakenOver(runs))
+      {
+         throw (new QException("Full reindex was recovered as stopped (no heartbeat for " + getConfig().getEffectiveFullReindexStaleMinutes() + " minutes) while still running; the alias was not swapped"));
+      }
+
+      for(QuickSearchableTableConfig tableConfig : tables)
+      {
+         String status = queryIndexRow(tableConfig.getTableName()).getValueString("status");
+         if(!STATUS_REBUILDING.equals(status))
+         {
+            throw (new QException("Table [" + tableConfig.getTableName() + "] is no longer REBUILDING (now " + status + "), so a delete since then may not have been captured; the alias was not swapped"));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Refresh one of the run's records, which keeps the whole run alive.
+    *******************************************************************************/
+   private void keepAlive(Collection<QuickSearchIndexRun> runs)
+   {
+      runs.stream().findFirst().ifPresent(this::heartbeat);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a recovery has marked any of the run records finished.
+    *******************************************************************************/
+   private boolean isTakenOver(Collection<QuickSearchIndexRun> runs) throws QException
+   {
+      if(runs.isEmpty())
+      {
+         return (false);
+      }
+
+      List<Serializable> ids = new ArrayList<>();
+      runs.forEach(run -> ids.add(run.getId()));
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, ids)));
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()).stream()
+         .anyMatch(run -> !RUN_RUNNING.equals(run.getValueString("status"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** isTakenOver for the failure path: never throws, and answers true when it
+    ** cannot tell, which leaves the tables to the next recovery.
+    *******************************************************************************/
+   private boolean isTakenOverOrUnknown(Collection<QuickSearchIndexRun> runs)
+   {
+      try
+      {
+         return (isTakenOver(runs));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not read whether a recovery took over the failed full reindex; leaving its tables to the next recovery", e);
+         return (true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The live RUNNING FULL_REINDEX run records other than the given ones.
+    *******************************************************************************/
+   private List<QRecord> queryLiveRunsOtherThan(Collection<QuickSearchIndexRun> own) throws QException
+   {
+      Set<Integer> ownIds = new HashSet<>();
+      own.forEach(run -> ownIds.add(run.getId()));
+
+      Instant cutoff = staleCutoff();
+      return (queryRunningRuns().stream()
+         .filter(run -> !ownIds.contains(run.getValueInteger("id")) && isAlive(run, cutoff))
+         .toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** Every RUNNING FULL_REINDEX run record, oldest first.
+    *******************************************************************************/
+   private List<QRecord> queryRunningRuns() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+      queryInput.setFilter(new QQueryFilter(
+         new QFilterCriteria("runType", QCriteriaOperator.EQUALS, RUN_TYPE),
+         new QFilterCriteria("status", QCriteriaOperator.EQUALS, RUN_RUNNING))
+         .withOrderBy(new QFilterOrderBy("id", true)));
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every quick_search_index row that is REBUILDING.
+    *******************************************************************************/
+   private List<QRecord> queryRebuildingIndexRows() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, STATUS_REBUILDING)));
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The id of the newest AWAITING_REINDEX row, or null. Ids grow in insert
+    ** order, so a row captured after this read is beyond it.
+    *******************************************************************************/
+   private Serializable queryLastCapturedDeleteId() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX))
+         .withOrderBy(new QFilterOrderBy("id", false))
+         .withLimit(1));
+      List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeIsEmpty(rows) ? null : rows.get(0).getValue("id"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Run records last seen before this are stale.
+    *******************************************************************************/
+   private Instant staleCutoff() throws QException
+   {
+      return (Instant.now().minus(getConfig().getEffectiveFullReindexStaleMinutes(), ChronoUnit.MINUTES));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a run record was last seen after the cutoff.
+    *******************************************************************************/
+   private static boolean isAlive(QRecord run, Instant cutoff)
+   {
+      Instant lastSeen = lastSeen(run);
+      return (lastSeen != null && lastSeen.isAfter(cutoff));
+   }
+
+
+
+   /*******************************************************************************
+    ** A run record's latest sign of life: its startTime or its heartbeat
+    ** (modifyDate), whichever is later; null when it has neither.
+    *******************************************************************************/
+   private static Instant lastSeen(QRecord run)
+   {
+      Instant startTime  = run.getValueInstant("startTime");
+      Instant modifyDate = run.getValueInstant("modifyDate");
+      if(startTime == null || modifyDate == null)
+      {
+         return (startTime == null ? modifyDate : startTime);
+      }
+      return (startTime.isAfter(modifyDate) ? startTime : modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** The earliest startTime among the run records of an index row, or null.
+    *******************************************************************************/
+   private static Instant earliestStart(List<QRecord> runs, Integer indexId)
+   {
+      return (runs.stream()
+         .filter(run -> Objects.equals(indexId, run.getValueInteger("quickSearchIndexId")))
+         .map(run -> run.getValueInstant("startTime"))
+         .filter(Objects::nonNull)
+         .min(Instant::compareTo)
+         .orElse(null));
+   }
+
+
+
    /*******************************************************************************
     ** Apply the deletes the listener captured during the rebuild (failed-event
     ** rows with status AWAITING_REINDEX) through the alias, after the rebuild's
     ** own writes, so an in-flight page cannot bring a deleted record back.
     ** Called once the tables are no longer REBUILDING, so the listener stops
     ** adding rows (one that races the status change hands its own rows to
-    ** basepull).
+    ** basepull). Only rows up to the newest one read at the start are applied,
+    ** since a later row belongs to a newer run. The run is kept alive before
+    ** every batch, and the drain stops once a recovery has taken it over.
     **
     ** Rows are read in id order, past the previous batch, so a batch whose rows
     ** could not be removed or updated (QQQ reports those as per-row errors, not
-    ** exceptions) is not read again; such rows stay AWAITING_REINDEX for the next
-    ** full reindex. Never throws.
+    ** exceptions) is not read again; such rows stay AWAITING_REINDEX until the
+    ** next full reindex, or a basepull that finds none running, picks them up.
+    ** Never throws.
     *******************************************************************************/
-   void applyCapturedDeletes(QuickSearchOpenSearchClient client, List<QuickSearchableTableConfig> tables)
+   void applyCapturedDeletes(QuickSearchOpenSearchClient client, List<QuickSearchableTableConfig> tables, Collection<QuickSearchIndexRun> runs)
    {
       List<Serializable> tableNames = new ArrayList<>();
       tables.forEach(tableConfig -> tableNames.add(tableConfig.getTableName()));
@@ -301,14 +794,23 @@ public class FullReindexStep extends AbstractIndexingStep
       try
       {
          QuickSearchQBitConfig config           = getConfig();
-         String                failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
+         String                failedEventTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+         Serializable          lastCapturedId   = queryLastCapturedDeleteId();
          Serializable          afterId          = null;
 
-         while(true)
+         while(lastCapturedId != null)
          {
+            if(isTakenOver(runs))
+            {
+               LOG.warn("A recovery took this full reindex over while it applied its captured deletes; leaving the rest to the recovery", logPair("applied", applied));
+               break;
+            }
+            keepAlive(runs);
+
             QQueryFilter filter = new QQueryFilter(
                new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX),
-               new QFilterCriteria("tableName", QCriteriaOperator.IN, tableNames))
+               new QFilterCriteria("tableName", QCriteriaOperator.IN, tableNames),
+               new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, lastCapturedId))
                .withOrderBy(new QFilterOrderBy("id", true))
                .withLimit(config.getSourceBatchSize());
             if(afterId != null)
@@ -449,35 +951,6 @@ public class FullReindexStep extends AbstractIndexingStep
 
 
 
-   /*******************************************************************************
-    ** The record IDs, among the rows' recordIds, that exist in the source table.
-    *******************************************************************************/
-   Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
-   {
-      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
-      if(tableConfig == null)
-      {
-         return (Set.of());
-      }
-
-      String             primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
-      List<Serializable> recordIds       = new ArrayList<>();
-      rows.forEach(row -> recordIds.add(row.getValueString("recordId")));
-
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(tableName);
-      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, recordIds)));
-
-      Set<String> existingIds = new HashSet<>();
-      for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
-      {
-         existingIds.add(record.getValueString(primaryKeyField));
-      }
-      return (existingIds);
-   }
-
-
-
    /*******************************************************************************
     ** Rebuild one table in place (kept for callers that used this directly).
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtils.java

@@ -18,9 +18,12 @@ package com.kingsrook.qbits.quicksearch.processes;
 
 
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -30,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -46,6 +50,8 @@ public class IndexingUtils
 {
    private static final QLogger LOG = QLogger.getLogger(IndexingUtils.class);
 
+   static final int MAX_ROLE_VALUE_LENGTH = 500;
+
 
 
    /*******************************************************************************
@@ -172,11 +178,123 @@ public class IndexingUtils
          doc.withRecordLabel(formatLabel(record, tableConfig, doc.getRecordLabel()));
       }
 
+      if(doc != null)
+      {
+         applyRolesAndAnalyzers(doc, record, tableConfig, table);
+      }
+
       return (doc);
    }
 
 
 
+   /*******************************************************************************
+    ** Apply the fields' roles and analyzers (from quick_search_index_field):
+    ** only snippet fields go into searchableText; WORD, ENGLISH and EXACT
+    ** fields are also written to fieldValuesWord, fieldValuesEnglish and
+    ** fieldValuesExact (lower-cased) for their analyzers; TITLE fields make the
+    ** record label, SUBTITLE fields the subtitle and DISPLAY fields the
+    ** displayValues (by field label). A table whose fields are all plain BODY
+    ** fields is left exactly as before.
+    *******************************************************************************/
+   static void applyRolesAndAnalyzers(OpenSearchDocument doc, QRecord record, QuickSearchableTableConfig tableConfig, QTableMetaData table)
+   {
+      List<String> fields = tableConfig.getSearchableFields() == null ? List.of() : tableConfig.getSearchableFields();
+      if(!tableConfig.isAllPlainFields())
+      {
+         List<String> snippetFields = new ArrayList<>();
+         for(String fieldName : fields)
+         {
+            if(tableConfig.isIncludedInSnippet(fieldName))
+            {
+               snippetFields.add(fieldName);
+            }
+         }
+         doc.withSearchableText(buildSearchableText(record, snippetFields, tableConfig.getFieldIncludeLabels(), table, tableConfig.getMaxFieldLength()));
+
+         Map<String, Object> word    = new LinkedHashMap<>();
+         Map<String, Object> english = new LinkedHashMap<>();
+         Map<String, Object> exact   = new LinkedHashMap<>();
+         for(String fieldName : fields)
+         {
+            String value = displayValue(record, fieldName, tableConfig.getMaxFieldLength());
+            if(value == null)
+            {
+               continue;
+            }
+            switch(tableConfig.getAnalyzer(fieldName))
+            {
+               case WORD -> word.put(fieldName, value);
+               case ENGLISH -> english.put(fieldName, value);
+               case EXACT -> exact.put(fieldName, normalizeExact(value));
+               default ->
+               {
+                  // PREFIX: fieldValues only
+               }
+            }
+         }
+         doc.withFieldValuesWord(word.isEmpty() ? null : word);
+         doc.withFieldValuesEnglish(english.isEmpty() ? null : english);
+         doc.withFieldValuesExact(exact.isEmpty() ? null : exact);
+      }
+
+      List<String>        titles    = new ArrayList<>();
+      List<String>        subtitles = new ArrayList<>();
+      Map<String, String> display   = new LinkedHashMap<>();
+      for(String fieldName : fields)
+      {
+         QuickSearchFieldRole role = tableConfig.getRole(fieldName);
+         if(role != QuickSearchFieldRole.TITLE && role != QuickSearchFieldRole.SUBTITLE && role != QuickSearchFieldRole.DISPLAY)
+         {
+            continue;
+         }
+         String value = displayValue(record, fieldName, MAX_ROLE_VALUE_LENGTH);
+         if(value == null)
+         {
+            continue;
+         }
+         switch(role)
+         {
+            case TITLE -> titles.add(value);
+            case SUBTITLE -> subtitles.add(value);
+            default -> display.put(fieldLabel(table, fieldName), value);
+         }
+      }
+      if(!titles.isEmpty())
+      {
+         doc.withRecordLabel(String.join(" ", titles));
+      }
+      doc.withSubtitle(subtitles.isEmpty() ? null : String.join(" - ", subtitles));
+      doc.withDisplayValues(display.isEmpty() ? null : display);
+   }
+
+
+
+   /*******************************************************************************
+    ** The form an EXACT value is stored and matched in: trimmed, whitespace
+    ** collapsed, lower-cased.
+    *******************************************************************************/
+   public static String normalizeExact(String value)
+   {
+      return (normalizeSearchText(value).toLowerCase(Locale.ROOT));
+   }
+
+
+
+   /*******************************************************************************
+    ** A field's label, or its name.
+    *******************************************************************************/
+   private static String fieldLabel(QTableMetaData table, String fieldName)
+   {
+      if(table != null && table.getFields().containsKey(fieldName) && StringUtils.hasContent(table.getField(fieldName).getLabel()))
+      {
+         return (table.getField(fieldName).getLabel());
+      }
+      return (fieldName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/