Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#92) from deps/align-gitea into develop

pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.15 -> 4.1.0-RC.26
pom.xml: removed Central snapshot repository
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.circleci/config.yml

@@ -1,14 +1,14 @@
 version: 2.1
 setup: true
 orbs:
-  qqq-orb: kingsrook/qqq-orb@0.7.13
+  qqq-orb: kingsrook/qqq-orb@0.8
 workflows:
   setup:
     jobs:
     - qqq-orb/generate_pipeline:
-        orb_version: 0.7.13
+        orb_version: '0.8'
         filters:
           branches:
             ignore: automation/qbit-rc-pending
           tags:
-            only: /v.*/
+            only: /^v\d+\.\d+\.\d+$/

.claude/settings.local.json

@@ -1,63 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(find:*)",
-      "Bash(grep:*)",
-      "Bash(mvn compile:*)",
-      "Bash(jar -tf:*)",
-      "Bash(jar -xf:*)",
-      "Bash(cat:*)",
-      "Bash(mvn test:*)",
-      "Bash(done)",
-      "Bash(mvn verify:*)",
-      "Bash(git add:*)",
-      "Bash(git commit:*)",
-      "Bash(git push:*)",
-      "mcp__circleci-mcp-server__get_latest_pipeline_status",
-      "mcp__circleci-mcp-server__list_followed_projects",
-      "mcp__circleci-mcp-server__run_pipeline",
-      "mcp__circleci-mcp-server__get_build_failure_logs",
-      "mcp__circleci-mcp-server__config_helper",
-      "mcp__github__get_file_contents",
-      "Bash(mvn dependency:tree:*)",
-      "Bash(mvn help:effective-pom:*)",
-      "Bash(git checkout:*)",
-      "Bash(ls /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/docs/*.md)",
-      "Bash(git:*)",
-      "Bash(ls:*)",
-      "Bash(open:*)",
-      "Bash(WT=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/.claude/worktrees/agent-a9dc6e8e)",
-      "Bash(DEST=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search)",
-      "Bash(cp \"$WT/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java\" \"$DEST/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/\")",
-      "Bash(mvn clean:*)",
-      "Bash(jar tf:*)",
-      "Bash(unzip -p /Users/james.maes/.m2/repository/com/kingsrook/qqq/qqq-backend-core/0.40.0-SNAPSHOT/qqq-backend-core-0.40.0-SNAPSHOT.jar com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java)",
-      "Bash(sed -i '' 's/postInsertRecord/postInsert/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java)",
-      "Bash(sed -i '' 's/postUpdateRecord/postUpdate/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java)",
-      "Bash(sed -i '' 's/postDeleteRecord/postDelete/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java)",
-      "Bash(docker info:*)",
-      "Bash(docker run:*)",
-      "Read(//var/run/**)",
-      "Read(//Users/james.maes/**)",
-      "Bash(echo \"DOCKER_HOST=$DOCKER_HOST\")",
-      "Bash(echo \"TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=$TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE\")",
-      "Bash(DOCKER_HOST=unix:///var/run/docker.sock TESTCONTAINERS_RYUK_DISABLED=true mvn verify -Dit.test=OpenSearchIntegrationTest)",
-      "Bash(docker version:*)",
-      "Bash(DOCKER_HOST=unix:///var/run/docker.sock mvn verify -Dit.test=OpenSearchIntegrationTest)",
-      "Bash(curl -s --unix-socket /var/run/docker.sock http://localhost/v1.24/info)",
-      "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)}  OS: {d.get\\(OperatingSystem,?\\)}''''\\)\")",
-      "Bash(curl -s --unix-socket /Users/james.maes/Library/Containers/com.docker.docker/Data/docker-cli.sock http://localhost/v1.47/info)",
-      "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)}  OS: {d.get\\(OperatingSystem,?\\)}  Containers: {d.get\\(Containers,?\\)}''''\\)\")",
-      "Bash(curl -s --unix-socket /Users/james.maes/.docker/run/docker.sock http://localhost/v1.47/info)",
-      "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)}  NCPU: {d.get\\(NCPU,?\\)}  OSType: {d.get\\(OSType,?\\)}''''\\)\")",
-      "Bash(DOCKER_HOST=unix:///Users/james.maes/.docker/run/docker.sock DOCKER_API_VERSION=1.47 mvn verify -Dit.test=OpenSearchIntegrationTest -pl .)",
-      "Bash(cp /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/LICENSE /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/LICENSE)",
-      "Bash(cp -r /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/checkstyle/* /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/checkstyle/)",
-      "Bash(gh pr:*)",
-      "Bash(gh repo:*)",
-      "Bash(gh api:*)",
-      "Bash(python3 -c \":*)",
-      "mcp__circleci-mcp-server__get_job_test_results"
-    ]
-  }
-}

.editorconfig

@@ -0,0 +1,20 @@
+# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 3
+tab_width = 3
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{cmd,bat}]
+end_of_line = crlf

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto eol=lf
+*.cmd text eol=crlf
+mvnw text eol=lf

.gitea/workflows/build.yaml

@@ -0,0 +1,7 @@
+name: build
+on:
+  push:
+jobs:
+  maven:
+    uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
+    secrets: inherit

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @KofTwentyTwo

.github/dependabot.yml

@@ -0,0 +1,25 @@
+# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
+# major updates stay separate so each gets its own review. At most three version-update PRs per
+# ecosystem are open at once. Security updates are enabled in the repository settings and are
+# not limited by this schedule.
+version: 2
+updates:
+  - package-ecosystem: maven
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      maven-minor-and-patch:
+        update-types: [minor, patch]
+
+  - package-ecosystem: github-actions
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      github-actions:
+        patterns: ['*']

.github/workflows/release-rc.yml

@@ -0,0 +1,74 @@
+name: Release candidate
+'on':
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Plan a cut without changing release state
+        type: boolean
+        default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
+  pull_request:
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
+permissions:
+  contents: write
+  actions: read
+  checks: read
+jobs:
+  release-rc:
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
+    with:
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.gitignore

@@ -18,3 +18,4 @@ hs_err_pid*
 dependency-reduced-pom.xml
 /.env.local
 /.cache/
+.claude/settings.local.json

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,3 @@
+wrapperVersion=3.3.4
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip

.qqq-ci.yml

@@ -1,16 +1,16 @@
 profile: qbit
-orb_version: 0.7.13
+orb_version: '0.8'
 default_branch: develop
 java_version: '21'
 browsers: true
 api_checks: false
 docs_only: false
-release_on_main: true
-release_tag_policy: v-prefix
-test_branch_policy: non-release-no-main-hotfix
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
 publish:
   snapshot: true
   release_candidate: true
   release: true
-  hotfix: true
   asciidoc: false
+release_completion: true

AGENTS.md

@@ -0,0 +1,68 @@
+# AGENTS.md
+
+This file provides guidance to Codex when working with code in this repository.
+
+## Project Overview
+
+**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications.
+
+## Build Commands
+
+```bash
+./mvnw compile          # Compile (always use the wrapper: Maven 3.10 cannot read the parent POM)
+./mvnw test             # Run unit tests
+./mvnw test -Dtest=ClassName  # Run single test class
+./mvnw package          # Build JAR
+CI=true ./mvnw verify   # Full verify: integration tests (Docker) and the JaCoCo gate
+./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0   # other server line
+./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1   # other qqq 4.1 build (profile is active by default)
+```
+
+Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
+Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
+Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped without Docker locally, failed when `CI=true`.
+
+## Architecture
+
+**Entry point:** `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`. QQQ's default `produce()` registers `QBitMetaData` (`com.kingsrook.qbits:quick-search`), validates the config, runs the component producers in `metadata/` (tables, processes, PVS, app), then `postProduceActions` discovers searchable tables, builds the `QuickSearchRuntime`, registers `QuickSearchRecordChangeListener` and `QuickSearchRuntimeService`, and in `FAIL_FAST` mode prepares the index. `produce(QInstance)` adds everything to the instance (0.x call style still works).
+
+**Key Classes:**
+- `QuickSearchQBitConfig` - Host-facing config: connection (`opensearchUrl`), `authMode`, `tls`, timeouts, startup mode, scheduling, permission rules, search bounds, tables. List-based, null-safe `validate`.
+- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
+- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
+- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
+- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quick_search_failed_event`.
+- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
+- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
+- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
+- `metadata/*` - Component producers and the table/process helpers.
+
+**Tables Produced:** `quick_search_index`, `quick_search_index_run`, `quick_search_failed_event` (plus a PVS over `quick_search_index`).
+
+**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
+
+## Code Style
+
+- 3-space indentation, opening braces on next line
+- Javadoc flower box comments (80-char width with `**` borders)
+- Fluent-style setters (`withX()`)
+- Use wrapper types (`Integer`, `Boolean`) not primitives
+
+## Dependencies
+
+- Java 21
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI
+- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
+- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
+- JUnit 5 + AssertJ + Mockito for testing
+- Testcontainers 1.21.4 for integration tests (requires Docker)
+
+## Knowledge base
+
+Second-brain vault notes covering this repo and the QQQ platform it plugs into:
+
+- Platform hub: `$SECOND_BRAIN_VAULT/knowledge/qqq/qqq-hub.md` (start here; read `knowledge/qqq/architecture/metadata-model.md` for QBit mechanics)
+- This repo's dossier: `$SECOND_BRAIN_VAULT/knowledge/qqq/repos/qbit-quick-search.md` (reviewed at develop @ `8d9ec2711fde`, 2026-07-04)
+- Production-readiness audit + P0-P3 roadmap: `$SECOND_BRAIN_VAULT/projects/qbit-quick-search.md`
+
+Note: the dossier was reviewed before the qqq 4.0 re-pin; trust this file for build and dependency facts.

CHANGELOG.md

@@ -0,0 +1,129 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
+
+## Release note correction (1.1.0-RC.11)
+
+1.1.0-RC.11 (Maven Central, 2026-10-15 13:05Z) was built from release/1.1 `2e2501d` and contains #61 only. The `v1.1.0-RC.11` tag mistakenly points at `6028a66`; use `2e2501d` as RC.11's source. Cluster caches (#62) first ship in 1.1.0-RC.12.
+
+## [Unreleased]
+
+Requires QQQ 4.1.0-RC.15 or later (central health checks, help bundles, the spec-managed UI API extension contract, and `ClusterCaches` for multi-node deployments).
+
+### Fixed
+- Single-node clusters: the Quick Search index no longer asks for a replica it cannot place, so a one-node OpenSearch stays green instead of yellow and `quick_search.opensearch` is no longer DEGRADED. New `indexReplicas` (default null: automatic, `min(maxAutoReplicas, data nodes - 1)`) and `maxAutoReplicas` (default 1) config; zero config gives 0 replicas on one node and 1 on larger clusters. The target is set on every new physical index and reconciled on existing ones at startup with `PUT /<index>/_settings` when it differs (idempotent, logged; a failure is logged and reported, never blocks startup; indexes with `auto_expand_replicas` and OpenSearch Serverless are left alone). The status snapshot and the `quick_search.opensearch` health details gain `unassignedReplicaShards`, `targetReplicas` and `replicaSettingsFailed`; yellow on a single node with every primary active (only replicas unassigned) is reported UP with a note. Yellow with an unassigned primary, or on a multi-node cluster, stays DEGRADED; red stays DOWN.
+- Priming config records at startup, and recording diagnostic events, build their system session with the instance, so it has its all-access keys. Startup logged "QInstance was not supplied when creating a QSystemUserSession" once per node.
+- Multi-node deployments: a Quick Search config change (an index row, its fields or the settings row, from the admin UI or a config process) now clears the cached config on every node, not only the node that made the change. The runtime's config cache (the config store snapshot and settings, and the per-table enabled flags) is registered with QQQ `ClusterCaches` as `qbit-quick-search.config`, and `QuickSearchRuntime.invalidateConfig()` publishes a cluster invalidation. The 30 s and 60 s TTLs remain as the safety net. `QuickSearchFieldRules`' warn-once table set is registered as node-local. `QuickSearchClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation.
+- Indexing bookkeeping is no longer record-audited: the `quick_search_index` runtime fields written by basepull, full reindex and real-time indexing (status, last basepull time, document count, last run, real-time error count) and the settings row's priming outcome (`lastPrimedAt`, summary) are written with `omitDmlAudit`. On a production host these wrote about 470 "Record was Edited during process: Update search index" audit rows an hour. Admin edits to an index, its fields or the settings are still audited.
+- Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write.
+
+### Added
+- UI API extension quick-search 1.1.0 (requires QQQ 4.1.0-RC.15, UI API 1.6.0): operationIds, standard error responses with BasicErrorResponseV1, extension tags, and retained deprecated QuickSearchErrorV1 schema. Existing operationIds and success payloads are preserved.
+- Central health checks `quick_search.opensearch`, `quick_search.backlog` and the dynamic `quick_search.index.<indexName>` supplier, all SEARCH/DEPENDENCY checks every 60 seconds. Reuses cached diagnostics with deadline-bounded waits and safe count/flag details. Includes registration, state, deadline and credential-sentinel tests.
+- Host-neutral `help/qbit-quick-search.yaml` bundle, registered through QQQ help bundles, with coverage and no-help-table tests. Replaces the temporary inline help loader; hosts can override entries with a HOST bundle.
+- Every `quick_search_*` table and the Quick Search Admin app carry help that says what the table is for and how its rows get created (`help/quick-search.yaml`, QQQ help-bundle format, applied by `QuickSearchHelp` as a stand-in for core help bundles; host help is never overridden). `QuickSearchTablesGuardrailTest` fails when a table is unused by runtime code, neither primed nor listed as starting empty, or has no such help; `QuickSearchSeededDefaultsTest` proves priming populates the config tables on a fresh backend, is idempotent and keeps admin edits.
+- Session-only typeahead: `GET /qqq/v1/qbit/quick-search/suggest` (`getQuickSearchSuggestions`) and `QuickSearchSuggestService.suggest(...)`. Reuses the permission-aware search action with one bounded candidate batch per table; groups expose only record IDs, filtered labels/subtitles, `hasMore` and `truncated`. Defaults to 5 items across 8 tables (maximum 10/20); short terms and unavailable search return empty groups. The existing UI extension and generated contract now carry both status and suggestions.
+- Supported host status API (`api.QuickSearchStatusService.get()`), immutable status values, and `QuickSearchApi` home navigation constants (issue #50/F10). Permission-checked, bounded diagnostics refreshes share a short per-runtime cache and reuse saved validation counts. New spec-managed `GET /qqq/v1/qbit/quick-search/status` (`getQuickSearchStatus`) with committed OpenAPI fragment and strict HTTP/contract tests. RC.6 requires the default `/qbit/quick-search` prefix; `/quick-search` is not permitted by core. Existing routes and wire shapes are unchanged.
+- Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`.
+
+- **Data-driven configuration.** The searchable tables and fields are records: `quick_search_index` (the table registry, new columns `source` and `ownerEdited`) and the new `quick_search_index_field` (weight, role `DISPLAY`/`TITLE`/`SUBTITLE`/`BODY`/`KEYWORD`/`EXACT`, analyzer `PREFIX`/`WORD`/`ENGLISH`/`EXACT`, snippet and label flags). Indexing, search, the record-search provider and the diagnostics read the effective config from them (cached 30 s, dropped on change), so changes need no redeploy. Code (`@QuickSearchable`, `searchableTables`) only seeds them; rows an admin edited are never overwritten, and removals are remembered in the new `quick_search_config_tombstone`. Precedence and DDL: `docs/MIGRATION-1.1.md`, "Data-driven configuration".
+- **Self-priming.** New code-declared tables and fields get their rows at startup and are built by the next basepull (`PENDING_BUILD`, `BUILDING`); a missing index is recreated and rebuilt, an empty one rebuilt; existing rows are migrated without a rebuild. The Search Diagnostics widget reports it; `QuickSearchStatus` gives hosts the same banner.
+- Admin processes with review and result steps (record or table actions, never in the nav): Add Search Index (the valid create: eligible-table picker, estimated record count, default fields), Apply Search Index Changes (previews field changes, then rebuilds), Disable Search Index and Remove Search Index (delete or keep documents), Cancel Index Run (cooperative cancel between batches; runs end `CANCELLED` with counts), Pause Indexing and Resume Indexing (global pause in the new `quick_search_settings`; search keeps serving, changes are deferred and replayed).
+- `QuickSearchResult.subtitle` and `displayValues` (from `SUBTITLE` and `DISPLAY` fields); document objects `fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`, added to existing indexes' mapping at startup.
+- Possible-value sources `quick_search_eligible_table`, `quick_search_indexable_field`, `quick_search_timestamp_field`, `quick_search_field_role`, `quick_search_field_analyzer`, `quick_search_documents_choice`, `quick_search_config_source`; the join `quick_search_index_join_quick_search_index_field` and the Searchable Fields widget on the index record. Config tables are audited at field level.
+
+### Removed
+- None for central health integration; the status API, diagnostics widget and developer panel are retained.
+
+### Changed
+- Distinguish protected fields from fields that simply cannot be searched. Boolean, blob and hidden fields no longer trigger result rebuilding or erase record labels. Labels are suppressed only when their source fields are protected; unprotected hidden primary keys and timestamps remain usable.
+- Exclude PII/PHI-classified fields (including table defaults) and field-security-locked fields everywhere Quick Search indexes or displays values. Configuration rejects protected fields; code defaults drop them with one warning per table. Search and diagnostics omit protected values from older documents. Rebuild existing indexes after upgrading or changing classification metadata. Uses QQQ 4.1.0-RC.6; no dependency changes.
+- `quick_search_index`: the generic Create and Delete are off (`TABLE_INSERT`, `TABLE_DELETE`); use Add Search Index and Remove Search Index. `quick_search_searchable_table` is now a custom source over the effective tables. `quick_search_index_run` gains `cancelRequested` and `cancelRequestedBy`.
+- Hidden and `isSensitive` fields are never indexed (previously hidden and password only).
+- Record screens (record-screen audit): `quick_search_index_run` and `quick_search_failed_event` are written by the QBit only, so Create and Edit are off (Delete stays, for clearing); both get named field sections, record labels and pickers (`quick_search_run_type`, `quick_search_run_status`, `quick_search_failed_event_status`, `quick_search_failed_event_action`, and the table picker for `tableName`). The index record lists its Recent Runs (join `quick_search_index_join_quick_search_index_run`). `quick_search_validation_run.reportJson` uses a JSON code editor. The Basepull, Full Reindex and Reconcile processes run from Quick Search Index's Actions and are no longer app (nav) children; their table input is `searchTableName` (`tableName` still works for API callers, and a Quick Search table name is ignored), so running them from the table no longer limits them to `quick_search_index`. Process input steps now carry `HELP_TEXT` and `EDIT_FORM` components, so the Next UI renders their fields.
+- **Snake_case names (owner naming rule, 2026-10-12), for 1.1.0-RC.3.** The tables are now `quick_search_index`, `quick_search_index_run`, `quick_search_failed_event`, `quick_search_diagnostic_event` and `quick_search_validation_run` (they were camelCase). The possible-value sources are `quick_search_index` and `quick_search_searchable_table`. The permission bases are `quick_search_admin` and `quick_search_debug`, and table permissions follow the table names. A table-name prefix is now converted to snake_case and joined with `_`. The old names remain as `@Deprecated` `LEGACY_*` constants. This is metadata only: the QBit ships no DDL. Upgrade steps, with SQL for camelCase databases and for stored references (audit table names, saved views, permission grants), are in `docs/MIGRATION-1.1.md` under "Snake_case names (1.1.0-RC.3)".
+
+## [1.1.0-RC.1] - 2026-10-11
+
+First release candidate of 1.1, published to Maven Central from `release/1.1`. It contains every change listed under 1.1.0 and 1.0.0 below (1.0.0 has not been released as GA). Requires QQQ 4.1.0-RC.2 or later. Upgrade notes: `docs/MIGRATION-1.1.md`.
+
+## [1.1.0] - Unreleased
+
+Requires QQQ 4.1.0-RC.4 or later (the record-search provider SPI arrived in 4.1.0-RC.2; its scores, highlights, totals and paging in 4.1.0-RC.4); built against 4.1.0-RC.4. See `docs/MIGRATION-1.1.md`.
+
+### Added
+- `QuickSearchRecordSearchProvider`: serves QQQ core record search (`RecordSearchAction`, `POST /qqq/v1/search`, the Next UI global search) from OpenSearch for the tables Quick Search indexes. One `_msearch` request with a per-table top-N search (same query, AND semantics and field weights as `QuickSearchAction`); returns record ids only, which core re-reads through `QueryAction` (permissions, record locks, labels). Audit finding F-11, decision D1.
+- `serveCoreRecordSearch` (default `true`): registers the provider and gives indexed tables core `searchFields` (visible string and integer searchable fields) when they have none. Host `searchFields` and a host-registered provider are never replaced.
+- `recordSearchTimeoutMillis` (default 2500): wall-clock limit on the OpenSearch request; on timeout or any failure core falls back to its own search, and the provider claims no tables for 30 seconds.
+- `QuickSearchOpenSearchClient.searchRecordIdsPerTable`.
+- Record search details for QQQ 4.1.0-RC.4's record-search contract: the provider passes each hit's relevance `score`, a plain-text highlight (match ranges, from OpenSearch highlighting of `fieldValues.*` with private-use markers) and each table's total hits (`totalCountByTable`, counted to 10,000), and honors the request `offset` (`from`). `sort: RECENT` keeps relevance order (the index has no record modify dates). New `QuickSearchOpenSearchClient.searchRecordsPerTable`; `searchRecordIdsPerTable` now delegates to it. This requires QQQ 4.1.0-RC.4 or later.
+- Diagnostic tools (#35), on by default (`enableDiagnosticTools`): Search Playground, Validate Search Index (schedulable with `validationCronExpression`; reports kept in the new `quickSearchValidationRun` table), Test Search Analyzer, and on every indexed table and on `quickSearchFailedEvent` rows the record actions Inspect Search Document (database vs index, and why a record does or does not match a term) and Reindex in Search. All are table or record actions, gated by the new `debugPermissionRules` (`quickSearchDebug.hasAccess`; Reindex in Search uses `adminPermissionRules`), rate-limited per user (`diagnosticRunsPerMinute`), and recorded in the new `quickSearchDiagnosticEvent` table. Settings: `enableRecordDiagnosticActions`, `validationCronExpression`, `validationCronTimeZoneId`, `validationSampleSize`.
+- Search failure diagnostics: a failed or timed-out OpenSearch request behind core record search or `QuickSearchAction` is recorded as a `SEARCH_FAILED` diagnostic event (tables, duration, error, server; never the term); the runtime counts failures and backoff fallbacks. The Search Diagnostics widget (`quickSearchDiagnostics`, on the admin app home) shows the provider's state, recent failures and the latest validation.
+- `QuickSearchDiagnosticsOperations`: read-only probe search (named clauses, field highlights, request and response JSON), explain, document reads, `_analyze`, document sampling, and raw cluster, mapping and settings reads.
+
+### Changed
+- Run history purging also purges `quickSearchDiagnosticEvent` and `quickSearchValidationRun` rows older than `runHistoryRetentionDays`.
+
+### Security
+- Fields of type `PASSWORD` are no longer indexed (as hidden fields already were). Reconcile or fully reindex once to remove values a 1.0 index may hold.
+
+## [1.0.0-RC.2] - 2026-10-06
+
+Second release candidate, published to Maven Central from `release/1.0`. Fixes the five pre-GA findings from the PR #10 review (#13 to #17); the changes are listed under 1.0.0 below. Requires QQQ 4.1. No schema change, but two new status values: `REBUILDING` on `quickSearchIndex.status` and `AWAITING_REINDEX` on `quickSearchFailedEvent.status`; a host that sized its own `quickSearchFailedEvent.status` column needs at least 16 characters. Known gaps before GA are tracked in #12.
+
+## [1.0.0-RC.1] - 2026-10-05
+
+First release candidate of 1.0, published to Maven Central from `release/1.0`. It contains every change listed under 1.0.0 below. Requires QQQ 4.1 (built against 4.1.0-RC.1). Known gaps before GA are tracked in #12.
+
+## [1.0.0] - Unreleased
+
+Requires QQQ 4.1. Closes every blocker of the 2026-10-04 production-readiness audit (`docs/audits/2026-10-04/README.md`).
+
+### Added
+- Standard QQQ 4.1 QBit contract: `QuickSearchQBitProducer implements QBitMetaDataProducer`, self-registered `QBitMetaData` (`com.kingsrook.qbits:quick-search`, real artifact version), `sourceQBitName` on produced metadata, component producers, `QuickSearchRuntimeService`.
+- Real-time indexing through `RecordChangeListenerInterface`, published after commit via `QBackendTransaction.addAfterCommitCallback`.
+- `quickSearchFailedEvent` table: failed real-time events are recorded and replayed by basepull.
+- Scheduling: `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`.
+- Full reindex of all tables builds a fresh physical index and swaps the alias; the configured index name is now an alias.
+- Mapping version 2: separate search analyzer, dynamic template mapping `fieldValues.*` as text, `_meta.quickSearchMappingVersion`; outdated indexes are detected and rebuilt by a full reindex.
+- Permission-aware search: table READ permission filter, `QueryAction` post-filter for record security locks, `tableNames`, `limitPerTable`, `tableLabel`, `totalHitsIsLowerBound`, term and page bounds.
+- Connection security: `opensearchUrl`, `authMode` (`NONE`, `BASIC`, `AWS_SIGV4`), `tls` block (CA, trust store, mTLS, hostname verification, guarded `insecureSkipVerify`), timeouts, pool size, `User-Agent`, `OpenSearchTransportCustomizer` SPI, `${env.X}` secret references, redacting `toString`, `@JsonIgnore` on secret getters.
+- `startupMode` (`FAIL_FAST` default, `DEGRADED`), `enableBasepullProcess` and `enableMaintenanceProcesses`, `basepullOverlapSeconds`, `runHistoryRetentionDays`, `maxFieldLength`, `maxSearchLimit`, `maxBulkRequestBytes`, `applyRecordSecurityLocks`, `adminPermissionRules`, `tablePermissionRules`, `tableMetaDataCustomizer`.
+- Admin table fields: `lastReconcileTime`, `lastRunStatus`, `lastErrorMessage`, `documentCount`, `realTimeErrorCount`; `quickSearchIndexRun.quickSearchIndexId` shows the table name; icons and permission rules on the app, tables and processes; unique key on `quickSearchIndex.tableName`.
+- Display values and possible-value labels are indexed; record labels fall back to QQQ's record label format; hidden fields are excluded; values are stored as strings and truncated to `maxFieldLength`.
+- External versioning (`external_gte`) from the basepull timestamp field so stale writes never overwrite newer documents.
+- Reconcile removes documents of tables no longer configured; drift between configured fields and the index row sets `status = NEEDS_REINDEX`.
+- Maven Wrapper (3.9.11), JaCoCo coverage gate actually enforced, parameterized Testcontainers image (`-Dopensearch.test.image`).
+
+### Changed
+- Build targets qqq 4.1 (4.1.0-RC.1 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 is released), opensearch-java 3.10.0, JUnit from the parent BOM.
+- Basepull and full reindex page by primary key; the basepull watermark is the run start (minus overlap) with greater-than-or-equal semantics.
+- Basepull fails at the end when any table failed (one table's failure does not stop the others).
+- Query terms use the AND operator and are no longer edge-ngrammed at search time.
+- `QuickSearchOpenSearchClient.search` takes a collection of allowed tables; `deleteDocumentsForTable` returns the deleted count; all OpenSearch errors surface as `QException`.
+- `IndexingUtils.normalizeSearchText` no longer lower-cases (analyzers do).
+- `enableScheduledProcesses` is deprecated in favour of `enableBasepullProcess` and `enableMaintenanceProcesses`.
+- `QuickSearchQBitContext` is deprecated; state lives on `QuickSearchRuntime`, resolved from the `QInstance`.
+- Search `offset` counts results the session can read. With record security locks on, hits are scanned from the start in batches of at most 1,000, filtered for access and deduplicated; one call reads at most 10,000 hits over 32 requests, and in per-table mode each of T tables gets max(10,000 / T, 2 x `limitPerTable`) hits. When that share runs out `hasMore` is false and `totalHitsIsLowerBound` is true. With locks on, `totalHits` counts hits confirmed readable. Per-table mode now honours `offset` (#16).
+- A full reindex of all tables marks each table `REBUILDING` while it runs; drift detection is suspended for those tables. After a killed run, run the full reindex again (#14).
+
+### Removed
+- The three table customizers (`QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`), replaced by `QuickSearchRecordChangeListener`.
+- Support for QQQ 4.0.
+
+### Security
+- `opensearchUrl` with userinfo (`https://user:pass@host`) fails validation; an `${env.X}` or `${prop.X}` credential or TLS path that resolves to an empty or blank value fails transport creation and names the field, never the value; `tls.hostnameVerification=false` logs a warning (#15).
+
+### Fixed
+- Rolled-back writes could be indexed; real-time failures were silent; basepull lost rows modified during a run and rows at the boundary second; unordered paging skipped or duplicated rows; full reindex blanked search; search leaked labels and snippets across tables regardless of permissions; mixed field types across tables broke indexing; non-text searchable fields failed every search; record labels were null for annotation tables; duplicate `quickSearchIndex` rows on concurrent first runs; unbounded `limit`/`offset`; unchecked `OpenSearchException`; unreachable cluster at startup left the host healthy with search broken.
+- Deletes made during a full reindex of all tables were lost from the new index; they are now captured and applied after the alias swap (#14).
+- An update was dropped without a `quickSearchFailedEvent` row when the listener's re-read failed or the optional AWS SDK was missing (`LinkageError`); both are now recorded for basepull, and a `DEGRADED` start without the AWS SDK no longer aborts host startup (#13).
+- Search pages overlapped under record locks, per-table mode ignored `offset`, and a large `maxSearchLimit` could exceed the 10,000 result window (#16).
+- A failing document count was swallowed and cleared `documentCount`; it is now logged at warn and the previous count is kept (#17).
+- A real-time insert was indexed with raw values (a foreign key as its id, the record label as "2 - email") until the next basepull. Inserts are now re-read in the write's transaction with display values and possible-value labels, as updates are; a failed re-read is recorded in `quickSearchFailedEvent` for basepull and never fails the insert (#28).
+
+## [0.2.0] - 2026-03-30
+- Config-driven table indexing (`SearchableTableConfig`).
+
+## [0.1.0] - 2026-03-29
+- Initial release.

CLAUDE.md

@@ -4,59 +4,44 @@ This file provides guidance to Claude Code when working with code in this reposi
 
 ## Project Overview
 
-**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications. Current version: 0.1.0.
+**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications.
 
 ## Build Commands
 
 ```bash
-mvn compile          # Compile
-mvn test             # Run all tests (174 tests)
-mvn test -Dtest=ClassName  # Run single test class
-mvn package          # Build JAR
-mvn verify           # Full verify with coverage check
+./mvnw compile          # Compile (always use the wrapper: Maven 3.10 cannot read the parent POM)
+./mvnw test             # Run unit tests
+./mvnw test -Dtest=ClassName  # Run single test class
+./mvnw package          # Build JAR
+CI=true ./mvnw verify   # Full verify: integration tests (Docker) and the JaCoCo gate
+./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0   # other server line
+./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1   # other qqq 4.1 build (profile is active by default)
 ```
 
 Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
 Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
+Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped without Docker locally, failed when `CI=true`.
 
 ## Architecture
 
-**Entry point:** `QuickSearchQBitProducer.produce(QInstance)` validates config, discovers annotated entity classes, initializes the singleton `QuickSearchOpenSearchClient`, creates an `IndexEventPublisher`, registers all metadata into the `QInstance`, and attaches customizers to source tables.
+**Entry point:** `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`. QQQ's default `produce()` registers `QBitMetaData` (`com.kingsrook.qbits:quick-search`), validates the config, runs the component producers in `metadata/` (tables, processes, PVS, app), then `postProduceActions` discovers searchable tables, builds the `QuickSearchRuntime`, registers `QuickSearchRecordChangeListener` and `QuickSearchRuntimeService`, and in `FAIL_FAST` mode prepares the index. `produce(QInstance)` adds everything to the instance (0.x call style still works).
 
 **Key Classes:**
-- `QuickSearchQBitProducer` - Orchestrator: produces tables, processes, app, and registers customizers
-- `QuickSearchQBitConfig` - Configuration (backend name, OpenSearch connection, feature flags)
-- `QuickSearchQBitContext` - Static singleton holding the live config, client, publisher, and discovered table configs
-- `QuickSearchOpenSearchClient` - All OpenSearch operations (ensure index, bulk index, delete, search)
-- `QuickSearchableTableConfig` - Runtime config for a single indexed table (fields, weights, basepull settings)
+- `QuickSearchQBitConfig` - Host-facing config: connection (`opensearchUrl`), `authMode`, `tls`, timeouts, startup mode, scheduling, permission rules, search bounds, tables. List-based, null-safe `validate`.
+- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
+- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
+- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
+- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quick_search_failed_event`.
+- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
+- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
+- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
+- `metadata/*` - Component producers and the table/process helpers.
 
-**Customizers** (`customizers/` package, all three registered when `enableRealTimeIndexing = true`):
-- `QuickSearchPostInsertCustomizer` - Fires after insert, publishes index events
-- `QuickSearchPostUpdateCustomizer` - Fires after update, publishes index events
-- `QuickSearchPostDeleteCustomizer` - Fires after delete, publishes delete events
+**Tables Produced:** `quick_search_index`, `quick_search_index_field`, `quick_search_settings`, `quick_search_config_tombstone`, `quick_search_index_run`, `quick_search_failed_event`, `quick_search_diagnostic_event`, `quick_search_validation_run` (plus PVSs).
 
-**Processes** (`processes/` package):
-- `AbstractIndexingStep` - Base class with shared indexing logic (query source table, build documents, publish events, update run record)
-- `BasepullIndexStep` - Queries records modified since last run timestamp
-- `FullReindexStep` - Wipes and rebuilds the index for one or all tables
-- `IndexingUtils` - Converts `QRecord` values to indexed text using field configs
+**Data-driven config (`config/`):** the records are the source of truth. `QuickSearchConfigStore` builds the effective `QuickSearchableTableConfig`s from `quick_search_index` + `quick_search_index_field` (30 s cache, dropped by the tables' customizers); `QuickSearchRuntime.getDiscoveredTables()/getTableConfig()` return them, `getCodeTables()` the code defaults. `QuickSearchConfigPrimer` seeds records from code (never over `ownerEdited` rows or tombstones). QBit writes to config tables go through `QuickSearchConfigWrites.asSystem` so customizers do not treat them as owner edits. Admin processes live in `processes/config/` (streamed ETL with review steps).
 
-**Publisher** (`publisher/` package):
-- `IndexEventPublisher` - Strategy interface with `publishIndexEvents()`, `publishDeleteEvents()`, `close()`
-- `SynchronousIndexEventPublisher` - Default implementation; processes events immediately via the OpenSearch client
-- `IndexEvent` / `IndexEventAction` - Event model
-
-**Actions** (`actions/` package):
-- `QuickSearchAction` - Executes a paginated multi-match search across all indexed fields with per-field boosting
-- `QuickSearchInput` / `QuickSearchOutput` / `QuickSearchResult` - Search API model
-
-**Tables Produced:**
-- `quickSearchIndex` - One row per indexed table; tracks enabled state, basepull interval, last run times
-- `quickSearchIndexRun` - Run history with status, record counts, and error messages
-
-**Annotation Discovery:**
-- `@QuickSearchable(tableName = "...")` - Mark entity classes; defines basepull settings
-- `@QuickSearchField(weight, includeLabel)` - Mark fields to include in indexed text; takes precedence over `fields[]` array
+**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
 
 ## Code Style
 
@@ -68,9 +53,18 @@ Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
 ## Dependencies
 
 - Java 21
-- QQQ Backend Core 0.40.0-SNAPSHOT (via `qqq-bom-pom`)
-- OpenSearch Java Client 2.10.0
-- Apache HttpClient5 5.3
-- jackson-datatype-jsr310 2.21.0 (Instant serialization for OpenSearch client)
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.4`: the record-search provider SPI arrived in RC.2, the developer panel contract (`DeveloperPanelProviderInterface`) in RC.4
+- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
+- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
 - JUnit 5 + AssertJ + Mockito for testing
-- Testcontainers for integration tests (requires Docker)
+- Testcontainers 1.21.4 for integration tests (requires Docker)
+
+## Knowledge base
+
+Second-brain vault notes covering this repo and the QQQ platform it plugs into:
+
+- Platform hub: `$SECOND_BRAIN_VAULT/knowledge/qqq/qqq-hub.md` (start here; read `knowledge/qqq/architecture/metadata-model.md` for QBit mechanics)
+- This repo's dossier: `$SECOND_BRAIN_VAULT/knowledge/qqq/repos/qbit-quick-search.md` (reviewed at develop @ `8d9ec2711fde`, 2026-07-04)
+- Production-readiness audit + P0-P3 roadmap: `$SECOND_BRAIN_VAULT/projects/qbit-quick-search.md`
+
+Note: the dossier was reviewed before the qqq 4.0 re-pin; trust this file for build and dependency facts.

README.md

@@ -1,86 +1,61 @@
 # QBit: Quick Search
 
-[![Version](https://img.shields.io/badge/version-0.2.0-blue.svg)](https://github.com/QRun-IO/qbit-quick-search)
+[![Version](https://img.shields.io/badge/version-1.1.0--RC.1-blue.svg)](https://github.com/QRun-IO/qbit-quick-search)
 [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://www.apache.org/licenses/LICENSE-2.0)
 [![Java](https://img.shields.io/badge/java-21+-blue.svg)](https://adoptium.net/)
 
-> **OpenSearch-Powered Full-Text Search for QQQ Applications**
+> OpenSearch-backed full-text search across the tables of a QQQ application.
 
-This QBit provides global search across QQQ application tables via OpenSearch. Annotate entity classes or declare tables via config, configure a connection, produce the QBit, and gain real-time indexing, scheduled basepull catch-up, per-field relevance boosting, and a pagination-aware search API.
+Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
 
-## Core Capabilities
+Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the current build requires QQQ 4.1.0-RC.7 for help bundles.
 
-- **Real-Time Indexing**: Post-insert, post-update, and post-delete customizers index records immediately on write
-- **Basepull Safety Net**: Scheduled process catches records missed during downtime or bulk imports
-- **Field-Level Boosting**: `@QuickSearchField(weight = N)` controls relevance ranking per field
-- **Edge-Ngram Typeahead**: Custom analyzer enables prefix matching ("ord" finds "order")
-- **Paginated Search**: `QuickSearchAction` returns results with scores, highlights, total hits, and hasMore flag
-- **Config-Driven Indexing**: `SearchableTableConfig` API indexes tables from third-party QBit jars without modifying their source
-- **Extensible Publisher**: `IndexEventPublisher` interface allows swapping in RabbitMQ, Kafka, or any async transport
-- **Admin Observability**: Operational tables track index status and run history
+## What it does
 
-## Open Source & Full Control
+| Capability | How |
+|---|---|
+| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quick_search_failed_event` and replayed by the scheduled basepull. |
+| Basepull catch-up | Scheduled process (`schedulerName` plus `basepullRepeatSeconds`) that re-indexes rows changed since the previous run's start, with a configurable overlap, paging by primary key. |
+| Reconcile | Scheduled or on-demand process that re-indexes everything in place, then removes documents whose source row is gone, including documents of tables no longer configured. No search blackout. |
+| Full rebuild | Indexes every table into a fresh physical index and atomically swaps the alias; the old index keeps serving until the new one is complete. Also upgrades an index created by an older release. |
+| Core search | `QuickSearchRecordSearchProvider` serves QQQ's `RecordSearchAction` (`POST /qqq/v1/search`, the Next UI global search box) from OpenSearch for the indexed tables, with a timeout and an automatic fallback to core's own search. |
+| Search | `QuickSearchAction`: AND semantics, edge-ngram typeahead with a plain search analyzer, per-field weights, highlights, `tableNames` and `limitPerTable`, bounded paging, `tableLabel` on results. Only tables the session may read are searched; hits are re-read through `QueryAction` so record security locks apply. |
+| Connections | HTTP or HTTPS, basic auth, AWS IAM SigV4, custom CA or trust store, mTLS, timeouts, pool size, a transport customizer SPI for anything else. Secrets by `${env.X}` reference. |
+| Operations | `quick_search_index` (per-table status, counts, last run, last error), `quick_search_index_run` (history, purged after `runHistoryRetentionDays`), `quick_search_failed_event`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
 
-QBit Quick Search is 100% open source under Apache 2.0. All data stays in your OpenSearch cluster.
+## Supported platforms
 
-## Architecture
+Tested in CI against the Docker images named in `pom.xml` (`opensearch.test.image`).
 
-### Design Principles
+| Target | Status |
+|---|---|
+| Self-managed OpenSearch 2.19.x and 3.x | Supported, security plugin disabled or HTTP basic authentication, HTTP or HTTPS with a trusted, configured or (development only) unverified certificate |
+| Amazon OpenSearch Service managed domains, OpenSearch 2.x or 3.x | Supported with fine-grained access control: either an internal-database master user with `authMode BASIC`, or IAM with `authMode AWS_SIGV4` (`awsRegion`, `awsServiceName es`, optional `awsAssumeRoleArn`). Port 443 over HTTPS. Keep `bulkBatchSize` and `maxBulkRequestBytes` under the instance type's 10 MiB payload limit on small instances. |
+| Amazon OpenSearch Serverless | Not supported in 1.0 (`_delete_by_query` and `_refresh`, used by reconcile and full rebuild, are not offered there) |
+| Elasticsearch, OpenSearch 1.x | Not supported |
 
-1. **Real-time first, basepull as safety net**: Table customizers index records synchronously on every insert, update, and delete. The basepull process runs on a schedule as a catch-up mechanism for bulk imports, OpenSearch downtime, or any records the real-time path missed.
+Least-privilege role for the identity the QBit uses, on the alias and its physical indexes (`{indexName}*`): `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*`, `indices:admin/aliases`, `indices:admin/delete` (for the alias swap) and `indices:admin/mapping/get`; cluster level `cluster_composite_ops` for `_bulk`. For AWS fine-grained access control map the same to the basic-auth user or the IAM role. Treat the index as sensitive data: it holds copies of every indexed field value.
 
-2. **Annotation and config-driven discovery**: Entity classes marked with `@QuickSearchable` and `@QuickSearchField` are discovered at produce time. Tables from third-party jars can be declared via `SearchableTableConfig` in code. Both paths merge during `produce()`.
+For `AWS_SIGV4` add these optional dependencies to your host (the QBit declares them `optional`):
 
-3. **Extensible transport**: The `IndexEventPublisher` strategy interface decouples the write path from the indexing path. The default `SynchronousIndexEventPublisher` writes directly to OpenSearch. Consumers can provide a queue-backed implementation for async indexing.
-
-### Technology Stack
-
-- **Java 21** with QQQ backend modules
-- **OpenSearch 2.x** for full-text search with custom edge-ngram analyzer
-- **QQQ Framework**: Entities, processes, customizers, permissions, API layer
-
-### Module Organization
-
-```
-qbit-quick-search/
-  src/main/java/com/kingsrook/qbits/quicksearch/
-    QuickSearchQBitConfig.java        -- Configuration and validation
-    QuickSearchQBitProducer.java      -- Produces tables, processes, customizers
-    QuickSearchQBitContext.java       -- Static runtime context
-    QuickSearchableTableConfig.java   -- Per-table index configuration
-    SearchableFieldConfig.java        -- Config-driven field definition
-    SearchableTableConfig.java        -- Config-driven table definition
-    annotations/                      -- @QuickSearchable, @QuickSearchField
-    model/                            -- QuickSearchIndex, QuickSearchIndexRun entities
-    opensearch/                       -- OpenSearch client, document model, bulk results
-    actions/                          -- QuickSearchAction, input/output DTOs
-    processes/                        -- AbstractIndexingStep, BasepullIndexStep, FullReindexStep
-    customizers/                      -- Post-insert, post-update, post-delete customizers
-    publisher/                        -- IndexEventPublisher interface, SynchronousIndexEventPublisher
+```xml
+<dependency><groupId>software.amazon.awssdk</groupId><artifactId>apache-client</artifactId></dependency>
+<dependency><groupId>software.amazon.awssdk</groupId><artifactId>sts</artifactId></dependency> <!-- only for awsAssumeRoleArn -->
 ```
 
-## Getting Started
-
-### Prerequisites
-
-- **Java 21+**
-- **Maven 3.8+**
-- **OpenSearch 2.x** instance
-- **QQQ Application** (this is a QBit, not a standalone application)
-
-### Usage
-
-#### Maven dependency
+## Getting started
 
 ```xml
 <dependency>
     <groupId>com.kingsrook.qbits</groupId>
     <artifactId>qbit-quick-search</artifactId>
-    <version>0.2.0-SNAPSHOT</version>
+    <version>1.1.0-RC.1</version>
 </dependency>
 ```
 
-#### Annotate entity classes
+`1.1.0-RC.1` is the current release candidate (the 1.1 line, published from `release/1.1`; it needs QQQ 4.1.0-RC.2 or later, see [Upgrading from 1.0](#upgrading-from-10)). Hosts staying on the 1.0 line use `1.0.0-RC.2`; `1.0.0` follows once host applications have adopted it (#12).
+
+Annotate entity classes:
 
 ```java
 @QuickSearchable(tableName = "customer")
@@ -97,265 +72,409 @@ public class Customer
 }
 ```
 
-#### Minimal setup
+Produce the QBit (after your backends, tables and scheduler are in the instance):
 
 ```java
 QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("yourBackendName")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
+   .withBackendName("rdbms")
+   .withOpensearchUrl("https://search.example.com:443")
+   .withAuthMode(QuickSearchAuthMode.BASIC)
+   .withOpensearchUsername("${env.OPENSEARCH_USER}")
+   .withOpensearchPassword("${env.OPENSEARCH_PASSWORD}")
    .withOpensearchIndexName("my-app-search")
+   .withSchedulerName("myScheduler")
    .withSearchableEntityClasses(List.of(Customer.class, Order.class));
 
-new QuickSearchQBitProducer()
-   .withConfig(config)
-   .produce(qInstance);
+new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
 ```
 
-#### With authentication and SSL
+The producer implements QQQ's `QBitMetaDataProducer`, registers its own `QBitMetaData` (`com.kingsrook.qbits:quick-search`), and adds its tables, processes and app to the instance. Do not add a `QBitMetaData` for it yourself.
+
+Tables from other QBits or jars, without annotations:
 
 ```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("yourBackendName")
-   .withOpensearchHost("search.example.com")
-   .withOpensearchPort(443)
-   .withUseSsl(true)
-   .withOpensearchUsername("admin")
-   .withOpensearchPassword("secretPassword")
-   .withOpensearchIndexName("my-app-search")
-   .withSearchableEntityClasses(List.of(Customer.class));
-```
-
-#### With custom publisher (async indexing)
-
-```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("yourBackendName")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
-   .withOpensearchIndexName("my-app-search")
-   .withSearchableEntityClasses(List.of(Customer.class))
-   .withIndexEventPublisher(new RabbitMqIndexEventPublisher(connectionFactory));
-```
-
-#### Config-driven table indexing
-
-When tables come from third-party QBit jars (like qbit-crm or qbit-wms), you cannot add annotations to their entity classes. Use `SearchableTableConfig` and `SearchableFieldConfig` to declare those tables as searchable in your host application config.
-
-```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("rdbms")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
-   .withOpensearchIndexName("voyage-search")
-   .withSearchableTable("crmContact", List.of(
+config.withSearchableTable(new SearchableTableConfig("crmContact", List.of(
       new SearchableFieldConfig("firstName").withWeight(2),
       new SearchableFieldConfig("lastName").withWeight(2),
-      new SearchableFieldConfig("email").withWeight(3),
-      new SearchableFieldConfig("phone").withWeight(1)))
-   .withSearchableTable(new SearchableTableConfig("crmFormSubmission", List.of(
-      new SearchableFieldConfig("email").withWeight(3),
-      new SearchableFieldConfig("firstName").withWeight(1),
-      new SearchableFieldConfig("message").withWeight(1)))
-      .withRecordLabelFormat("%s - %s", "formType", "email"));
+      new SearchableFieldConfig("email").withWeight(3)))
+   .withRecordLabelFormat("%s %s", "firstName", "lastName"));
 ```
 
-`withSearchableTable()` accepts either a `(String tableName, List<SearchableFieldConfig> fields)` shorthand or a full `SearchableTableConfig` for advanced options.
+Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden and password fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull, and the QBit logs a warning for it at startup. Set `basepullTimestampField` to null on a config-driven table to opt out without the warning.
 
-##### SearchableFieldConfig
-
-| Method | Description |
-|--------|-------------|
-| `new SearchableFieldConfig(fieldName)` | Constructor; QQQ field name is required |
-| `.withWeight(int)` | Boost factor for relevance ranking (default 1) |
-| `.withIncludeLabel(boolean)` | Prefix indexed text with the field label (default false) |
-
-##### SearchableTableConfig
-
-| Method | Description |
-|--------|-------------|
-| `new SearchableTableConfig(tableName, fields)` | Constructor; table name and at least one field required |
-| `.withBasepullIntervalMinutes(int)` | Override basepull schedule (defaults to config-level setting) |
-| `.withBasepullTimestampField(String)` | Change detection field (default `"modifyDate"`) |
-| `.withEnabledByDefault(boolean)` | Whether indexing starts active (default `true`) |
-| `.withRecordLabelFormat(String format, String... fields)` | `String.format` pattern and field names for search result display label |
-
-##### Record label override
-
-By default, search results use QQQ's record label for display. Config-driven tables can override this with `withRecordLabelFormat()`. The format string is passed to `String.format()` with field values pulled from the record in the order specified.
+### Searching
 
 ```java
-new SearchableTableConfig("crmFormSubmission", fields)
-   .withRecordLabelFormat("%s - %s", "formType", "email");
-// produces labels like "Contact Form - user@example.com"
-```
-
-#### Mixing annotations and config
-
-Annotations and config-driven declarations can be used together. Both paths merge during `produce()`. Each table must appear in only one source; duplicate table names across annotation and config sources will produce a validation error.
-
-```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("rdbms")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
-   .withOpensearchIndexName("my-app-search")
-   .withSearchableEntityClasses(List.of(Customer.class, Order.class))
-   .withSearchableTable("crmContact", List.of(
-      new SearchableFieldConfig("firstName").withWeight(2),
-      new SearchableFieldConfig("lastName").withWeight(2),
-      new SearchableFieldConfig("email").withWeight(3)));
-```
-
-## Searching
-
-Once the QBit is produced, use `QuickSearchAction` to execute searches:
-
-```java
-QuickSearchOutput output = new QuickSearchAction().execute(
-   new QuickSearchInput()
-      .withSearchTerm("widget")
-      .withLimit(25)
-      .withOffset(0));
-
-// Check results
-System.out.println("Total hits: " + output.getTotalHits());
-System.out.println("Has more: " + output.getHasMore());
+QuickSearchOutput output = new QuickSearchAction().execute(new QuickSearchInput()
+   .withSearchTerm("widget")
+   .withTableNames(List.of("customer", "order"))   // optional
+   .withLimit(25)
+   .withOffset(0));
 
 for(QuickSearchResult result : output.getResults())
 {
-   System.out.println(result.getTableName() + ":" + result.getRecordId()
-      + " - " + result.getRecordLabel()
-      + " (score: " + result.getScore() + ")");
+   result.getTableName(); result.getTableLabel(); result.getRecordId();
+   result.getRecordLabel(); result.getScore(); result.getHighlightSnippet();
 }
+output.getTotalHits(); output.getTotalHitsIsLowerBound(); output.getHasMore();
 ```
 
-To filter results to a single table:
+`limitPerTable` returns up to that many hits from each table instead of one ranked list. Terms shorter than 2 characters return nothing; longer than 100 characters are rejected. `limit` is capped at `maxSearchLimit` and `offset + limit` at 10,000.
+
+`offset` skips N accessible results: with record security locks on, hits a lock hides are not counted, so consecutive pages never overlap or skip a visible hit, and with `limitPerTable` the offset applies within each table. With locks on, `totalHits` counts hits confirmed accessible so far; `totalHitsIsLowerBound` is true when the scan stopped before reading every candidate (page full, result window, or scan budget). A single-list search reads at most 10,000 raw hits in batches of at most 1,000. In per-table mode the T tables share that budget, each getting max(10,000 / T, 2 x `limitPerTable`) hits, so one call reads at most max(10,000, T x 2 x `limitPerTable`) raw hits. Paging within a table stops past that share of raw hits, as it does past 10,000 in a single list: `hasMore` turns false and `totalHitsIsLowerBound` stays true to signal unreachable matches.
+
+Search runs as the current `QContext` session: tables the session cannot read are skipped, and with `applyRecordSecurityLocks` (default) hits are re-read through `QueryAction` so record security locks apply.
+
+### Core search (`POST /qqq/v1/search`)
+
+QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v1/search` and used by the Next UI's global search box): a case-insensitive contains-query over each table's `searchFields`. With `serveCoreRecordSearch` (default `true`) the producer registers `QuickSearchRecordSearchProvider` on the instance, and core asks it to search every table Quick Search indexes. The API and the UI do not change.
+
+- **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quick_search_index`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
+- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights), skipping the request's `offset`; each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order, each with its score and a highlight of the matching `fieldValues.*` fields (plain text plus match ranges), and the table's total hits (counted to 10,000). Core re-reads the ids by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, records deleted since they were indexed drop out, and only highlights of fields the user can see (where core's own highlighting found no match) reach the response, with `tables[].totalCount` and `hasMore`. The index holds no record modify dates, so `sort: RECENT` gets relevance order for indexed tables.
+- **Failure.** The OpenSearch request is cancelled after `recordSearchTimeoutMillis` (default 2500). Any failure (timeout, cluster down, index missing) makes core search those tables with its own contains-query instead, so search degrades but never fails. After a failure, Quick Search claims no tables for 30 seconds, so a down cluster does not add a timeout to every search.
+- **Opting out.** `withServeCoreRecordSearch(false)` leaves core search and `searchFields` untouched. If the host already registered another record-search provider, the QBit keeps it and logs a warning (an instance has one provider).
+
+`QuickSearchAction` remains the direct API (all hits in one ranked list, `searchableText` highlights, pagination across tables).
+
+### Processes
+
+| Process | Trigger | What it does |
+|---|---|---|
+| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Recovers a full reindex that stopped without finishing, replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
+| Quick Search Reconcile Index | Cron via `reconcileCronExpression` when `schedulerName` is set; also on demand (optional `tableName` input) | Re-indexes everything in place, removes documents without a source row and documents of unconfigured tables |
+| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place. Fails to start while another full reindex is running |
+
+Without `schedulerName` the processes exist but nothing runs them; the QBit logs a warning at startup.
+
+### Managing searchable tables (data-driven, 1.1.0-RC.3)
+
+Which tables and fields are indexed is data: `quick_search_index` (one row per table) and `quick_search_index_field` (its fields: weight, role, analyzer, snippet and label flags), edited in the Quick Search admin app with the standard screens and these record actions (each with a review step; none is in the nav):
+
+- **Add Search Index** (Quick Search Index, Actions): pick a readable, eligible table, review the estimated record count and the fields it starts with, save; the first build is queued.
+- **Searchable Fields** on the index record: add, edit or remove fields; then **Apply Search Index Changes** previews the changes and rebuilds.
+- **Disable Search Index** and **Remove Search Index**: stop indexing at once and delete or keep the documents. Removals are remembered, so code defaults do not bring them back.
+- **Cancel Index Run** (Quick Search Index Run): stops a running reindex after its current batch (`CANCELLED`, with counts).
+- **Pause Indexing** and **Resume Indexing** (Quick Search Settings): pause all indexing while search keeps serving; changes are indexed after resume.
+
+`@QuickSearchable` entities and `searchableTables` only seed these records (self-priming at startup: new tables and fields are added and built automatically; rows an admin edited are never overwritten). The precedence rules, the DDL for the new tables and the upgrade steps are in `docs/MIGRATION-1.1.md`, "Data-driven configuration".
+
+### Scheduling
+
+The schedules run only when the host starts through QQQ's `QApplicationLauncher` (`com.kingsrook.qqq.middleware.javalin.QApplicationLauncher`, in `qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` (because the QBit's processes are scheduled), then the instance's runtime services. A host that starts only a server, or builds the `QInstance` itself, gets the processes but nothing runs them on a schedule, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
+
+Basepull's fixed interval runs on any QQQ scheduler. `reconcileCronExpression` needs a scheduler that supports cron, which in QQQ 4.1 means a `QuartzSchedulerMetaData`; a `SimpleSchedulerMetaData` runs fixed intervals only. The expression uses Quartz syntax, seconds first (`0 0 3 * * ?` is 03:00 every day), and needs `reconcileCronTimeZoneId`, a Java time zone ID such as `UTC` or `America/Chicago`; QQQ has no default zone. `produce()` rejects a cron without a zone, an invalid expression or zone, and a cron on a scheduler in the instance that does not support cron.
 
 ```java
-QuickSearchOutput output = new QuickSearchAction().execute(
-   new QuickSearchInput()
-      .withSearchTerm("blue")
-      .withTableName("customer")
-      .withLimit(10));
+qInstance.addScheduler(new QuartzSchedulerMetaData().withName("quartz").withProperties(quartzProperties));
+
+config.withSchedulerName("quartz")
+   .withReconcileCronExpression("0 0 3 * * ?")
+   .withReconcileCronTimeZoneId("America/Chicago");
 ```
 
-### Triggering a Full Reindex
+### Diagnostic tools
 
-The full reindex process can be triggered programmatically:
+With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate the search and debug it from the admin UI. They are table and record actions (never left-nav entries), gated by permissions, recorded in `quick_search_diagnostic_event`, limited to `diagnosticRunsPerMinute` runs per user and tool, and read-only except Reindex in Search.
 
-```java
-RunBackendStepInput input = new RunBackendStepInput();
-input.addValue("tableName", "customer"); // optional: omit to reindex all tables
-RunBackendStepOutput output = new RunBackendStepOutput();
-new FullReindexStep().run(input, output);
-```
+| Tool | Where | What it does |
+|---|---|---|
+| Search Playground | Quick Search Index, Actions (select rows to pre-fill their tables) | Runs a term the way the global search does, as you: the path each table takes (OpenSearch provider, core database search, or not searched) and why; the provider's own request, timed; the global search results you would see; per indexed table, the production query with scores, highlights, matched fields, which hits you can read and which the global search shows, the query DSL and the raw response (hits you cannot read are redacted); the term's tokens; and `explain` for a hit (`table:id`). The form stays open, so you can change the term and run again. |
+| Validate Search Index | Quick Search Index, Actions; also scheduled by `validationCronExpression` | Cluster and index health, the alias, leftover physical indexes from interrupted rebuilds, the live mapping (version, field types, analyzers, the `fieldValues` template) and analysis settings, then per table: database rows vs documents, a sample of records checked present and current in the index, a random sample of documents checked to still have a record, the index row's status and last run, and waiting failed events. Each finding is OK, INFO, WARN or FAIL with what to do. The report is saved as a `quick_search_validation_run` row (HTML for the record view, JSON for tools). Database reads run as the system user so counts compare like with like; the report holds counts, ids and field names, never record values. One validation runs at a time per server. |
+| Test Search Analyzer | Quick Search Index, Actions | `_analyze` on the live index: the tokens of a text as indexed (edge n-grams of 2 to 20 characters) and as a query, and, given a term, whether every term token is among the text's indexed tokens. |
+| Inspect Search Document | Every indexed table's records (Actions), and Quick Search Failed Event rows | The record's document in the index next to the document indexing would write from the database now, field by field (current, stale or missing), with versions and the raw document. Enter a term to see why the record does or does not match it: term length, table enabled, read permission, document present and current, each term token found or not (and why: one character, longer than 20, not in the text, or only in the database's newer values), `explain`, and the record's rank against the number of ids the global search takes. |
+| Reindex in Search | Every indexed table's records, and Quick Search Failed Event rows | Rebuild the selected records' documents from the database now (versioned writes, so a newer document is never overwritten); a record that no longer exists has its document removed; a record you cannot read is skipped. Asks to confirm. Needs the admin permission. |
+| Search Diagnostics widget | The Quick Search Admin app home (hosts with their own home add `quickSearchDiagnostics`) | Whether the global search is served by OpenSearch on this server or backing off (and why), failure and fallback counts, the latest search failures from every server, and the latest validation run. |
 
-Or through the QQQ admin UI via the "Full Reindex" process.
+Search failures are recorded too: when the provider's OpenSearch request fails or times out (and core falls back to its own search), or `QuickSearchAction`'s search fails, a `SEARCH_FAILED` event keeps the tables, duration, error class and message, and the server. The user's term is never stored, only its length. At most 120 events are written per server per minute.
 
-## Data Model
+Permissions: the read-only tools and the widget use `debugPermissionRules` (default `quick_search_debug.hasAccess`); Reindex in Search uses `adminPermissionRules` (`quick_search_admin.hasAccess`); the two history tables use `tablePermissionRules` (grant `.read` only: rows are written by the QBit). Impersonation ("search as user X") is deliberately not offered: a fabricated session would not carry the real user's security key values, so its results would mislead.
 
-### Tables (2)
+### Record developer panel
 
-| Table | Description |
-|-------|-------------|
-| `quickSearchIndex` | Per-table index configuration, status, and basepull tracking |
-| `quickSearchIndexRun` | Individual indexing run history with record counts and error tracking |
+With QQQ 4.1.0-RC.4 or later the QBit registers a record developer panel provider (`QuickSearchDeveloperPanelProvider`, id `quickSearch`, in QQQ's `DeveloperPanelsMetaData`). Frontends that render developer panels (the Next UI, `/app/{table}/{id}/dev`) show a "Quick Search" panel on every record of an indexed table, to users who may read Quick Search Index. Every part is permission-checked:
 
-### Processes (2)
+| Part | Shown when |
+|---|---|
+| Index: enabled, status, record count, last basepull, last full reindex, searchable fields, link to the index row | always (read on Quick Search Index) |
+| Latest index run: type, status, start and end, processed, indexed, errors | the user may read Quick Search Index Run |
+| Search document: verdict (current, stale, missing) and a Field / Database / Index / Match table | `enableDiagnosticTools` and the debug permission (`debugPermissionRules`); otherwise a note says why it is missing |
+| Failed index events of the record | the user may read Quick Search Failed Event and the record has any |
+| Actions: Inspect search document, Reindex this record | the record actions exist and the user may run them (debug and admin permission) |
+| Link to Quick Search Admin | the user may open the app |
 
-| Process | Description |
-|---------|-------------|
-| Basepull Index | Scheduled: queries each source table for records modified since last run, indexes in batches |
-| Full Reindex | On-demand: deletes and re-indexes all records for one or all tables |
+The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned off in Quick Search Index), or OpenSearch unavailable. When OpenSearch cannot be reached the panel still shows the database-side state, with a warning in place of the document comparison. Hosts without a developer-panel frontend are unaffected.
 
-## Configuration Reference
+## Configuration reference
 
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `backendName` | `String` | required | QQQ backend for operational tables |
-| `opensearchHost` | `String` | required | OpenSearch hostname |
-| `opensearchPort` | `Integer` | required | OpenSearch port |
-| `opensearchIndexName` | `String` | required | OpenSearch index name |
-| `searchableEntityClasses` | `List<Class<?>>` | none | Entity classes with `@QuickSearchable` |
-| `searchableTables` | `List<SearchableTableConfig>` | none | Config-driven table definitions |
-| `opensearchUsername` | `String` | none | HTTP basic auth username |
-| `opensearchPassword` | `String` | none | HTTP basic auth password |
-| `useSsl` | `Boolean` | `false` | Use HTTPS |
-| `tableNamePrefix` | `String` | none | Prefix for produced table names |
-| `enableRealTimeIndexing` | `Boolean` | `true` | Register post-insert/update/delete customizers |
-| `enableScheduledProcesses` | `Boolean` | `true` | Register scheduled basepull process |
-| `defaultBasepullIntervalMinutes` | `Integer` | `5` | Default basepull schedule |
-| `bulkBatchSize` | `Integer` | `500` | Documents per OpenSearch bulk request |
-| `sourceBatchSize` | `Integer` | `1000` | Records per source table query batch |
-| `indexEventPublisher` | `IndexEventPublisher` | sync | Custom publisher for async transport |
+| Field | Default | Description |
+|---|---|---|
+| `backendName` | required | QQQ backend for the operational tables |
+| `opensearchUrl` | | `https://host[:port]`; preferred over `opensearchHost`, `opensearchPort`, `useSsl`; credentials in the URL are rejected |
+| `opensearchHost`, `opensearchPort`, `useSsl` | `useSsl=false` | Legacy connection fields |
+| `opensearchIndexName` | required | Alias name owned by this QBit; lowercase, OpenSearch naming rules |
+| `authMode` | inferred | `NONE`, `BASIC` (inferred when credentials are set), `AWS_SIGV4` |
+| `opensearchUsername`, `opensearchPassword` | | BASIC credentials; `${env.X}` references recommended (an empty value counts as missing); refused over plain HTTP unless `allowPlaintextCredentials` |
+| `awsRegion`, `awsServiceName`, `awsAssumeRoleArn` | region from `AWS_REGION`; `es` | AWS_SIGV4 settings |
+| `tls` | JVM defaults | `caCertificatePath` or `trustStorePath`/`trustStorePassword`/`trustStoreType`, `keyStorePath`/`keyStorePassword`/`keyStoreType` (mTLS), `hostnameVerification` (`false` logs a warning), `insecureSkipVerify` (loopback only unless `allowInsecureInProduction`) |
+| `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections` | 5000, 60000, 30 | Transport settings |
+| `transportCustomizer` | | `QCodeReference` to an `OpenSearchTransportCustomizer` (bearer tokens, API keys, interceptors) |
+| `startupMode` | `FAIL_FAST` | `FAIL_FAST` fails `produce()` when the cluster is unreachable; `DEGRADED` boots and retries on first use |
+| `enableRealTimeIndexing` | `true` | Register the record-change listener |
+| `enableBasepullProcess`, `enableMaintenanceProcesses` | `true` | Register basepull; register full reindex and reconcile. `enableScheduledProcesses` is a deprecated alias for both |
+| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none, none | Scheduling; a cron needs the time zone ID and a Quartz scheduler, and nothing runs without `QApplicationLauncher` (see [Scheduling](#scheduling)) |
+| `basepullOverlapSeconds` | 300 | Re-read window before the previous run start |
+| `defaultBasepullIntervalMinutes` | 5 | Per-table due interval for config-driven tables |
+| `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
+| `fullReindexStaleMinutes` | 30 | A running full reindex that shows no sign of life for this long is treated as stopped: it no longer blocks a new one, and basepull recovers its tables. Keep it well above the time one page of `sourceBatchSize` records takes; null means the default |
+| `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
+| `indexReplicas` | null (automatic) | Index `number_of_replicas`; null means `min(maxAutoReplicas, data nodes - 1)`, so a single node gets 0 and stays green. Set on new indexes, reconciled on existing ones at startup |
+| `maxAutoReplicas` | 1 | Most replicas automatic sizing asks for |
+| `maxFieldLength` | 10000 | Truncate each indexed value |
+| `maxSearchLimit` | 100 | Largest page size |
+| `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
+| `serveCoreRecordSearch` | `true` | Register `QuickSearchRecordSearchProvider` so core `POST /qqq/v1/search` uses OpenSearch for indexed tables, and give those tables `searchFields` when they have none |
+| `recordSearchTimeoutMillis` | 2500 | Wall-clock limit on the OpenSearch request behind a core search; past it, core falls back to its own search |
+| `enableDiagnosticTools` | `true` | Add the diagnostic tools, the `quick_search_diagnostic_event` and `quick_search_validation_run` tables and the diagnostics widget |
+| `enableRecordDiagnosticActions` | `true` | With the tools on, add Inspect Search Document and Reindex in Search to every indexed table |
+| `validationCronExpression`, `validationCronTimeZoneId` | none | Schedule Validate Search Index; same rules as the reconcile schedule (a cron-capable scheduler such as Quartz, a valid Quartz expression, a time zone ID) |
+| `validationSampleSize` | 50 | Records and documents sampled per table by validation (at most 1000) |
+| `diagnosticRunsPerMinute` | 30 | Per-user, per-tool limit on diagnostic runs |
+| `debugPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quick_search_debug` | Rules for the read-only diagnostic tools and the widget |
+| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quick_search_admin` | Rules for the app, the maintenance processes and Reindex in Search |
+| `tablePermissionRules` | `READ_WRITE_PERMISSIONS` | Rules for the operational tables |
+| `tableMetaDataCustomizer` | | Applied to every produced table (backend details, for example) |
+| `searchableEntityClasses`, `searchableTables` | at least one | What to index |
+| `indexEventPublisher` | synchronous | Replace the publisher (experimental SPI; see below) |
 
-At least one of `searchableEntityClasses` or `searchableTables` must be provided. Both can be used together.
+`@QuickSearchable(tableName, fields, basepullIntervalMinutes, basepullTimestampField, enabledByDefault)` and `@QuickSearchField(weight, includeLabel)` keep their 0.x meaning.
 
-## Annotations
+## Operational tables
 
-### `@QuickSearchable`
+- `quick_search_index`: one row per table with `source` (`CODE` or `OWNER`), `ownerEdited`, `enabled` (honoured by indexing and search; a change applies at once), `basepullIntervalMinutes`, `basepullTimestampField`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`; `PENDING_BUILD` when a build is queued and `BUILDING` while the basepull builds it; `NEEDS_REINDEX` when the fields changed since the last build; `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
+- `quick_search_index_field`: the searchable fields of each index row: `fieldName`, `weight`, `role` (`DISPLAY`, `TITLE`, `SUBTITLE`, `BODY`, `KEYWORD`, `EXACT`), `analyzer` (`PREFIX`, `WORD`, `ENGLISH`, `EXACT`; blank: by role), `includeInSnippet`, `includeLabel`, `sortOrder`, `source`, `ownerEdited`. Unique on (`quickSearchIndexId`, `fieldName`).
+- `quick_search_settings`: one row: `indexingPaused` (with `pausedAt`, `pausedBy`, `pauseReason`) and the last self-priming report (`lastPrimedAt`, `lastPrimingSummary`).
+- `quick_search_config_tombstone`: tables (blank `fieldName`) and fields an admin removed; priming never re-adds them.
+- `quick_search_index_run`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status` (`RUNNING`, `COMPLETED`, `FAILED`, `CANCELLED`), counts, `errorMessage`, `cancelRequested`, `cancelRequestedBy`.
+- `quick_search_diagnostic_event` (with `enableDiagnosticTools`): `eventType` (`SEARCH_FAILED`, `TOOL_RUN`, `REINDEX`, `RATE_LIMITED`), `source`, `outcome`, `userName`, `tableNames`, `searchTerm` (tool runs only), `durationMillis`, `errorClass`, `errorMessage` (text), `detail` (text), `hostName`, `createDate`. Purged after `runHistoryRetentionDays`.
+- `quick_search_validation_run` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
+- `quick_search_failed_event`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
 
-| Attribute | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `tableName` | `String` | required | QQQ table name |
-| `fields` | `String[]` | `{}` | Fallback field list if no `@QuickSearchField` annotations |
-| `basepullIntervalMinutes` | `int` | `5` | Basepull schedule for this table |
-| `basepullTimestampField` | `String` | `"modifyDate"` | Field for change detection |
-| `enabledByDefault` | `boolean` | `true` | Active on first produce |
+### Starter defaults and empty-state help
 
-### `@QuickSearchField`
+This qbit ships no DDL and no seed changelog: startup self-priming (`QuickSearchConfigPrimer`) is its seed mechanism. On a fresh database it writes a `quick_search_index` row and `quick_search_index_field` rows for every table the code declares searchable, and the single `quick_search_settings` row (id 1). `quick_search_config_tombstone` and the five log and history tables start empty by design (rows appear when an admin removes a table or field, or when the qbit runs).
 
-| Attribute | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `weight` | `int` | `1` | Boost factor for search relevance |
-| `includeLabel` | `boolean` | `false` | Prefix value with field label in indexed text |
+Real-time error counts on Quick Search Index show failures since the last successful full reindex or basepull. Each failed event increases the count; deferred changes while indexing is paused do not. Counter updates are best effort and serialized across listeners in one JVM; concurrent application nodes can still race because QQQ has no arithmetic update API.
 
-## Companion QBit Integration
+## Extending
 
-### qbit-crm
+`IndexEventPublisher` lets a host replace the synchronous publisher with a queue. In 1.0 the SPI is experimental: `IndexEvent` carries the table, record id, action and the record snapshot, but no sequence number, so an asynchronous consumer must order INDEX and DELETE for the same record itself. `OpenSearchTransportCustomizer` customizes the HTTP transport.
 
-Index `crmContact`, `crmCompany`, and `crmDeal` for full-text global search across CRM records.
-
-### qbit-wms
-
-Index `wmsItem` and `wmsLocation` for warehouse item and location search.
-
-## Testing
+## Building
 
 ```bash
-mvn test                    # Run all 174 unit tests
-mvn verify                  # Run unit tests + integration tests (requires Docker)
-mvn test -Dtest=ClassName   # Run single test class
+./mvnw test                                 # unit tests (memory backend, mocked client)
+CI=true ./mvnw verify                       # plus Testcontainers integration tests and the coverage gate (Docker required)
+./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
 ```
 
-### Coverage
+Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports QQQ 4.1.0-RC.8 through the default-active `qqq-snapshot` profile ahead of the parent BOM (override only with a compatible released version using `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
 
-- **70%+ instruction coverage** (JaCoCo enforced)
-- **90%+ class coverage** (JaCoCo enforced)
-- All unit tests run in-memory via MemoryRecordStore (no OpenSearch required)
-- Integration tests use Testcontainers with real OpenSearch
+## Upgrading from 1.0
 
-## Documentation
+See [docs/MIGRATION-1.1.md](docs/MIGRATION-1.1.md). In short: 1.1 needs the QQQ release with the record-search provider SPI; core search starts using OpenSearch for indexed tables by default, and indexed tables gain `searchFields` (set `serveCoreRecordSearch(false)` to keep 1.0 behaviour).
 
-- **[QQQ Wiki](https://github.com/Kingsrook/qqq/wiki)** - Framework documentation
-- **[QBit Development Guide](https://github.com/Kingsrook/qqq/wiki/QBit-Development)** - How QBits work
+## Upgrading from 0.x
 
-## Contributing
+See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, grant `quick_search_admin.hasAccess` and the table permissions, add the new operational table columns and `quick_search_failed_event`, and run a full reindex once so the index gets the new mapping.
 
-QBit Quick Search is open source and welcomes contributions.
+## Adopting in another app
 
-- **[Report Issues](https://github.com/QRun-IO/qbit-quick-search/issues)** - Bug reports and feature requests
-- **[QQQ Contribution Guide](https://github.com/Kingsrook/qqq/wiki/Contribution-Guidelines)** - How to contribute
+Follow [Getting started](#getting-started), the [Configuration reference](#configuration-reference) and [Operational tables](#operational-tables) (the host application creates the three tables; this QBit ships no DDL), then run a full reindex. Hosts on 1.0 or 0.x should read the upgrade sections above.
 
-## About Kingsrook
-
-QBit Quick Search is built by **[Kingsrook](https://qrun.io)** - making engineers more productive through intelligent automation and developer tools.
-
-- **Website**: [https://qrun.io](https://qrun.io)
-- **Contact**: [contact@kingsrook.com](mailto:contact@kingsrook.com)
-- **GitHub**: [https://github.com/QRun-IO](https://github.com/QRun-IO)
+See also the [QQQ adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) for the full per-feature list (QQQ upgrade, Next dashboard, QBits, migrations, permissions).
 
 ## License
 
-This project is licensed under the **Apache License, Version 2.0** - see the [LICENSE](LICENSE) file for details.
+Apache License, Version 2.0. See [LICENSE](LICENSE).
+
+## UI API for hosts
+
+The `quick-search` extension registers both status and suggestions in the committed
+[OpenAPI fragment](src/main/resources/openapi/qqq-v1/quick-search.yaml).
+
+### Typeahead suggestions
+
+`GET /qqq/v1/qbit/quick-search/suggest` (`getQuickSearchSuggestions`) requires a
+signed-in session. Quick Search admin permission is not required. Java hosts use
+`QuickSearchSuggestService.suggest(q, limitPerTable, maxTables, tables)` in the caller's
+`QContext`.
+
+| Query parameter | Meaning |
+| --- | --- |
+| `q` | Required text, trimmed with consecutive whitespace collapsed. Fewer than 2 characters returns empty groups; more than 100 returns HTTP 400. |
+| `limitPerTable` | Integer 1..10; default 5. |
+| `maxTables` | Integer 1..20; default 8. |
+| `tables` | Optional comma-separated table metadata names. Unknown, disabled and unreadable tables are omitted; an empty filter returns no groups. |
+
+```json
+{
+  "query": "blue wid",
+  "groups": [{
+    "tableName": "product",
+    "tableLabel": "Products",
+    "hasMore": true,
+    "items": [{"recordId": "42", "label": "Blue Widget", "subtitle": "Small"}]
+  }],
+  "truncated": false
+}
+```
+
+The existing edge-ngram query supports the last word's prefix. Groups are ordered
+by their best accessible score, with per-table relevance order preserved. Labels
+fall back to `tableLabel #recordId`; subtitles are nullable and come only from the
+already-filtered result. Scores, highlights and display values are never returned.
+The existing action enforces table READ permissions, enabled-table selection,
+configured record security locks (on by default), PHI/PII exclusions and protected
+primary keys.
+
+Suggestions make one bounded OpenSearch request per eligible table, fetching at
+most `2 * (limitPerTable + 1)` candidates under record locks (also subject to the
+host's `maxSearchLimit`). Locked rows can leave a group partially filled.
+`hasMore` confirms an extra accessible hit in this batch; false does not prove
+there are no further matches. `truncated` means more tables had accessible hits
+than `maxTables`. Missing runtime/client or unavailable search returns empty groups.
+Missing `q`, invalid integers and out-of-range limits return HTTP 400.
+
+### Administrative status
+
+Requires QQQ **4.1.0-RC.6**. Hosts should use the supported API rather than reach into
+`QuickSearchRuntime`, `QuickSearchOpenSearchClient`, diagnostic renderers, or qbit tables:
+
+```java
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatusService;
+
+// Runs in the host request's QContext, with the caller's QSession.
+QuickSearchStatus status = QuickSearchStatusService.get();
+String homeHref = status.homeHref();
+```
+
+UI clients use **`GET /qqq/v1/qbit/quick-search/status`**, operationId
+`getQuickSearchStatus`, extension `quick-search` version `1.1.0` (requires QQQ `4.1.0-RC.15`, UI API `1.6.0`). The endpoint returns
+that same immutable value as JSON; it requires the Quick Search app's configured
+`adminPermissionRules` (default `quick_search_admin.hasAccess`). Permission is checked
+before every read, including cache hits. No additional source-table permissions are
+required: like index validation, this is an administrator's aggregate view of the index.
+The committed contract is [quick-search.yaml](src/main/resources/openapi/qqq-v1/quick-search.yaml).
+
+The default core namespace is intentional: RC.6 permits `/qbit/quick-search` but rejects
+`/quick-search`, which is not an established legacy prefix. There were no existing
+Quick Search UI API routes to preserve. Core source is unchanged.
+
+- `state`: `OK`, `DEGRADED` (including paused indexing, backlog, incomplete diagnostics
+  or a refresh timeout), `DOWN` (unavailable health or red health), or `DISABLED`
+  (qbit absent or every persisted index disabled). Pausing indexing does not disable search.
+- `clusterReachable`, `clusterHealth`, `nodes`, `activeShards`, `unassignedShards` come
+  from the existing diagnostics health operation. Null means unknown/not probed.
+- `indexes` contains up to 100 persisted index rows. `documentCount` and `sourceCount`
+  come from the latest saved `IndexValidator` report, with `countsCheckedAt`. Without
+  matching validation, only the last indexing run's document count is available and
+  its observation time is unknown. These are **saved counts, not live counts**. A
+  missing source count must not be displayed as zero or “in sync.”
+- `pending` includes `PENDING` and `AWAITING_REINDEX` failed events; `exhausted` counts
+  failed events that gave up. At most 1000 backlog rows are read; `backlogTruncated`
+  makes both counts lower bounds. `tablesTruncated` indicates incomplete index rows.
+  These counters do not measure an external publisher's queue.
+- `lastSuccessfulSync` is the newest successful indexing watermark among the included
+  tables (basepull is start-anchored), not proof that all tables are current.
+  `lastError` is the newest available indexing/runtime search error or snapshot failure.
+- `checkedAt` is snapshot completion time. A per-runtime five-second cache bounds
+  polling cost; each refresh uses `DiagnosticsRateLimiter`'s per-user `STATUS` key.
+  Rate-limit refusal uses the existing diagnostics error (HTTP 400). A refresh waits
+  at most two seconds and retains at most one in-flight worker, even if a backend is
+  stuck. Runtime shutdown stops the worker. No source counts, record samples, full
+  validation, or per-table OpenSearch calls run on this polling path.
+
+Link to `status.homeHref()` instead of copying the Quick Search home. It includes the
+configured metadata prefix. `QuickSearchApi.APP_NAME` (`quickSearchAdmin`) and
+`QuickSearchApi.HOME_HREF` (`/app/quickSearchAdmin`) are available for default-name
+navigation. Keep detailed runs, failed events, validation and operator tools in the
+qbit's own home. [kof22 migration mapping](docs/STATUS-API-HOST-MIGRATION.md).
+
+## Health checks
+
+The QBit producer registers these checks with QQQ's central health registry. All use
+component `quick_search`, category `SEARCH`, a 60-second interval and a 5-second timeout.
+
+| Name | Kind | Interval | Reports |
+| --- | --- | --- | --- |
+| `quick_search.opensearch` | DEPENDENCY | 60s | Cluster reachability and colour: red is DOWN, yellow is DEGRADED, disabled is UNKNOWN. |
+| `quick_search.backlog` | DEPENDENCY | 60s | DEGRADED for any exhausted events, more than 100 pending events, or paused indexing. |
+| `quick_search.index.<indexName>` | DEPENDENCY | 60s | One check per persisted index row; non-active status, count mismatch, last error or real-time errors are DEGRADED. Disabled indexes are UNKNOWN. |
+
+Index names come from the row's table name, normalized to lowercase health identifiers.
+The supplier refreshes the set through core's normal discovery cycle. Checks reuse the
+five-second status cache and saved validation counts, with waits bounded by the runner's
+deadline. Incomplete measurements never become zero; indexes absent from the bounded
+100-row status snapshot report UNKNOWN. Backlog truncation reports DEGRADED.
+Details contain counts, timestamps and flags, never stored error text or credentials.
+
+The status endpoint, diagnostics widget and record developer panel remain available.
+This integration requires QQQ 4.1.0-RC.8 or later, which includes the central health SPI. The POM pins that release. For local offline verification:
+
+```powershell
+./mvnw.cmd -o verify
+```
+
+## Classified data
+
+Quick Search excludes fields classified as **PII** or **PHI**, including fields
+that inherit the table's classification. It also excludes hidden, sensitive,
+password/masked fields and every field with a `FieldSecurityLock`; a shared
+search index cannot enforce access to individual fields for each user. PUBLIC,
+CONFIDENTIAL and unclassified fields remain eligible under the usual field rules.
+
+The same rule applies to annotation and code configuration, admin field choices,
+real-time writes, basepull, rebuilds, replay, labels, snippets, analyzer copies and
+result display fields. An admin trying to add a protected field gets an error.
+Protected fields in existing or explicit code configuration are dropped, with one
+warning per table metadata lifetime; warnings never contain field values. A
+protected primary key prevents indexing the record or sending its ID for deletion.
+
+Protection is separate from search eligibility: boolean, blob, hidden and other
+unsupported fields alone do not trigger result filtering or remove record labels.
+Labels are omitted when their source fields are protected; unrelated protected
+fields do not erase safe labels. Unprotected hidden keys and timestamps can still
+provide record IDs and document versions.
+
+After upgrading, or after changing a field's classification or security metadata,
+run **Full Reindex** across all indexed tables to remove previously indexed values.
+Until then, tables containing protected fields are searched through allowed field
+values, and results are rebuilt without old combined labels or highlights.
+Diagnostic views also omit protected values and the raw response for these tables.
+This does not purge old OpenSearch backups or retained physical indexes; apply your
+host's retention policy to those copies.
+
+This release builds against QQQ **4.1.0-RC.7**. Support for the host's
+`AuditValueProtector` classification (`ClassifiedValues`, qqq#1182) and
+`FieldAccessHelper.getLockTier` (qqq#1172) is deferred until the qbit moves to RC.7.
+
+## In-app help
+
+The producer registers [help/qbit-quick-search.yaml](src/main/resources/help/qbit-quick-search.yaml)
+with QQQ's help registry (requires QQQ 4.1.0-RC.7 or later). The bundle covers
+Quick Search's own app, tables, actions, widgets and possible-value sources.
+Registration is metadata-only and works without a `helpContent` table. Hosts
+with help enabled load help content and call `HelpBundles.syncOnStartup(qInstance)`
+after producing all metadata; QQQ guards sync when the help table is absent.
+
+To customize wording or document deployment schedules, register your own
+`layer: HOST` bundle with the same entry keys and roles using
+`HelpBundles.register(qInstance, "help/my-host.yaml")`. HOST text takes precedence
+over QBIT text; edits made by users are preserved by sync. Host-specific record
+actions (Inspect Search Document and Reindex in Search on host tables) belong in
+the host bundle. Keep the bundle's literal version aligned with the POM revision.
+
+`QuickSearchTablesGuardrailTest.helpBundleIsComplete` uses `HelpBundleAssertions`
+to check owned objects, non-trivial fields, summary length and live references.
+The same test class checks registration without the help subsystem.

checkstyle/config.xml

@@ -6,18 +6,17 @@
   ~ contact@kingsrook.com
   ~ https://github.com/Kingsrook/
   ~
-  ~ This program is free software: you can redistribute it and/or modify
-  ~ it under the terms of the GNU Affero General Public License as
-  ~ published by the Free Software Foundation, either version 3 of the
-  ~ License, or (at your option) any later version.
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
   ~
-  ~ This program is distributed in the hope that it will be useful,
-  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
-  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  ~ GNU Affero General Public License for more details.
+  ~     https://www.apache.org/licenses/LICENSE-2.0
   ~
-  ~ You should have received a copy of the GNU Affero General Public License
-  ~ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
   -->
 
 <!DOCTYPE module PUBLIC

checkstyle/license.txt

@@ -5,16 +5,15 @@
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
  *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */

com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -1,288 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.customizers;
-
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Common interface used by all (core) TableCustomizer types (e.g., post-query,
- ** and {pre,post}-{insert,update,delete}.
- **
- ** Note that the abstract-base classes for each action still exist, though have
- ** been back-ported to be implementors of this interface.  The action classes
- ** will now expect this type, and call this type's methods.
- **
- *******************************************************************************/
-public interface TableCustomizerInterface
-{
-   QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
-
-   /*******************************************************************************
-    ** custom actions to run after a query (or get!) takes place.
-    **
-    *******************************************************************************/
-   default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postQuery is running...  Probably not expected!", logPair("tableName", queryInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions before an insert takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview field, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (the inputs to
-    ** the insert action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly manipulating values (`setValue`)
-    ** - possibly throwing an exception - if you really don't want the insert operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
-    *******************************************************************************/
-   default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      try
-      {
-         return (preInsertOrUpdate(insertInput, records, isPreview, Optional.empty()));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
-   {
-      return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions after an insert takes place.
-    **
-    ** General implementation would be, to iterate over the records (the outputs of
-    ** the insert action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly throwing an exception - though doing so won't stop the update, and instead
-    **   will just set a warning on all of the updated records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back to the caller.
-    *******************************************************************************/
-   default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
-   {
-      try
-      {
-         return (postInsertOrUpdate(insertInput, records, Optional.empty()));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    ** custom actions before an update takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview field, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (the inputs to
-    ** the update action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly manipulating values (`setValue`)
-    ** - possibly throwing an exception - if you really don't want the update operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
-    **
-    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
-    ** available (if the backend supports it)
-    *******************************************************************************/
-   default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      try
-      {
-         return (preInsertOrUpdate(updateInput, records, isPreview, oldRecordList));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    ** custom actions after an update takes place.
-    **
-    ** General implementation would be, to iterate over the records (the outputs of
-    ** the update action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
-    ** - possibly throwing an exception - though doing so won't stop the update, and instead
-    **   will just set a warning on all of the updated records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back to the caller.
-    **
-    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
-    ** available (if the backend supports it).
-    *******************************************************************************/
-   default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      try
-      {
-         return (postInsertOrUpdate(updateInput, records, oldRecordList));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    ** Custom actions before a delete takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview param, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (which the DeleteAction
-    ** would look up based on the inputs to the delete action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly throwing an exception - if you really don't want the delete operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) - this is how errors
-    **   and warnings are propagated to the DeleteAction.  Note that any records with
-    **   an error will NOT proceed to the backend's delete interface - but those with
-    **   warnings will.
-    *******************************************************************************/
-   default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      LOG.info("A default implementation of preDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** Custom actions after a delete takes place.
-    **
-    ** General implementation would be, to iterate over the records (ones which didn't
-    ** have a delete error), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
-    ** - possibly throwing an exception - though doing so won't stop the delete, and instead
-    **   will just set a warning on all of the deleted records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back
-    **   to the caller - this is how errors and warnings are propagated .
-    *******************************************************************************/
-   default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
-      return (records);
-   }
-
-
-   /***************************************************************************
-    ** Optional method to override in a customizer that does the same thing for
-    ** both preInsert & preUpdate.
-    ***************************************************************************/
-   default List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      throw NotImplementedHereException.instance;
-   }
-
-
-   /***************************************************************************
-    ** Optional method to override in a customizer that does the same thing for
-    ** both postInsert & postUpdate.
-    ***************************************************************************/
-   default List<QRecord> postInsertOrUpdate(AbstractActionInput input, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      throw NotImplementedHereException.instance;
-   }
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   default Optional<Map<Serializable, QRecord>> oldRecordListToMap(String primaryKeyField, Optional<List<QRecord>> oldRecordList)
-   {
-      if(oldRecordList.isPresent())
-      {
-         return (Optional.of(CollectionUtils.listToMap(oldRecordList.get(), r -> r.getValue(primaryKeyField))));
-      }
-      else
-      {
-         return (Optional.empty());
-      }
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   class NotImplementedHereException extends QException
-   {
-      private static NotImplementedHereException instance = new NotImplementedHereException();
-
-
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      private NotImplementedHereException()
-      {
-         super("Not implemented here");
-      }
-   }
-}

com/kingsrook/qqq/backend/core/model/metadata/SupplementalCustomizerType.java

@@ -1,35 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.model.metadata;
-
-
-/*******************************************************************************
- ** Interface to be implemented by enums of the possible supplemental types
- **
- *******************************************************************************/
-public interface SupplementalCustomizerType
-{
-   /*******************************************************************************
-    ** allows implementing enums to return the expected class being used
-    *******************************************************************************/
-   public Class<?> getCodeReferenceType();
-}

docs/MIGRATION-1.0.md

@@ -0,0 +1,96 @@
+# Migrating from 0.x to 1.0
+
+1.0 is a trust and platform release. Hosts that used the 0.2 fluent API keep compiling; the steps below are what changes at runtime.
+
+## Prerequisites
+
+- QQQ 4.1 (the QBit uses record-change listeners, after-commit callbacks and runtime services, none of which exist in 4.0). Until `qbit-build-parent` 2.1.0 is released, the QBit builds against 4.1.0-RC.1.
+- OpenSearch 2.19.x or 3.x, or an Amazon OpenSearch Service managed domain. Serverless is not supported.
+
+## Required changes
+
+1. **Remove any manual `QBitMetaData` registration.** The producer now registers `com.kingsrook.qbits:quick-search` itself; a second `addQBit` with that name throws at startup. (qqq-all's `DemoQBits` did this in 0.x.)
+2. **Decide the startup mode.** `startupMode` defaults to `FAIL_FAST`: `produce()` fails when OpenSearch is unreachable. Demo profiles that may run without a cluster should set `withStartupMode(QuickSearchStartupMode.DEGRADED)`.
+3. **Run a full reindex once after upgrading.** The index mapping changed (version 2: search analyzer, text-typed `fieldValues`). The QBit logs a warning when it finds an index with the old mapping; "Quick Search Full Reindex" with no table name builds a new physical index and swaps the configured name to an alias. Search keeps working during the rebuild. The identity the QBit uses needs index create, delete and alias privileges for this (see the README role).
+4. **Set `schedulerName` and start the host through `QApplicationLauncher`** (and optionally `basepullRepeatSeconds`, or `reconcileCronExpression` with `reconcileCronTimeZoneId` on a Quartz scheduler) so basepull and reconcile actually run. In 0.x they were never scheduled. See [Scheduling](#scheduling).
+5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `withAllowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
+6. **Grant the permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant the names listed under [Permissions](#permissions) to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
+7. **Update the operational tables before starting 1.0.** `quickSearchIndex` gains five columns, `searchableFieldsJson` changes from `STRING` to `TEXT`, and `tableName` becomes required with a unique key. `quickSearchIndexRun.errorMessage` changes from `STRING` to `TEXT`. There is a new `quickSearchFailedEvent` table. A host that checks its columns at startup rejects the 0.x tables. See [Schema changes](#schema-changes).
+
+## Permissions
+
+QQQ names a permission `{baseName}.{type}`. The admin rules use the base name `quickSearchAdmin`; the table rules have no base name, so each table's name is its base name. With the default rules, grant these to operators:
+
+| Permission | Covers |
+|---|---|
+| `quickSearchAdmin.hasAccess` | The `quickSearchAdmin` app and the three processes (`quickSearchBasepullIndex`, `quickSearchReconcileIndex`, `quickSearchFullReindex`) |
+| `quickSearchIndex.read`, `quickSearchIndex.write` | The `quickSearchIndex` table |
+| `quickSearchIndexRun.read`, `quickSearchIndexRun.write` | The `quickSearchIndexRun` table |
+| `quickSearchFailedEvent.read`, `quickSearchFailedEvent.write` | The `quickSearchFailedEvent` table |
+
+The tables use `READ_WRITE_PERMISSIONS`: `.read` covers query, get and count, and `.write` covers insert, edit and delete. If you set `tablePermissionRules` to `READ_INSERT_EDIT_DELETE_PERMISSIONS`, grant `.read`, `.insert`, `.edit` and `.delete` on each table instead. With `tableNamePrefix` set, the table permissions carry the prefix (`{prefix}quickSearchIndex.read`); the admin permission stays `quickSearchAdmin.hasAccess`.
+
+Users who only search need none of these. Search checks the read permission of each searched table as that table's own rules name it (for example `customer.read`). The QBit's own reads and writes of its tables go through QQQ's table actions, which do not check permissions, so real-time indexing does not depend on these grants.
+
+For a worked example, qqq-all's RC.2 adoption (QRun-IO/qqq-all#30) adds `infra/postgres/06-quick-search-permissions.sql`, which grants the seven names above to its admin role.
+
+## Schema changes
+
+The QBit declares its tables in metadata and ships no DDL. A backend that creates tables from metadata gets the 1.0 shape on a new database. A host that manages its own schema, or checks columns at startup, must change the 0.x tables before starting 1.0. Names below are QQQ field names and types; your backend's naming decides the physical table and column names.
+
+`quickSearchIndex`:
+
+| Field | Type | Change from 0.x |
+|---|---|---|
+| `lastReconcileTime` | `DATE_TIME` | New |
+| `lastRunStatus` | `STRING` | New. Run type and outcome, for example `FULL_REINDEX COMPLETED` (at most 22 characters) |
+| `lastErrorMessage` | `TEXT` | New. Up to 4,000 characters |
+| `documentCount` | `INTEGER` | New |
+| `realTimeErrorCount` | `INTEGER` | New |
+| `searchableFieldsJson` | `TEXT` | Was `STRING` |
+| `tableName` | `STRING` | Now required, with a unique key |
+
+`status` holds `ACTIVE`, `NEEDS_REINDEX` (new in 1.0) or `REBUILDING` (new in RC.2), so it needs at least 13 characters. QQQ checks the `tableName` unique key on insert; a unique index in the database also stops two concurrent first runs from both inserting. 0.x could leave duplicate rows, so keep one row per table before adding the index.
+
+`quickSearchIndexRun` has no new columns. `errorMessage` changed from `STRING` to `TEXT` and holds up to 4,000 characters.
+
+`quickSearchFailedEvent` is new:
+
+| Field | Type | Notes |
+|---|---|---|
+| `id` | `INTEGER` | Primary key, generated by the backend like the other two tables |
+| `tableName` | `STRING` | Source table |
+| `recordId` | `STRING` | Source record's primary key as text |
+| `action` | `STRING` | `INDEX` or `DELETE` |
+| `errorMessage` | `TEXT` | Up to 4,000 characters |
+| `attempts` | `INTEGER` | Replay attempts |
+| `status` | `STRING` | `PENDING`, `EXHAUSTED` or `AWAITING_REINDEX` (new in RC.2); size it for at least 16 characters |
+| `createDate`, `modifyDate` | `DATE_TIME` | |
+
+The QBit sets no maximum length on its `STRING` fields. Map `STRING` to a character column sized for the values above, `TEXT` to a type that holds at least 4,000 characters, and `DATE_TIME` to a timestamp, the same way your other QQQ tables do.
+
+If your backend creates tables from metadata and you do not need the 0.x run history, you can instead drop the 0.x `quickSearchIndex` and `quickSearchIndexRun` tables and let startup recreate them, as qqq-all#30 documents. The next basepull, reconcile or full reindex recreates each `quickSearchIndex` row, with `enabled` and `basepullIntervalMinutes` back at their configured defaults.
+
+## Behaviour changes to be aware of
+
+- Real-time indexing happens after commit. Code that searched for a record inside the same transaction that inserted it will not find it until commit.
+- Real-time failures no longer vanish: they land in `quickSearchFailedEvent` and are replayed by the next basepull. Review that table after an OpenSearch outage.
+- Inserted and updated records are re-read inside the write's transaction, with display values and possible-value labels, before they are indexed. That is one extra query per write batch on an indexed table, and it runs the host's possible-value providers. If the re-read fails, the write still succeeds and the records are replayed by basepull.
+- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile. The QBit logs a warning for each such table at startup, for annotated and config-driven tables alike; a config-driven table with `basepullTimestampField` set to null has opted out and gets no warning.
+- `limit` is capped (default 100), `offset + limit` at 10,000, and search terms must be 2 to 100 characters.
+- Query semantics changed from OR over n-grams to AND over whole terms with prefix matching: "blue widget" now requires both words.
+- `enableScheduledProcesses` still works and sets both `enableBasepullProcess` and `enableMaintenanceProcesses`.
+- `QuickSearchQBitContext` is deprecated and removed in 1.1; use `QuickSearchRuntime.get()`.
+
+## Scheduling
+
+Setting `schedulerName` puts schedules on the basepull and reconcile processes; QQQ runs them only once something starts its `QScheduleManager`.
+
+- **Start through `QApplicationLauncher`.** In QQQ 4.1 that is `com.kingsrook.qqq.middleware.javalin.QApplicationLauncher` (`qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` when any process is scheduled, then the instance's runtime services, including this QBit's. A host that starts only a `QApplicationJavalinServer`, or builds its `QInstance` without the launcher, gets the processes but no scheduled runs, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
+- **A reconcile cron needs a Quartz scheduler.** Basepull's fixed interval runs on any QQQ scheduler, but only `QuartzSchedulerMetaData` supports cron; QQQ's `SimpleSchedulerMetaData` runs fixed intervals only, and a cron schedule on it fails when the schedule manager starts.
+- **A reconcile cron needs a time zone.** Set `reconcileCronTimeZoneId` to a Java time zone ID (`UTC`, `America/Chicago`). QQQ has no default zone: its instance validation rejects a cron schedule without one at startup. `produce()` now catches this first, along with an invalid Quartz expression (seconds first, for example `0 0 3 * * ?`), an unknown zone ID, and a cron on a scheduler in the instance that does not support cron.
+
+## Removed
+
+- `QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`. Hosts that referenced them (for example to compose with their own customizers) should remove those references; the record-change listener coexists with host customizers.
+- `QuickSearchOpenSearchClient.search(String term, String tableName, ...)`; use the collection overload.

docs/MIGRATION-1.1.md

@@ -0,0 +1,240 @@
+# Migrating from 1.0 to 1.1
+
+1.1 lets QQQ core's record search (`POST /qqq/v1/search`, the Next UI global search box) use the Quick Search index. Hosts that do nothing get it by default.
+
+## Prerequisites
+
+- The QQQ release that carries the record-search provider SPI (`RecordSearchProviderInterface`, `QInstance.withRecordSearchProvider`), first released in QQQ `4.1.0-RC.2`. 1.1 builds against `4.1.0-RC.2` through the `qqq-snapshot` profile.
+
+## What changes at startup
+
+1. **The QBit registers a record-search provider.** `QuickSearchRecordSearchProvider` becomes the instance's `recordSearchProvider`. If the host already registered a different provider, the QBit keeps it and logs a warning; core search then does not use OpenSearch.
+2. **Indexed tables get `searchFields`.** Core only searches tables with `searchFields`. Each indexed table that has none gets its visible string and integer searchable fields. Tables that already have `searchFields` keep them. A table added to the instance after the QBit is produced gets none, so add tables first (as for field validation) or set `searchFields` yourself.
+3. **Password fields are no longer indexed.** Fields of type `PASSWORD` are now dropped from the searchable fields like hidden fields. If a 1.0 index holds such values, run "Quick Search Reconcile Index" (or a full reindex) to rewrite the documents without them.
+
+## Behaviour to be aware of
+
+- Next UI search results for indexed tables now come from OpenSearch: all words must match (in any searchable field), with prefix matching and field weights, instead of a contains-match on one field. Tables not indexed by Quick Search keep core's search.
+- Results still pass core's security: core re-reads every hit through `QueryAction`, so table permissions and record security locks apply, and records deleted since they were indexed do not appear.
+- Records are only as fresh as the index: a record updated in a way that is not yet indexed (for example while real-time indexing is off) is found by its indexed text.
+- When OpenSearch is slow or down, core waits at most `recordSearchTimeoutMillis` (default 2500) and then searches with its own contains-query; for the next 30 seconds it skips OpenSearch entirely.
+
+## Opting out
+
+```java
+new QuickSearchQBitConfig()
+   // ...
+   .withServeCoreRecordSearch(false);
+```
+
+restores 1.0 behaviour: no provider, no `searchFields` changes.
+
+## Diagnostic tools (1.1.0-RC.2)
+
+The QBit now adds admin tools to validate and debug search (see the README, "Diagnostic tools"). They are on by default.
+
+- **Two new operational tables**, on `backendName` like the others: `quick_search_diagnostic_event` and `quick_search_validation_run` (columns in the README, "Operational tables"). Create them in the host's schema, with the naming the host applies to the other operational tables (`tableMetaDataCustomizer` applies to them too). Until they exist, writing events and saving reports fails quietly (logged at warn) and the tools still run.
+- **Permissions.** Grant `quick_search_debug.hasAccess` to the people who may use the read-only tools and see the widget, `quick_search_admin.hasAccess` for Reindex in Search, and `.read` on the two new tables (the QBit writes them; users never do).
+- **New processes** have a `tableName` (Quick Search Index, Quick Search Failed Event, and each indexed table), so they appear as table and record actions. They are not added to the admin app's children; a host that strips processes from its nav needs no change.
+- **Opting out.** `withEnableDiagnosticTools(false)` adds none of it; `withEnableRecordDiagnosticActions(false)` keeps the tools on Quick Search Index but adds nothing to the indexed tables.
+
+## Snake_case names (1.1.0-RC.3)
+
+Owner naming rule (2026-10-12): table names are lowercase snake_case, and a QBit's tables are prefixed with its full name. RC.3 renames the QQQ metadata names. The QBit ships no DDL, so nothing is dropped or rewritten; what you change depends on how your database tables are named today.
+
+| Kind | Before (RC.1, RC.2) | From RC.3 |
+|---|---|---|
+| Table | `quickSearchIndex` | `quick_search_index` |
+| Table | `quickSearchIndexRun` | `quick_search_index_run` |
+| Table | `quickSearchFailedEvent` | `quick_search_failed_event` |
+| Table | `quickSearchDiagnosticEvent` | `quick_search_diagnostic_event` |
+| Table | `quickSearchValidationRun` | `quick_search_validation_run` |
+| Possible-value source | `quickSearchIndex`, `quickSearchSearchableTable` | `quick_search_index`, `quick_search_searchable_table` |
+| Table permissions | `quickSearchIndex.read`, `quickSearchIndexRun.read`, ... | `quick_search_index.read`, `quick_search_index_run.read`, ... |
+| Admin permission | `quickSearchAdmin.hasAccess` | `quick_search_admin.hasAccess` |
+| Debug permission | `quickSearchDebug.hasAccess` | `quick_search_debug.hasAccess` |
+| Record actions on failed events | `quickSearchInspect_quickSearchFailedEvent`, `quickSearchReindexRecords_quickSearchFailedEvent` | `quickSearchInspect_quick_search_failed_event`, `quickSearchReindexRecords_quick_search_failed_event` |
+
+Unchanged: Java class names, field names (`quickSearchIndexId`, `tableName`), process, widget and app names (`quickSearchFullReindex`, `quickSearchDiagnostics`, `quickSearchAdmin` app), the OpenSearch index, and the developer panel id.
+
+**With a table-name prefix** (`withTableNamePrefix` or the producer namespace), table and possible-value source names are now the prefix in snake_case joined with `_`: prefix `myApp_` or `myApp` gives `my_app_quick_search_index`, where RC.2 gave `myApp_quickSearchIndex`. Process, widget and app names keep the prefix as given.
+
+**The old names stay available** as `@Deprecated` constants for upgrade scripts: `QuickSearchIndex.LEGACY_TABLE_NAME` (and the same on the other four entities), `QuickSearchSearchableTablePossibleValueSourceProducer.LEGACY_NAME`, and `QuickSearchQBitConfig.LEGACY_ADMIN_PERMISSION_BASE_NAME` / `LEGACY_DEBUG_PERMISSION_BASE_NAME`. They will be removed in 2.0.
+
+### Database tables
+
+- **Your tables are already `quick_search_*`** (for example you set the RDBMS backend name with `inferBackendName` or a table customizer): no DDL. You can drop the customizer, since the QQQ name now equals the database name.
+- **Your tables are camelCase** (`"quickSearchIndex"`): rename them, in a maintenance window, before starting RC.3. PostgreSQL example:
+
+```sql
+ALTER TABLE "quickSearchIndex"           RENAME TO quick_search_index;
+ALTER TABLE "quickSearchIndexRun"        RENAME TO quick_search_index_run;
+ALTER TABLE "quickSearchFailedEvent"     RENAME TO quick_search_failed_event;
+ALTER TABLE "quickSearchDiagnosticEvent" RENAME TO quick_search_diagnostic_event;
+ALTER TABLE "quickSearchValidationRun"   RENAME TO quick_search_validation_run;
+```
+
+  Rename the matching sequences, indexes and constraints the same way if your tooling depends on their names. Data is untouched. If you cannot rename yet, point each table at its old name with an `RDBMSTableBackendDetails` table name in a table customizer.
+
+### Stored references
+
+QQQ stores table, process and permission names in your own data. Update them in the same window (PostgreSQL examples; adjust schema and column names to your instance):
+
+```sql
+-- audit history: audit_table.name holds the QQQ table name
+UPDATE audit_table SET name = 'quick_search_index'            WHERE name = 'quickSearchIndex';
+UPDATE audit_table SET name = 'quick_search_index_run'        WHERE name = 'quickSearchIndexRun';
+UPDATE audit_table SET name = 'quick_search_failed_event'     WHERE name = 'quickSearchFailedEvent';
+UPDATE audit_table SET name = 'quick_search_diagnostic_event' WHERE name = 'quickSearchDiagnosticEvent';
+UPDATE audit_table SET name = 'quick_search_validation_run'   WHERE name = 'quickSearchValidationRun';
+
+-- saved views and saved reports
+UPDATE saved_view   SET table_name = 'quick_search_index' WHERE table_name = 'quickSearchIndex';
+UPDATE saved_report SET table_name = 'quick_search_index' WHERE table_name = 'quickSearchIndex';
+-- ... and the same for the other four tables
+```
+
+- **Permission grants.** Wherever your roles live (QQQ role tables, an identity provider's group attributes, code), replace `quickSearchAdmin.hasAccess` with `quick_search_admin.hasAccess`, `quickSearchDebug.hasAccess` with `quick_search_debug.hasAccess`, and `quickSearchIndex.read` / `.edit` (and the other tables' permissions) with the `quick_search_*` names. Users without the new grants lose the admin app and tools until they are granted.
+- **Bookmarks and links** to `/app/quickSearchIndex/...` move to `/app/quick_search_index/...`.
+- **Index rows and failed events** store *your* table names (`quick_search_index.table_name`, `quick_search_failed_event.table_name`), not the QBit's, so they need no change for this rename. The OpenSearch documents are keyed by your tables too, so **no reindex is needed**. If you rename your own tables at the same time, update those two columns and run a full reindex.
+
+### Checklist
+
+1. Back up the database.
+2. Rename the tables (only if they are camelCase) and update the stored references above.
+3. Update permission grants.
+4. Upgrade to RC.3 and start; `startupMode` `FAIL_FAST` reports any table the backend cannot find.
+5. Open Admin › Quick Search and run Validate Search Index.
+
+## Data-driven configuration (1.1.0-RC.3)
+
+From RC.3, which tables and fields Quick Search indexes is **data**: standard QQQ tables an admin edits, secured by permissions and audited. Code (`@QuickSearchable` / `@QuickSearchField` entities and `searchableTables`) still works, but it only **seeds** the records. Changes take effect without a redeploy.
+
+### New tables and columns
+
+| Table | One row per | Notes |
+|---|---|---|
+| `quick_search_index` (existing) | searchable table | New columns `source` (`CODE` or `OWNER`) and `owner_edited`. `basepull_timestamp_field` is now editable. The generic Create and Delete screens are off: use Add Search Index and Remove Search Index. |
+| `quick_search_index_field` | searchable field of an index | `field_name`, `weight` (1-1000), `role` (`DISPLAY` T1 display, `TITLE`, `SUBTITLE`, `BODY`, `KEYWORD`, `EXACT`), `analyzer` (blank, `PREFIX`, `WORD`, `ENGLISH`, `EXACT`), `include_in_snippet` (blank: by role), `include_label`, `sort_order`, `source`, `owner_edited`. Unique on (`quick_search_index_id`, `field_name`). Edited from the index record's Searchable Fields list. |
+| `quick_search_settings` | one row (id 1) | The global pause (`indexing_paused`, `paused_at`, `paused_by`, `pause_reason`) and the last self-priming report. Created by the QBit. |
+| `quick_search_config_tombstone` | table or field an admin removed | Priming never adds back what a tombstone names. Delete the tombstone to let code seed it again. |
+| `quick_search_index_run` (existing) | run | New columns `cancel_requested` and `cancel_requested_by` (Cancel Index Run). |
+
+The config tables are audited at field level (QQQ's `audit` tables; without them the audit is skipped with a log line) and use `tablePermissionRules` (by default `quick_search_index_field.read`, `.edit`, and so on). The new processes use `adminPermissionRules` (`quick_search_admin.hasAccess`).
+
+### Precedence: code versus data
+
+1. **The records are the source of truth** once the config tables exist and priming has run. Indexing (real-time, basepull, reconcile, full reindex), search, the core record-search provider and the diagnostic tools read the effective config from them. It is cached for 30 seconds and dropped at once on the server where a config row changes, so an edit takes effect immediately there and within 30 seconds on other servers.
+2. **Code seeds; it never overrides an owner.** At startup (and on each basepull until it succeeds), priming:
+   - adds a row (source `CODE`, `enabled` per `enabledByDefault`) and its field rows for every code-declared table that has none, and queues its first build;
+   - updates the interval and timestamp field of rows nobody edited (`owner_edited` false); `enabled` is set only when a row is created;
+   - adds code-declared fields that have no row, and updates the weight and label flag of field rows nobody edited; either change queues a rebuild of that table;
+   - never touches a row or field an admin added or edited (`owner_edited` true), and never re-adds anything a tombstone names;
+   - never deletes anything: removing an annotation leaves the rows for the admin to remove.
+3. **Removals are remembered.** Remove Search Index, and deleting a field row, write a tombstone, so code does not bring them back.
+4. **Without the config tables** (before you add the DDL below), or until priming succeeds, Quick Search uses the code-declared tables exactly as before RC.3. So the upgrade can be staged.
+
+### Self-priming
+
+- A new code-declared table gets its rows on startup and is built by the next basepull (status `PENDING_BUILD`, then `BUILDING`, then `ACTIVE`; the run is recorded as `FULL_REINDEX`). No manual step is needed.
+- A new install, or an index that was deleted, is recreated by the basepull, and the tables that had documents in it are rebuilt. An index that is empty while a table last counted documents in it is rebuilt too.
+- The Search Diagnostics widget on the Quick Search home reports it ("New: 2 tables added and building (...)"), and `QuickSearchStatus.read(runtime).bannerHtml()` gives the same lines for a host's own home widget.
+- Existing rows from RC.2 or earlier get their field rows on the first start, with **no rebuild** when the index was built with exactly the code's fields.
+
+### Admin actions
+
+All are standard streamed-ETL processes with a review step and a result step, run from the table's Actions (never from the nav):
+
+| Action | On | What it does |
+|---|---|---|
+| Add Search Index | Quick Search Index (no selection) | Pick a table you can read that is not indexed and is not internal or secret; set the interval, timestamp field and enabled flag. The review shows the estimated record count, the fields it starts with (record-label fields as titles, then text fields) and what is never indexed (hidden and `isSensitive` fields). Saving queues the first build (or runs it at once when no scheduler runs the basepull). |
+| Searchable Fields (child list on the index record) | Quick Search Index Field | Add, edit or remove fields with the standard screens; a field must exist, be indexable and not sensitive, and not be listed twice. The change applies to search and new writes at once and marks the index `NEEDS_REINDEX`. |
+| Apply Search Index Changes | Quick Search Index (selected) | The review previews the field changes since the last build and the records the rebuild reads; executing rebuilds the table. |
+| Disable Search Index, Remove Search Index | Quick Search Index (selected) | Stop indexing at once. The input step's "Search documents" choice (delete now, or keep) is restated on the review step. Remove also deletes the field rows and pending failed events and leaves a tombstone. Enabling a disabled index again (edit Enabled) queues a build. |
+| Cancel Index Run | Quick Search Index Run (selected `RUNNING` runs) | Sets a cooperative cancel flag that the run checks between batches; it ends `CANCELLED` with its counts. A cancelled all-tables rebuild keeps the previous index; a cancelled queued build waits for Apply Search Index Changes. |
+| Pause Indexing, Resume Indexing | Quick Search Settings | Pauses all indexing (real-time and scheduled; a running rebuild stops after its batch) while search keeps serving the current index. Changes made while paused are kept (`DEFERRED` failed events) and indexed by the basepull after Resume. The home shows "Paused since ... by ...". |
+
+Roles and analyzers: `TITLE` fields make the hit's label, `SUBTITLE` its subtitle (`QuickSearchResult.subtitle`), `DISPLAY` fields its short details (`QuickSearchResult.displayValues`); `KEYWORD` and `EXACT` fields stay out of the highlighted snippet and match on their own. The `WORD`, `ENGLISH` and `EXACT` analyzers use new document objects (`fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`); their mapping is added to an existing index at startup (no mapping-version change), and documents get them as they are rebuilt. Tables of plain fields produce the same documents and queries as before.
+
+### DDL
+
+The QBit ships no DDL. PostgreSQL, with the snake_case column names QQQ infers (`QInstanceEnricher.setInferredFieldBackendNames`); use the field names as written if your backend maps fields to camelCase columns. Liquibase formatted SQL:
+
+```sql
+-- liquibase formatted sql
+
+-- changeset quick-search:1.1.0-RC.3-1
+ALTER TABLE quick_search_index ADD COLUMN IF NOT EXISTS source VARCHAR(20);
+ALTER TABLE quick_search_index ADD COLUMN IF NOT EXISTS owner_edited BOOLEAN;
+ALTER TABLE quick_search_index_run ADD COLUMN IF NOT EXISTS cancel_requested BOOLEAN;
+ALTER TABLE quick_search_index_run ADD COLUMN IF NOT EXISTS cancel_requested_by VARCHAR(250);
+
+-- changeset quick-search:1.1.0-RC.3-2
+CREATE TABLE IF NOT EXISTS quick_search_index_field (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   quick_search_index_id INTEGER NOT NULL,
+   field_name VARCHAR(255) NOT NULL,
+   weight INTEGER,
+   role VARCHAR(20),
+   analyzer VARCHAR(20),
+   include_in_snippet BOOLEAN,
+   include_label BOOLEAN,
+   sort_order INTEGER,
+   source VARCHAR(20),
+   owner_edited BOOLEAN,
+   create_date TIMESTAMP WITHOUT TIME ZONE,
+   modify_date TIMESTAMP WITHOUT TIME ZONE,
+   CONSTRAINT pk_quick_search_index_field PRIMARY KEY (id),
+   CONSTRAINT uq_quick_search_index_field UNIQUE (quick_search_index_id, field_name)
+);
+
+-- changeset quick-search:1.1.0-RC.3-3
+CREATE TABLE IF NOT EXISTS quick_search_settings (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   indexing_paused BOOLEAN,
+   paused_at TIMESTAMP WITHOUT TIME ZONE,
+   paused_by VARCHAR(250),
+   pause_reason VARCHAR(500),
+   last_primed_at TIMESTAMP WITHOUT TIME ZONE,
+   last_priming_summary TEXT,
+   last_priming_new_tables INTEGER,
+   create_date TIMESTAMP WITHOUT TIME ZONE,
+   modify_date TIMESTAMP WITHOUT TIME ZONE,
+   CONSTRAINT pk_quick_search_settings PRIMARY KEY (id)
+);
+
+-- changeset quick-search:1.1.0-RC.3-4
+CREATE TABLE IF NOT EXISTS quick_search_config_tombstone (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   table_name VARCHAR(255) NOT NULL,
+   field_name VARCHAR(255),
+   removed_by VARCHAR(250),
+   reason VARCHAR(500),
+   create_date TIMESTAMP WITHOUT TIME ZONE,
+   CONSTRAINT pk_quick_search_config_tombstone PRIMARY KEY (id)
+);
+CREATE INDEX IF NOT EXISTS idx_quick_search_config_tombstone_table ON quick_search_config_tombstone (table_name);
+```
+
+MySQL: use `INT AUTO_INCREMENT` for the ids, `DATETIME` for the timestamps and `TINYINT(1)` for the booleans, and drop `IF NOT EXISTS` from `ADD COLUMN`.
+
+### Behaviour changes to be aware of
+
+- `quick_search_index` no longer offers Create or Delete (`TABLE_INSERT` and `TABLE_DELETE` are off), so the API cannot insert or delete rows either. Rows come from priming and Add Search Index, and go through Remove Search Index.
+- Basepull builds queued tables (`PENDING_BUILD`) itself and recreates a missing index. `NEEDS_REINDEX` still waits for an admin (Apply Search Index Changes, or Rebuild).
+- With the config tables, a table that is not in the instance is not part of the effective config, and `ensureIndexRowExists` no longer creates rows (priming does, honouring tombstones).
+- Hidden and `isSensitive` fields are never indexed, whether declared in code or added as rows.
+- `quick_search_index_run` and `quick_search_failed_event` offer no Create or Edit (Delete stays). The Basepull, Full Reindex and Reconcile processes are table actions on Quick Search Index rather than app children; their table input is now `searchTableName` (API callers may keep sending `tableName`).
+- `quick_search_searchable_table` is now a custom possible-value source over the effective tables (it was an enum of the code-declared ones).
+- Tables added at runtime get core `searchFields` (for the global search) when they have none, but the per-table Inspect Search Document and Reindex in Search record actions appear only after the next restart.
+
+### Table help (no action needed)
+
+The qbit now sets bundle help (`help/quick-search.yaml`) on the Quick Search Admin app and each `quick_search_*` table, so an empty table says what it is for and how its rows get created. It fills only empty `help` slots; help a host already set on these tables is kept. There is no DDL change and no host include.
+
+### Checklist for data-driven configuration
+
+1. Add the DDL above (it is additive).
+2. Grant `quick_search_index_field`, `quick_search_settings` and `quick_search_config_tombstone` permissions (read, and edit where admins should change them) next to the existing Quick Search grants.
+3. Upgrade and start. The Search Diagnostics widget should not say the searchable tables come from code, and shows the first priming's summary.
+4. Optionally move code-declared search config into the records and delete the annotations: the rows stay and are now the owner's.

docs/PLAN-1.0-release.md

@@ -0,0 +1,50 @@
+# PLAN: Quick Search 1.0 release readiness
+
+> Executed natively in-session, task by task, test-first, one commit per task on `feature/GH-1-0-release-readiness`. Spec: `docs/initiatives/quick-search-1.0.md` (PRD) and `docs/audits/2026-10-04/README.md` (blockers B1 to B25). Decisions D1 to D11 are resolved with the PRD's recommendations unless noted under "Decisions applied".
+
+## Goal
+
+Close every audit blocker and ship a QBit that is a standard QQQ 4.1 citizen, keeps a trustworthy index, searches safely, connects to real OpenSearch deployments, and documents all of it.
+
+## Decisions applied
+
+| ID | Resolution used here |
+|---|---|
+| D1 | API-only in 1.0; `QuickSearchResult` gains `tableLabel`, input gains `tableNames` and `limitPerTable`; the core SPI is proposed separately (TODO) |
+| D2 | 1.0.0-SNAPSHOT on this branch; 0.3.0 compat release remains the qqq#921 chain's job |
+| D3 | Table-level READ filter plus `QueryAction` post-filter by primary key (`applyRecordSecurityLocks`, default true) |
+| D4 | `schedulerName` null leaves processes unscheduled with a startup WARN (not an error), so qqq-all keeps booting; set it to schedule |
+| D5 | Full reindex of all tables builds a fresh physical index and swaps the alias; a single-table full reindex uses the reconcile algorithm (index in place, then delete stale) |
+| D6 | `authMode` `AWS_SIGV4` included with optional AWS SDK v2 dependencies |
+| D7 | `RecordChangeListenerInterface` plus `addAfterCommitCallback` |
+| D8 | Listener never fails the write; failures go to `quickSearchFailedEvent`, replayed by the scheduled basepull |
+| D9 | Supported platforms statement as in the audit |
+| D10 | 4.1-only: the build uses qqq 4.1.0-RC.1 through the (now default-active) `qqq-snapshot` profile until parent 2.1.0 ships; customizers are deleted |
+| D11 | `startupMode` default `FAIL_FAST`; `DEGRADED` available |
+
+## Architecture
+
+- `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`; component producers in the package build the two tables, three processes, failed-event table, table PVS and app; `postProduceActions` registers the record-change listener and the runtime service and resolves discovered tables.
+- `QuickSearchRuntime` (instance-scoped, held `transient` on the config, resolved through `QContext.getQInstance().getQBits()`) owns discovered tables, the lazily built client, the publisher, the enabled-table cache and lifecycle. `QuickSearchQBitContext` stays as a deprecated static facade delegating to it (tests keep working).
+- `QuickSearchRecordChangeListener` replaces the three customizers; publishes after commit when a transaction is present; records failures in `quickSearchFailedEvent`.
+- `QuickSearchOpenSearchClient` gains a transport factory (`NONE`, `BASIC`, `AWS_SIGV4`, TLS block, timeouts), mapping v2 (search analyzer, dynamic templates, `_meta` version), alias management, health, and wraps `OpenSearchException`.
+- Indexing steps page by primary key, anchor the basepull watermark at run start, honour `enabled`, sweep orphans, detect drift, purge run history, replay failed events, and use `external_gte` versioning from the timestamp field.
+- `QuickSearchAction` filters tables by permission and enabled state, bounds inputs, supports `tableNames` and `limitPerTable`, post-filters through `QueryAction`, returns `tableLabel`.
+
+## Tasks (each ends with green `mvn test`, then a commit)
+
+1. Build: default-active `qqq-snapshot` profile on `4.1.0-RC.1`, `revision` 1.0.0-SNAPSHOT, opensearch-java 3.10.0 with the `Refresh.True` fix, JaCoCo `prepare-agent` with explicit version, drop JUnit 5.10.0 pins (fallback: keep if JUnit 6 breaks a dependency). Files: `pom.xml`, `QuickSearchOpenSearchClient.java:427`.
+2. Config surface and runtime holder: new fields (`startupMode`, `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`, `enableBasepullProcess`, `enableMaintenanceProcesses`, `adminPermissionRules`, `tablePermissionRules`, `applyRecordSecurityLocks`, `maxSearchLimit`, `basepullOverlapSeconds`, `runHistoryRetentionDays`, `maxFieldLength`, `opensearchUrl`, `authMode`, `awsRegion`, `awsServiceName`, `awsAssumeRoleArn`, `allowPlaintextCredentials`, `tls*`, `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `transportCustomizer`), deprecated `enableScheduledProcesses` alias, null-safe validation with field-existence checks, `getDefaultBackendNameForTables`, settable `tableMetaDataCustomizer`, `@JsonIgnore` secret getters, redacting `toString`, `transient` publisher. `QuickSearchRuntime` class; `QuickSearchQBitContext` facade. Tests: `QuickSearchQBitConfigTest`, new `QuickSearchRuntimeTest`.
+3. Producer contract: `QBitMetaDataProducer`, `QBitMetaData` (artifactId `quick-search`, version from `quick-search.properties` filtered by Maven), component producers, unique key on `tableName`, permission rules, schedules, `QuickSearchRuntimeService`, `isEnabled` false without config, compat shims (`withConfig`, `getConfig`, `produce(QInstance)` adds and returns an empty output). Tests: `QuickSearchQBitProducerTest` updated plus new assertions (qbit registered, sourceQBitName, schedule, permission rules, unique key).
+4. Real-time path: `QuickSearchRecordChangeListener`, `quickSearchFailedEvent` table and entity, failed-event replay in basepull, delete customizers and their tests, new `QuickSearchRecordChangeListenerTest` (insert/update/delete, after-commit via a fake transaction, rollback discards, failure writes a failed-event row).
+5. Client: transport factory with auth modes, TLS, timeouts, user agent; secret interpretation via `QMetaDataVariableInterpreter`; mapping v2 with `_meta.mappingVersion`; `lenient` boosts and `operator=AND`; `OpenSearchException` wrapping; alias helpers (`createIndexForAlias`, `swapAlias`, `deleteIndex`, `resolvePhysicalIndex`); `health()`; `external_gte` versioning on bulk index; byte-capped bulk batches. Tests: `QuickSearchOpenSearchClientTest` (config to transport mapping, validation of modes), integration tests updated for mapping v2 and the "stella/stone" and mixed-type cases.
+6. Indexing steps: keyset paging in `AbstractIndexingStep`; basepull start-anchored watermark with overlap and `GREATER_THAN_OR_EQUALS`, `enabled` honoured, failures aggregated, run purge, failed-event replay; full reindex alias swap (all tables) or in-place (single table), error message on FAILED; reconcile orphan sweep and `enabled`; drift detection sets `status = NEEDS_REINDEX`; `IndexingUtils` display values, label fallback, hidden-field exclusion, truncation, safe label formatting. Tests: step tests updated plus new scenarios from audit 05.
+7. Search: `QuickSearchInput extends AbstractActionInput`, `tableNames`, `limitPerTable`, bounds, permission and enabled filtering, `QueryAction` post-filter, `tableLabel`, `totalHitsIsLowerBound`, `track_total_hits`. Tests: `QuickSearchActionTest` with a permission-denied session and bounds.
+8. Admin surface: process frontend steps with an optional `tableName` field backed by a discovered-tables PVS, admin table fields (`lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`, `realTimeErrorCount`), read-only tracking fields, run-to-index PVS, labels, icon, health process.
+9. Docs and CI: README rewrite, `CHANGELOG.md`, `docs/MIGRATION-1.0.md`, configuration reference, supported platforms and roles, parameterized Testcontainers image with a 2.19.6 and 3.9.0 matrix in CI, CLAUDE.md and AGENTS.md architecture updates.
+10. Close-out: full `mvn verify` with Docker, PRD and audit status columns, TODO and SESSION-STATE, second-brain save.
+
+## Open Questions
+
+- Parent 2.1.0 re-pin happens when qbit-bom publishes it; until then the default-active profile carries 4.1.0-RC.1 (recorded as a deviation from ADR-0007, to be reverted).
+- The qqq core `RecordSearchProviderInterface` proposal is filed as a follow-up, not implemented here.

docs/STATUS-API-HOST-MIGRATION.md

@@ -0,0 +1,79 @@
+# Quick Search host migration (kof22 #164)
+
+Issue #50 / architecture finding F10. Paths below are under
+`Website-Backend/src/main/java/com/kof22/website/metadata/adminhome/`;
+line numbers refer to the local clone inspected on 2026-10-13. No host files were changed.
+
+| Existing host code | Replacement |
+| --- | --- |
+| `LiveAdminHomeProbes.java:340` `searchConfigured()` | Use `QuickSearchApi.APP_NAME` for the default-name app lookup; for an authorized snapshot use `QuickSearchStatusService.get()` and its state/homeHref. `DISABLED` also includes all indexes disabled, so it is not exactly “qbit absent.” |
+| `LiveAdminHomeProbes.java:351` `searchCluster()` | One `QuickSearchStatusService.get()`; map `clusterReachable`, `clusterHealth`, `nodes`, `activeShards`, `unassignedShards`. Remove direct cluster HTTP/auth/CA handling. |
+| `LiveAdminHomeProbes.java:386` `searchDocumentCounts(...)` | Reuse the same snapshot's `indexes()` and each row's `tableName()` / `documentCount()` / `sourceCount()` / `countsCheckedAt()`. Remove `QuickSearchOpenSearchClient` access and relabel counts as saved diagnostics, not live. Preserve null as unknown. |
+| `QuickSearchHomeWidgets.java:118` `widgets()` | Stop adding the host's duplicate Quick Search home widgets. Link to `status.homeHref()` (or default `QuickSearchApi.HOME_HREF`). Keep the qbit-provided home/diagnostics widget. |
+| `QuickSearchHomeWidgets.java:133` `indexFacts()`, `:160` `summary()` and `:196` `syncTile()` | For a compact host overview, use the same `QuickSearchStatusService.get()` value: `state`, `indexes`, `pending`, `exhausted`, `lastSuccessfulSync`, `lastError`. Remove SQL aggregates and live source counts; use saved count timestamps and truncation flags. |
+| `QuickSearchHomeWidgets.java:267` `runs()`, `:315` `failedEvents()` | Replace duplicate detailed views with the home link; drill down using the qbit's own Index Run / Failed Event tables from that home. The summary API intentionally does not export detailed histories. |
+| `AdminOperationsWidgets.java:209` `quickSearch()` | Render `status.indexes()` for the compact admin overview, and use `status.homeHref()` for its footer link. Remove qbit-table SQL and source-table counts. |
+| `AdminOperationsWidgets.java:343` `searchFooter()` | Use `clusterHealth`, `pending`, `exhausted`, `lastSuccessfulSync`, `lastError`, and `checkedAt` from the same snapshot. Detailed recent-run information belongs in the qbit home. |
+
+Call with the existing request's `QContext`/`QSession`; do not elevate a host user's
+session to bypass `adminPermissionRules`. Cache hits still check permission.
+
+## UI contract
+
+| Method | Path | operationId |
+| --- | --- | --- |
+| GET | `/qqq/v1/qbit/quick-search/status` | `getQuickSearchStatus` |
+| GET | `/qqq/v1/qbit/quick-search/suggest` | `getQuickSearchSuggestions` |
+
+Suggestions require only a signed-in session, with table READ permissions and
+configured record locks applied to results. Do not gate the typeahead UI on the
+status endpoint's admin permission. Send `q`, optionally `limitPerTable` (default 5,
+maximum 10), `maxTables` (default 8, maximum 20), and comma-separated `tables`.
+The response contains `query`, relevance-ordered `groups` and `truncated`; each
+group has `tableName`, `tableLabel`, `hasMore` and `items` of `recordId`, `label`,
+and nullable `subtitle`. See the [API details](../README.md#typeahead-suggestions)
+for normalization, bounded-batch semantics and empty responses during outages.
+
+Extension: `quick-search`, version `1.1.0`; core dependency: `4.1.0-RC.15` (UI API `1.6.0`).
+The requested `/quick-search` prefix cannot validate on the supplied RC.6 baseline:
+`MiddlewareSpecExtensionsMetaData.ESTABLISHED_PREFIXES` only admits esb, alerts,
+assistant, and sql-workbench. We use the default namespace without modifying core.
+
+The suggestions change adds a second endpoint and its schemas; the status
+method, path, request and response are unchanged. Next UI `src/` contains no calls to a
+Quick Search-specific endpoint today; its core `/qqq/v1/search` integration is unchanged.
+Vendor `src/main/resources/openapi/qqq-v1/quick-search.yaml` for clients.
+
+Regenerate the fragment with:
+
+```powershell
+./mvnw.cmd -B -Dtest=QuickSearchApiContractTest -Dquicksearch.openapi.update=true test
+./mvnw.cmd -B -Dtest=QuickSearchApiContractTest test
+```
+
+The fragment covers both registered specs using the committed dependency version.
+
+## Suggestions verification (2026-10-14)
+
+The two-spec contract regenerates and passes without the update flag. Final
+`CI=true ./mvnw.cmd -B verify` passes 495 unit/guardrail tests, including strict
+HTTP checks for reader access and invalid suggestion parameters. All 9 integration
+classes fail their Docker prerequisite because this sandbox cannot access Docker's
+named pipe; the 3 new suggest integration tests compile but remain unexecuted.
+The configured JaCoCo gates pass when invoked separately with
+`./mvnw.cmd -B jacoco:report jacoco:check@check` (74.38% instruction, 91.88% class).
+New API files and changed tests pass repository Checkstyle; the full source check
+still reports 446 existing violations.
+
+## Original status verification and limits
+
+`./mvnw.cmd -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT` passes
+468 unit tests and JaCoCo, including strict HTTP server startup, permission checks,
+OpenAPI contract consistency, cached/disabled/degraded/down states, timeout recovery,
+bounded backlog/table lists and prefixed QInstance isolation. All 41 Docker integration
+tests skip locally because Docker is unavailable.
+
+The separate Checkstyle check is **not green**: the existing checked-in rules report
+471 main-source violations at HEAD and 438 in this worktree, with zero new violations.
+New source files and the two new tests pass those rules. No repository-wide header
+rewrite or unrelated style cleanup was included.

docs/TODO.md

@@ -0,0 +1,81 @@
+# TODO
+
+## UI API 1.6 migration (2026-10-15)
+
+- [x] Bump `quick-search` extension to 1.1.0, retain deprecated `QuickSearchErrorV1`, and use core `BasicErrorResponseV1` for errors; preserve operation IDs and success payloads.
+- [x] Regenerate the two-operation contract with `-Dquicksearch.openapi.update=true`, then pass the contract test without regeneration; no dangling references or separate baseline were found.
+- [x] Set the release BOM default to QQQ `4.1.0-RC.15`; validate offline against the locally installed `4.1.99-SNAPSHOT` using `./mvnw.cmd -o test -Dqqq.snapshot.version=4.1.99-SNAPSHOT -Denforcer.skipRules=requireReleaseDeps` (540 tests, no failures/errors/skips). The command-line exception is needed only because the requested local core is a SNAPSHOT; other guardrails remain enabled.
+- [x] Verify strengthened HTTP error-body assertions: 8 status/HTTP tests passed after the full suite.
+- [ ] Run Docker integration tests through `verify` in CI; this migration ran the requested `test` lifecycle.
+
+
+## In-app help bundle (2026-10-14)
+
+- [x] Pin QQQ RC.7, register the host-neutral QBIT bundle, replace the temporary loader, document HOST overrides, and add coverage/no-help-table checks.
+- [ ] Host adoption: enable QQQ help startup sync and keep deployment schedules and host-table record actions in HOST bundles.
+
+## Real-time error counter
+
+- [x] Count failed real-time events after commit, serialize concurrent listeners within a JVM, tolerate counter-write failures, and reset after successful basepull/full reindex; add regression tests and field/documentation help.
+
+## Architecture status API (#50 / F10)
+
+- [x] Supported Java status API, bounded cache/rate limiting, administrator authorization, home navigation, spec endpoint and committed fragment.
+- [x] Contract and strict HTTP tests; host migration mapping for kof22#164.
+- [ ] kof22#164: replace internal probes/duplicate home using `docs/STATUS-API-HOST-MIGRATION.md`.
+- [ ] Next UI: vendor the additive Quick Search status fragment when adopting RC.6.
+- [ ] Run Docker integration tests in CI; local Docker is unavailable.
+- [ ] Resolve the pre-existing repository Checkstyle baseline (438 remaining main-source violations, down from 471 at HEAD); status API adds none.
+
+## Before tagging 1.0.0
+
+- [ ] qqq-all and Voyage adopt `1.0.0-RC.2` (items below) and report back.
+
+- [x] PR #10 (`feature/GH-1-0-release-readiness`) opened and reviewed 2026-10-05; `AGENTS.md` kept.
+- [ ] Re-pin to `qbit-build-parent` 2.1.0 when qbit-bom publishes it, and remove the `activeByDefault` on the `qqq-snapshot` profile (ADR-0007).
+- [ ] qqq-all follow-up PR: remove the manual `instance.addQBit(...)` for quick-search in `DemoQBits`, pass `withSchedulerName("demoScheduler")`, set `withStartupMode(QuickSearchStartupMode.DEGRADED)` for the full profile test, pin 1.0.0.
+- [ ] Voyage: pin 1.0.0, set `schedulerName`, switch to `opensearchUrl` and `${env.}` credentials.
+- [ ] CI: add a second test job with `-Dopensearch.test.image=opensearchproject/opensearch:3.9.0` (needs a qqq-orb parameter for extra Maven args, or a plain job).
+- [x] Core SPI proposed as QRun-IO/qqq#1042 (D1); the 1.1 adapter `QuickSearchRecordSearchProvider` is on `feature/record-search-provider`.
+- [x] Tracking issue #12 created and recorded in the PRD (2026-10-06).
+- [x] PR #8 merged 2026-10-05; Dependabot PR #1 closed as superseded.
+- [x] PR #10 merged and `1.0.0-RC.1` published to Maven Central from `release/1.0` (2026-10-05).
+- [ ] Upstream: tell qbit-bom that the published parent POM has `<version>${revision}</version>`, which Maven 3.10 rejects (this repo works around it with the wrapper on 3.9.11).
+
+## Before 1.0.0 GA (from the PR #10 review, 2026-10-05; filed as #13 to #17 under #12)
+
+- [x] #13 Listener: a failed in-transaction re-read or a missing AWS SDK (`LinkageError`) records `quickSearchFailedEvent` rows (PR #22).
+- [x] #14 Full reindex: deletes made during a rebuild are captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap (PR #20).
+- [x] #15 Transport: warn on `tls.hostnameVerification=false`, empty `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected (PR #18).
+- [x] #16 Search: `offset` counts readable results under record locks, per-table `offset`, bounded scan and 10,000 window cap (PR #19).
+- [x] #17 `safeCount` logs at warn and keeps the previous `documentCount` (PR #21).
+- [x] `1.0.0-RC.2` published to Maven Central from `release/1.0` (2026-10-06).
+- [ ] Stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review): only another all-tables reindex clears it, `AWAITING_REINDEX` rows accumulate and drift detection stays off for those tables. No data is lost. Fix needs a liveness check on the `FULL_REINDEX` run record before reconcile may reset the status, plus refusing a second concurrent full reindex.
+- [ ] Drift detection still writes `NEEDS_REINDEX` into `status`, so a few-millisecond window can overwrite `REBUILDING`; the durable fix is a separate drift field (schema change, so 1.1 at the earliest).
+
+## Before releasing 1.1.0
+
+- [x] Re-pointed the `qqq-snapshot` profile from `4.1.0-SNAPSHOT` to `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI (2026-10-11).
+- [ ] qqq-all / Website-Backend: confirm the Next UI search box is served from OpenSearch (provider registered, indexed tables have `searchFields`).
+
+## Later (1.x)
+
+- [ ] Health widget for the admin app (data exists: `isHealthy`, `documentCount`, `lastRunStatus`).
+- [ ] `IndexEvent` sequence number so an async publisher can order INDEX and DELETE; promote the publisher SPI from experimental.
+- [ ] Amazon OpenSearch Serverless (needs removal of `_delete_by_query` and `_refresh` from the design).
+- [ ] Jev table routing prototype; see `docs/reference/typesafe-jev-candidates.md`.
+
+## Done
+
+- [x] Status assessment, deep-dive audit (25 blockers), PRD (2026-10-04).
+- [x] 4.1.0-RC.1 compatibility verified (2026-10-04).
+- [x] All 25 audit blockers implemented and tested on the release branch (2026-10-04).
+
+## Secured-data section 15 (2026-10-14)
+
+- [x] Review fix: separate protection from indexability; preserve safe labels, ordinary results and unprotected hidden keys/timestamps, with regression tests.
+
+- [x] Exclude PII/PHI, hidden, sensitive, password and field-locked values from indexing and results on RC.6.
+- [x] Add canary coverage for real-time indexing, basepull and stale results; refuse classified config fields.
+- [ ] On the QQQ RC.7 upgrade, honour `ClassifiedValues.isClassified(table, field)` (qqq#1182) and `FieldAccessHelper.getLockTier` (qqq#1172).
+- [ ] Deployment: rebuild existing indexes and apply host retention policy to old physical indexes/backups containing previously indexed values.

docs/audits/2026-10-04/01-qqq-4.1-integration.md

@@ -0,0 +1,93 @@
+# Audit 01: qbit-quick-search readiness for QQQ 4.1
+
+Date: 2026-10-04. Read-only audit of `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` (0.2.1-SNAPSHOT, parent `qbit-build-parent` 2.0.0, qqq 4.0.0) against the QQQ tag `v4.1.0-RC.1`, the sibling qbits on their `origin/develop` branches (local checkouts are stale, Dec 2025), `qqq-all` `origin/develop` (e35f1f6, 2026-10-01) and `qbit-bom` `v2.0.0`.
+
+Path abbreviations used below:
+
+- `QS/` = `src/main/java/com/kingsrook/qbits/quicksearch/` in the qbit
+- `CORE/` = `qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/` at `v4.1.0-RC.1`
+- `JAV/` = `qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/` at `v4.1.0-RC.1`
+- `WH/` = `qbit-webhooks` `origin/develop:src/main/java/com/kingsrook/qbits/webhooks/`
+- `ALL/` = `qqq-all` `origin/develop:qqq-all-app/src/`
+
+Taken as given: 233 unit tests and 12 Testcontainers integration tests pass on qqq 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1. Nothing below is a compile or test failure; everything is a contract, consistency or operability gap.
+
+## Findings table
+
+| ID | Severity | Finding | Evidence (qbit) | Evidence (qqq tag / siblings / consumers) | Recommended change | 1.0 blocker |
+|---|---|---|---|---|---|---|
+| F-01 | High | Producer is a plain class, not a `QBitMetaDataProducer`; it registers no `QBitMetaData`, so the host must stamp one by hand, and the version it stamps is a hand-copied string. | `QS/QuickSearchQBitProducer.java:60` (`public class QuickSearchQBitProducer`), `:112` (`produce(QInstance)`), `:200-203` (static context populated), `:214-215,230-232,260` (`addTable/addProcess/addApp` directly) | `CORE/model/metadata/qbits/QBitMetaDataProducer.java:43,50,56,108-189` (contract: `getQBitConfig`, `getQBitMetaData`, default `produce` that calls `qInstance.addQBit` at `:115`, pushes `QBitProductionContext` at `:117-118`, validates at `:122`, finds producers at `:124`, applies `getTableMetaDataCustomizer`/`getDefaultBackendNameForTables` at `:156-166`, stamps `sourceQBitName` at `:172-175`). Siblings: `WH/WebhooksQBitProducer.java:45,55-64` and `qbit-standard-process-trace` `StandardProcessTraceQBitProducer.java:42,90-94`, `qbit-workflows` `WorkflowsQBitProducer.java:65,77-81` all implement `QBitMetaDataProducer<C>`; only `qbit-user-role-permissions` still uses the gen-1 `QBitProducer` (`UserRolePermissionsQBitProducer.java:41`). Consumer workaround: `ALL/main/java/com/kingsrook/qqq/all/DemoQBits.java:161-163` calls `produce(instance)` then `instance.addQBit(new QBitMetaData()...withVersion("0.2.1-SNAPSHOT"))` itself; `ALL/test/java/com/kingsrook/qqq/all/FullQBitsTest.java:44-48,78-79` depends on the artifactId `quick-search` being present. | Implement `QBitMetaDataProducer<QuickSearchQBitConfig>`; return `QBitMetaData` with groupId `com.kingsrook.qbits`, artifactId `quick-search` (keep this exact string, qqq-all asserts it), version read from a build-time resource or constant kept in sync by the release step, `withNamespace(getNamespace())`, `withConfig(getQBitConfig())`. Keep `withConfig(...)`/`produce(QInstance)` as thin compatibility shims (see migration plan) so `DemoQBits.java:161` keeps compiling; the host's manual `addQBit` at `:162-163` then becomes a duplicate and should be removed in qqq-all once this ships. | Yes |
+| F-02 | High | Real-time indexing runs inside the caller's uncommitted transaction. Post-insert/update/delete customizers fire before any commit, so a rolled-back bulk/ETL write is still indexed; a committed write that OpenSearch rejected is logged and dropped. | `QS/customizers/QuickSearchPostInsertCustomizer.java:67-105` (publishes synchronously, swallows failure at `:99-102`); `QuickSearchPostUpdateCustomizer.java:82-123,149-154` (re-reads with `updateInput.getTransaction()`); `QuickSearchPostDeleteCustomizer.java:68-118` | `CORE/actions/tables/InsertAction.java:286` (`runPostInsertCustomizers` is the last thing before `return insertOutput` at `:292`); `UpdateAction.java:268`; `DeleteAction.java:370-390`. Commit happens later in the caller: `CORE/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java:204-207` commits after the actions ran, `:215-218` rolls back on exception. 4.1 adds the fix: `CORE/actions/QBackendTransaction.java:110-113` (`addAfterCommitCallback`, callbacks discarded on `rollback()` at `:89-96`), `qqq-backend-module-rdbms/.../RDBMSTransaction.java:128` (`runAfterCommitCallbacks` after JDBC commit). Reference use: `qqq-esb/src/main/java/com/kingsrook/qqq/esb/publish/EsbRecordChangeListener.java:88-97`. Not in 4.0.0 (0 hits for `addAfterCommitCallback` in `v4.0.0:CORE/actions/QBackendTransaction.java`). | When `input.getTransaction() != null`, build the events now (the record data is correct inside the transaction) but defer `publisher.publish*` via `transaction.addAfterCommitCallback(...)`; publish immediately only when there is no transaction (RDBMS without a transaction autocommits per statement, `RDBMSInsertAction.java:73-80`). Behind a `Capability`/reflection guard if 4.0 support must be kept. | Yes |
+| F-03 | High | The customizers are the wrong 4.1 hook. 4.1 adds an instance-level `RecordChangeListenerInterface` that is designed for exactly this (fires for INSERT/UPDATE/DELETE, cannot fail the write, carries old records for updates and the stored records for deletes, carries the transaction for after-commit work). The qbit's per-table `withCustomizer` approach also throws when the host already has a customizer in the slot (see F-04). | `QS/QuickSearchQBitProducer.java:571-578` | `CORE/actions/tables/listeners/RecordChangeListenerInterface.java:28-61` (contract, `appliesTo(tableName, type)` at `:56`, `onRecordsChanged` at `:61`); `RecordChangeEvent.java:30-54` (records/oldRecords/transaction semantics per type); `CORE/model/metadata/QInstance.java:1989-1996` (`withRecordChangeListener`); fire points `InsertAction.java:311-328`, `UpdateAction.java:259,488-537` (oldRecords paired by primary key), `DeleteAction.java:365,809-823`; `RecordChangeListenerHelper.java:54-70,78-102` (never fails the write). Introduced by qqq commit `8db90166a`; absent in 4.0.0 (`v4.0.0:CORE/model/metadata/QInstance.java` has 0 `RecordChangeListener` hits). | Replace the three customizers with one `QuickSearchRecordChangeListener implements RecordChangeListenerInterface`: `appliesTo` returns true only for discovered tables (so no extra old-record fetches elsewhere), `onRecordsChanged` builds INDEX events from `event.getRecords()` for INSERT/UPDATE and DELETE events from `event.getRecords()` for DELETE, and publishes after commit when `event.getTransaction() != null` (F-02). The update re-read in `QuickSearchPostUpdateCustomizer.java:132-155` stays necessary (UPDATE `records` are "as the backend returned them", which for RDBMS is the sparse input), so keep it, using `event.getTransaction()`. Register once in `postProduceActions` with `qInstance.withRecordChangeListener(new QCodeReference(...))`. Keep the customizer classes for one release as a 4.0 fallback if dual support is required. | Yes (choose F-03 or F-04; F-03 preferred) |
+| F-04 | High | `QTableMetaData.withCustomizer` throws `IllegalArgumentException` when the slot is occupied, so any host table that already has a POST_INSERT/UPDATE/DELETE customizer makes `produce` blow up (and not with a `QException`). | `QS/QuickSearchQBitProducer.java:571-578` | `CORE/model/metadata/tables/QTableMetaData.java:644-657` (`:651-654` throws `IllegalArgumentException("Attempt to add a second customizer with role ...")`); same at `v4.0.0`. Composition options: `CORE/actions/customizers/MultiCustomizer.java:66-70` (`MultiCustomizer.of(...)`) and `:79-83` (`addTableCustomizer(existing, codeRef)`); instance-level `CORE/model/metadata/QInstance.java:1927-1945` (`withTableCustomizer(role, codeRef)`), consumed in `InsertAction.java:356-362` and `UpdateAction.java:303-309` but NOT in `DeleteAction.java` (0 hits for `getTableCustomizers`), so instance-level customizers never fire for deletes. Sibling precedent: `WH/WebhooksQBitProducer.java:102-103` (instance-level, insert/update only), `WH/model/Webhook.java:154-157` (`MultiCustomizer.of`). | If customizers are kept (instead of F-03): read `table.getCustomizers().get(role)`; if absent add directly; if present and it is a `QCodeReferenceWithProperties` for `MultiCustomizer`, call `MultiCustomizer.addTableCustomizer`; otherwise replace the slot with `MultiCustomizer.of(existing, ours)` via `table.getCustomizers().put(role, ...)` (the map is mutable). Do not use `QInstance.withTableCustomizer` for this qbit: it is not per-table (it would have to filter by `QuickSearchQBitContext.getTableConfig`) and it skips deletes. | Yes (if F-03 not taken) |
+| F-05 | High | Processes are produced without `QScheduleMetaData` or `BasepullConfiguration`, so nothing ever runs the basepull or reconcile unless the host schedules it by hand; `enableScheduledProcesses` is misnamed (it only gates registration). The qbit hand-rolls last-run tracking on its own table instead of QQQ's basepull support, and the basepull window has a gap (query at T0, stamp `Instant.now()` at T1 after the batches finish; rows modified in (T0, T1] with the same second are skipped next run). | `QS/QuickSearchQBitProducer.java:509-549` (three `QProcessMetaData` with only a step); `:228-233`; `QS/QuickSearchQBitConfig.java:55` (`enableScheduledProcesses = true`); `QS/processes/BasepullIndexStep.java:202-209` (filter `GREATER_THAN lastBasepullTime`), `:268-278` (stamps `Instant.now()` after processing); `README.md:306-307` describes it as a schedule | `CORE/model/metadata/processes/QProcessMetaData.java:75,564` (`withSchedule(QScheduleMetaData)`), `:61,598` (`withBasepullConfiguration`); `CORE/model/metadata/scheduleing/QScheduleMetaData.java:40-49,89,284` (`repeatSeconds`, `cronExpression`, `schedulerName`); `CORE/scheduler/QScheduleManager.java:284-289,413-429` (processes with a schedule are set up against `getSchedule().getSchedulerName()`); `JAV/QApplicationLauncher.java:139-153,216-222` (launcher starts the scheduler only if some process/queue/automation has a schedule); `CORE/instances/QInstanceValidator.java:2255-2258,2290-2326` (a schedule must have repeat or cron, and a scheduler name that exists in the instance); `CORE/processes/implementations/basepull/BasepullConfiguration.java:33-43` (table/keyField/lastRunTimeFieldName/timestampField/seconds-to-subtract); `CORE/actions/processes/RunProcessAction.java:147-155,183-187,914` (framework reads the stored timestamp before the step, stores the new one after, keyed by process name). Consumers: `ALL/.../DemoQBits.java:63` adds `SimpleSchedulerMetaData "demoScheduler"`, `:106,171` pass `withSchedulerName("demoScheduler")` to webhooks and sftp. | Add `schedulerName` (nullable) and `basepullRepeatSeconds`/`reconcileCronExpression` (nullable) to `QuickSearchQBitConfig`; when `schedulerName` is set, attach `new QScheduleMetaData().withSchedulerName(...).withRepeatSeconds(...)` to the basepull process and a cron schedule to reconcile. Leave processes unscheduled when `schedulerName` is null (hosts that use the `scheduledJob` table can still schedule them). Validate `schedulerName` in `validate` only when `qInstance.getSchedulers()` is non-empty (see F-07). Basepull: either adopt `BasepullConfiguration` (one process per table would be needed because `BASEPULL_KEY_VALUE` is per run; or run one process that overrides `basepullKeyValue` per table) or keep the own table but stamp the run start time (captured before the first query, minus a small overlap) instead of `Instant.now()` after the loop. Rename `enableScheduledProcesses` to `enableIndexingProcesses` with a deprecated alias. | Yes |
+| F-06 | High | Search results carry no permission filtering. `QuickSearchAction` returns table/recordId/label from OpenSearch for any table the index holds, regardless of the session's table permissions or record security locks; produced tables/processes/app set no `QPermissionRules`, which is fine for the demo (instance default is `NOT_PROTECTED`) but means a host that sets a restrictive `defaultPermissionRules` silently locks the admin app, and a host that does not gets an unprotected admin app. | `QS/actions/QuickSearchAction.java:63-130` (no `PermissionsHelper` use; `:81-82` reads static context); `QS/QuickSearchQBitProducer.java:457-549` (no `withPermissionRules`) | `CORE/actions/tables/RecordSearchAction.java:103-113` (core search skips tables without `PermissionsHelper.hasTablePermission(input, table, READ)` and queries through `QueryAction` so record locks apply, `:63-67`); `CORE/instances/QInstanceEnricher.java:544-575` (null rules get `qInstance.getDefaultPermissionRules()`, else `QPermissionRules.defaultInstance()` = `NOT_PROTECTED`, `CORE/model/metadata/permissions/QPermissionRules.java:44-47`). Siblings: `qbit-workflows` processes set `withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))` explicitly (`GetWorkflowRevisionProcess.java:72` etc.); qqq-all protects webhook tables/processes itself (`ALL/test/.../FullQBitsTest.java:67-77`, `DemoUsers.protectWebhookResources`). | In `QuickSearchAction`: require a `QContext` session; filter `tableConfigs` to tables with READ permission before building the OpenSearch query (pass the filtered table list as a `terms` filter on `sourceTable`); for tables with record security locks, re-verify hits by primary key through `QueryAction` (or document that locked tables are not indexable). Add `permissionRules` to the config for the admin app/tables/processes, defaulting to `HAS_ACCESS_PERMISSION` for the processes and `READ_INSERT_EDIT_DELETE_PERMISSIONS` for the two operational tables, with `NOT_PROTECTED` opt-out, mirroring what qqq-all does for webhooks. | Yes |
+| F-07 | Medium | `validate` does a backend lookup, which works today only because the qbit is produced after backends exist. Under `QBitMetaDataProducer` the same timing holds (validate runs inside `produce` with the host's instance), but if a host lets `MetaDataProducerHelper` discover the producer by package scan, it is instantiated with a no-arg constructor and null config, and `QBitMetaDataProducer.produce` dereferences `getQBitConfig()` with no null check. | `QS/QuickSearchQBitConfig.java:73-81` (`qInstance.getBackend(backendName) == null`); `QS/QuickSearchQBitProducer.java:117-122` | `CORE/model/metadata/qbits/QBitMetaDataProducer.java:113-122` (`qBitConfig.validate(qInstance)` on the live instance; no null guard); `CORE/model/metadata/qbits/QBitConfig.java:58-75` (exceptions from `validate` are caught and only logged at `:66-69`, so a thrown NPE inside the list-based validate is swallowed, and only `errors` entries fail); `CORE/model/metadata/MetaDataProducerHelper.java:200-202,585-595` (any `MetaDataProducerInterface` in the scanned package with a no-arg constructor is instantiated and run). Known gap note: `WH/WebhooksQBitConfig.java:85-99` ("qbit validation is happening before the full instance is built"; backend and scheduler checks commented out). `QInstanceValidator` has no qbit-specific checks (0 hits for `sourceQBit`/`qbit` in `CORE/instances/QInstanceValidator.java`), so produced metadata passes as long as apps list every child in a section (`:2597-2605`; the enricher creates a default section) and schedules reference a known scheduler (`:2323-2326`). | Keep the list-based `validate`; guard null `qInstance` and null `getBackends()`; return `isEnabled() == false` from the producer when `getQBitConfig() == null` so package scanning skips it (`MetaDataProducerHelper.java:323,345` honors `isEnabled`). Add a `QInstanceValidatorPluginInterface` (registered via `QInstanceValidator.addValidatorPlugin`, `:319`) for late checks (backend exists, each searchable table exists, scheduler exists) so they run against the full instance. | No |
+| F-08 | Medium | Static `QuickSearchQBitContext` holds config/client/publisher/tables for the JVM. Two instances in one JVM (tests, namespace use, blue/green instance reload via `QInstance` hot swap) collide; customizers and steps read it instead of the instance. 4.1 gives two sanctioned places: `QBitMetaData.getConfig()` reachable from any `SourceQBitAware` table/process, and `QBitProductionContext` during production. | `QS/QuickSearchQBitContext.java:32-37` (static volatile fields), `:152-158` (`clear()` for tests); readers `QS/processes/AbstractIndexingStep.java:68-73,85-90,102-122`, `QS/customizers/*:74,89,75`, `QS/actions/QuickSearchAction.java:81-82` | `CORE/model/metadata/qbits/SourceQBitAware.java:54-75` (`getSourceQBit()` / `getSourceQBitConfig()` via `QContext.getQInstance().getQBits()`), implemented by `QTableMetaData` and `QProcessMetaData` (not `QAppMetaData`); `CORE/model/metadata/qbits/QBitProductionContext.java:48-66` (config stack during produce). Sibling pattern: `WH/WebhooksQBitConfig.java:146-157` (`getConfigValue` resolves config from a produced table's `getSourceQBitConfig()`, with a default fallback); `WH/model/WebhookEvent.java:121` (`QBitProductionContext.peekQBitConfig()` inside component producers). | Keep `QuickSearchQBitConfig` as the single source of truth and put the runtime objects (client, publisher, discovered tables) on it lazily (`transient`, built on first use, closed by a runtime service, F-10). Resolve the config in steps via `QContext.getQInstance().getProcess(input.getProcessName()).getSourceQBitConfig()`, in the listener via the discovered table's `getSourceQBitConfig()`, and in `QuickSearchAction` via an explicit `QuickSearchInput.withQBitName`/default lookup over `qInstance.getQBits()`. Keep the static accessors as deprecated delegates for one release. | No |
+| F-09 | Medium | Produce is "best effort" on OpenSearch: a missing or unreachable cluster yields a null client and publisher, the warnings are the only signal, every write then logs "publisher is null", and `QuickSearchAction` throws at query time. There is no fail-fast toggle and no health exposure. qqq-all depends on the degraded behaviour: `FullQBitsTest` builds the full instance with OpenSearch at `127.0.0.1:1`. | `QS/QuickSearchQBitProducer.java:159-167,172-182,187-195`; `QS/customizers/QuickSearchPostInsertCustomizer.java:74-79`; `QS/actions/QuickSearchAction.java:84-87`; `QS/opensearch/QuickSearchOpenSearchClient.java:94-150` (transport is built lazily; only `ensureIndexExists` at `:161` touches the network) | `JAV/QApplicationLauncher.java:118-175` (start order: server, scheduler, runtime services; a thrown `start` stops what started and fails `run`), `:192-207` (`failOnMetaDataProducerError` only reaches producer-based apps); `CORE/instances/QRuntimeServiceInterface.java:39-68` (`start(QInstance)`, `stop()`, `afterApplicationStop()`); `CORE/model/metadata/QInstance.java:2153` (`withRuntimeService`). Consumer: `ALL/test/.../FullProfileTest.java:123-124` (`OPENSEARCH_HOST=127.0.0.1`, `OPENSEARCH_PORT=1`) and `FullQBitsTest.java:41-42` (instance must define and validate with that); `ALL/main/.../QqqAllApplication.java:98` sets `withFailOnMetaDataProducerError(true)`, `:154` registers a runtime service; `qqq-all` plan: "disabled with a notice in core" (`origin/develop:docs/plan.md:80`). | Add `startupMode` to the config: `DEGRADED` (default, current behaviour, keeps qqq-all's test green) and `FAIL_FAST` (throw `QBitConfigValidationException`/`QException` from `produce` when the client or index cannot be created). Move the connect and `ensureIndexExists` out of `produce` into a `QuickSearchRuntimeService implements QRuntimeServiceInterface` registered with `qInstance.withRuntimeService(...)` in `postProduceActions`: `start` connects (fail-fast or warn per mode), `stop` closes the transport (today nothing calls `SynchronousIndexEventPublisher.close()` or `QuickSearchOpenSearchClient.close()`, `QS/publisher/SynchronousIndexEventPublisher.java:179-182`, `QS/opensearch/QuickSearchOpenSearchClient.java:595-606`). Expose a `quickSearchHealth` process or widget that reports connected/index exists/document count so the host's `/health` can show a notice. | No |
+| F-10 | Medium | Lifecycle: nothing closes the OpenSearch transport or publisher at shutdown; index creation happens during metadata production (a side effect on an external system inside `defineQInstance`). | `QS/QuickSearchQBitProducer.java:174-182`; `QS/publisher/SynchronousIndexEventPublisher.java:179-182` (only `close()` path, never invoked) | `JAV/QApplicationLauncher.java:155-168,243-273` (runtime services started after the instance is built, stopped in reverse order); `CORE/instances/QRuntimeServiceInterface.java:47-55` | Covered by F-09: runtime service owns connect/ensureIndex/close. | No |
+| F-11 | Medium | Core 4.1 ships `RecordSearchAction` and `POST /qqq/v1/search` with no provider SPI; Quick Search cannot back it, so a 4.1 host gets two incompatible "search" features (core: LIKE over `searchFields`; qbit: OpenSearch) and Next's search UI will call the core one. | `QS/actions/QuickSearchAction.java:63-130`, `QS/actions/QuickSearchInput.java`, `QuickSearchOutput`, `QuickSearchResult` (parallel model, no relation to core) | `CORE/actions/tables/RecordSearchAction.java:71-119` (`new RecordSearchAction().execute(input)` concrete class, iterates `qInstance.getTables()` with `getSearchFields()`, no code-reference indirection); `CORE/model/actions/tables/search/RecordSearchInput.java:35-40` (`searchTerm`, `tableNames`, `limitPerTable`, `inputSource`); `RecordSearchResult` (tableName, tableLabel, recordId, recordLabel); `JAV/executors/RecordSearchExecutor.java:51-57` (hard-wires `new RecordSearchAction()`); `JAV/specs/v1/RecordSearchSpecV1.java:67-68` (`POST /search`); qqq commit `7dd36c8ad`; `QTableMetaData.java:100,2016` (`searchFields`), validator `QInstanceValidator.java:1282-1298`. | Smallest core SPI (propose in qqq): `QInstance.withRecordSearchProvider(QCodeReference)` plus `interface RecordSearchProviderInterface { boolean appliesTo(QTableMetaData table); List<RecordSearchResult> search(RecordSearchInput input, QTableMetaData table, String term, int limit) throws QException; }`. `RecordSearchAction.searchTable` consults the provider when `appliesTo` is true (and the table has `searchFields` or the provider says it is searchable), still after the permission check at `:110`, so record-security filtering stays core's job. Until then: make `QuickSearchResult` shape-compatible with `RecordSearchResult` (add `tableLabel`), and set `searchFields` on indexed tables via the table customizer so Next's UI at least shows the search box; document the overlap. | No (SPI is a qqq change; the qbit-side adapter is 1.1) |
+| F-12 | Low | Namespace support is absent: produced names are prefixed with the qbit's own `tableNamePrefix`, not with the `QBitMetaData.namespace`; `getNamespace()` is not implemented. Core does not prefix names automatically either, so a namespace-aware qbit has to do it itself. | `QS/QuickSearchQBitConfig.java:169-195` (`applyPrefix`, `tableNamePrefix`); `QS/QuickSearchQBitProducer.java:208-209,220-222,246` | `CORE/model/metadata/qbits/QBitMetaData.java:60-68` (namespace is part of the qbit's unique name only); `QBitMetaDataProducer.java:62-65` (`getNamespace()` default null); qqq test qbit does no name prefixing (`qqq-backend-core/src/test/.../testqbit/metadata/SomeTableMetaDataProducer.java:59-61`); `ALL/.../DemoQBits.java:120,182` pass a namespace to gen-1 producers (`produce(instance, "shipping-geo")`). | Add `namespace` to the producer (`withNamespace`), use it as the default `tableNamePrefix` when the prefix is null, and feed it to `QBitMetaData.withNamespace`. Keep `tableNamePrefix` as an override. | No |
+| F-13 | Low | Build: the pom declares `jacoco-maven-plugin` with a `check` execution but no version and no `prepare-agent`, so the check is skipped ("missing execution data file") and the plugin version floats; the parent manages no jacoco plugin. JUnit is pinned to 5.10.0 (BOM import and direct dependency) while the parent manages `junit-bom` 6.0.1. | `pom.xml:156-181` (jacoco `check` only), `:69-77` (`junit-bom` 5.10.0 import), `:118-122,147-150` (junit-jupiter 5.10.0) | `qbit-bom` `v2.0.0:pom.xml:54-55` (`qqq-bom-pom` 4.0.0), `:62-63` (`junit-bom` 6.0.1), `:70-90` (pluginManagement contains only `maven-compiler-plugin`); no 2.1.0 tag exists yet (`git tag` ends at `v2.0.0`; `origin/main aaf52d2` "prepare build parent 2.0.0 for QQQ 4.0"); CI orb `kingsrook/qqq-orb@0.5.2` in `.circleci/config.yml:4` (parent uses 0.5.0). qqq's own BOM manages no JUnit (`v4.1.0-RC.1:qqq-bom/pom.xml` has no junit entries). | Either add `prepare-agent` (and a `prepare-agent-integration`/`report` pair for failsafe) with an explicit plugin version and make the 0.70/0.90 thresholds real, or delete the jacoco block (ADR). Drop the local `junit-bom` import and the 5.10.0 pins and inherit 6.0.1 from the parent (JUnit 6 requires Java 17+, fine on 21). When `qbit-build-parent` 2.1.0 (qqq 4.1) is released, re-pin and delete the `qqq-snapshot` profile's `qqq-bom-pom` import (ADR-0007). | No |
+| F-14 | Low | Source-table batch paging is unordered (`limit`/`skip` with no `orderBy`), so full reindex, reconcile and basepull can skip or duplicate rows on backends without a stable default order. | `QS/processes/AbstractIndexingStep.java:348-374` (no `withOrderBy`); `QS/processes/BasepullIndexStep.java:220-262` | `CORE/actions/tables/RecordSearchAction.java:165` (core orders by primary key before limiting, as a reference pattern) | Add `effectiveFilter.withOrderBy(new QFilterOrderBy(primaryKeyField))` and prefer keyset paging (`primaryKey > lastSeen`) over `skip`. | No (listed in the PRD's E2) |
+
+## Answers to the audit questions
+
+### 1. QBit contract
+
+What must change (all evidence above, F-01, F-08, F-12):
+
+1. `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>` with `getQBitConfig()`, `getQBitMetaData()` and `postProduceActions(...)` (`CORE/.../QBitMetaDataProducer.java:50,56,70`). The default `produce` scans the producer's own package for `MetaDataProducerInterface` implementations (`:124`, `getPackageNameForFindingMetaDataProducers` at `:80-101`), so the two operational tables, three processes and the app become small producers (`QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>` etc., `CORE/.../QBitComponentMetaDataProducer.java:34-69`) or `@QMetaDataProducingEntity` annotations on `QuickSearchIndex`/`QuickSearchIndexRun` (today they are plain `QRecordEntity` with `@QField`, `QS/model/QuickSearchIndex.java:32-36`). Customizer/listener registration and the OpenSearch bootstrap belong in `postProduceActions`, as webhooks does (`WH/WebhooksQBitProducer.java:89-104`).
+2. `QBitMetaData` with groupId `com.kingsrook.qbits`, artifactId `quick-search`, real version, namespace, config (`CORE/.../QBitMetaData.java:47-52`). The framework then stamps `sourceQBitName` on every produced table and process (`QBitMetaDataProducer.java:172-175`; `QTableMetaData` and `QProcessMetaData` implement `SourceQBitAware`), which is what makes F-08's static context removable.
+3. `QuickSearchQBitConfig` already implements `QBitConfig` (`QS/QuickSearchQBitConfig.java:45`); add `getDefaultBackendNameForTables()` returning `backendName` and `getTableMetaDataCustomizer()` (settable, like `WH/WebhooksQBitConfig.java:563,805`) so hosts can set RDBMS backend details the way `ALL/.../DemoQBits.java:80-88` does for the other qbits.
+
+Idiomatic choice: `QBitMetaDataProducer` (3 of 4 siblings; qqq's own interface doc calls it the "2nd generation ... evolution over QBitProducer", `:39-40`). `QBitProducer` (gen 1) is only kept by user-role-permissions and qqq's test qbit.
+
+qqq-all impact: `DemoQBits.java:161` uses `new QuickSearchQBitProducer().withConfig(searchConfig).produce(instance)`. `QBitMetaDataProducer.produce(QInstance)` returns `MetaDataProducerMultiOutput` instead of `void`, so that call still compiles (the return value is ignored) only if the metadata is added to the instance. The default `produce` does NOT add to the instance; it returns the multi-output and the host calls `.addSelfToInstance(instance)` (`DemoQBits.java:93,101,108,116`). To keep source compatibility: keep `withConfig(...)` as an alias of `withQBitConfig(...)`, and override `produce(QInstance)` to call the interface's default `produce`, then `addSelfToInstance(qInstance)` on the result before returning it (idempotent for hosts that also call `addSelfToInstance`, since `QInstance.addTable` etc. would throw on duplicates; so the override must either add and return an empty output, or be documented as "already added"; returning an empty `MetaDataProducerMultiOutput` after adding is the safe choice). The manual `instance.addQBit(...)` at `DemoQBits.java:162-163` will then duplicate the qbit's own `addQBit` (`QBitMetaDataProducer.java:115`); `QInstance.addQBit` behaviour on duplicate names should be checked when that qqq-all change is made (`CORE/model/metadata/QInstance.java:1800`).
+
+### 2. Validation timing
+
+- `QBitConfig.validate` runs at the start of `QBitMetaDataProducer.produce` (`:122`) against whatever is in the host's instance at that moment. It is not "before the full instance exists" in the sense of the host's backends: hosts add backends first (`ALL/.../FullProfileApplication.java:70-76`). The webhooks note (`WH/WebhooksQBitConfig.java:85-99`) refers to hosts that produce qbits before their own tables/schedulers; the backend lookup in this qbit (`QS/QuickSearchQBitConfig.java:79`) is safe under both the current flow and the `QBitMetaDataProducer` flow for the qqq-all ordering, but a host that produces the qbit before `addBackend` gets a false "Backend not found".
+- Under `MetaDataProducerHelper` scanning the producer would be instantiated with a null config (`MetaDataProducerHelper.java:585-595`) and `produce` would NPE at `QBitMetaDataProducer.java:113-122`; `isEnabled()` returning false when the config is null avoids that (`:145-149`).
+- `QBitConfig.validate(QInstance)` swallows exceptions from the list-based validate and only fails on collected errors (`QBitConfig.java:62-74`), so validation must add to `errors`, never throw.
+- `QInstanceValidator` 4.1 adds nothing qbit-specific. Relevant checks the produced metadata must satisfy: process steps have names and `BackendStep` code (`:2178-2197`), a schedule has repeat or cron plus an existing scheduler (`:2290-2326`), app children appear in sections (`:2597-2605`; the enricher builds a default section), table `searchFields` if set must be visible string/integer fields (`:1282-1298`). Nothing validates `sourceQBitName`.
+
+### 3. Scheduling
+
+- A process is scheduled by `QProcessMetaData.withSchedule(new QScheduleMetaData().withSchedulerName(name).withRepeatSeconds(n) | .withCronExpression(expr).withCronTimeZoneId(tz))` (`QProcessMetaData.java:564`, `QScheduleMetaData.java:40-49`). The host must own a `QSchedulerMetaData` (`SimpleSchedulerMetaData` or `QuartzSchedulerMetaData`, `CORE/model/metadata/scheduleing/{simple,quartz}/*.java:45-54`) and either start `QScheduleManager` itself or use `QApplicationLauncher`, which starts it only when something has a schedule (`JAV/QApplicationLauncher.java:139-153,216-222`). `QScheduleManager.start` sets up every process with a schedule (`:284-289`) under the system session supplier (`:163`), and `qqq.scheduleManager.enabled`/`onlyStartNamesMatching` gate it (`:153`, `SchedulerUtils.java:61-71`). Cron needs Quartz (`SimpleSchedulerMetaData.supportsCronSchedules`, `:56`).
+- Minimal correct way for this qbit: config fields `schedulerName`, `basepullRepeatSeconds` (default 300), `reconcileCronExpression`+`reconcileCronTimeZoneId` (default null, so no reconcile schedule unless the host asks); attach schedules only when `schedulerName` is set. Host provides the scheduler and the launcher (qqq-all already has `demoScheduler`, `DemoQBits.java:63`).
+- Basepull duplication: yes. `BasepullConfiguration` + `RunProcessAction` already persist a last-run timestamp per process key, pass it to the step as `RunBackendStepInput.getBasepullLastRunTime()` (`RunProcessAction.java:724-728`), and only store the new time when the step sets `BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD` (`:183-187`), with overlap seconds (`BasepullConfiguration.java:42-43`). `BasepullIndexStep` re-implements this per table on `quickSearchIndex.lastBasepullTime` (`:202-209,268-278`) and stamps `Instant.now()` after the loop (window gap). The qbit's per-table interval and per-table enable flag are not expressible with one `BasepullConfiguration` (its key is the process name, `RunProcessAction.java:880-885`), so the pragmatic fix is to keep the own table but (a) capture the run start before the first query and store that, minus an overlap, and (b) run the process on a `QScheduleMetaData`. Full adoption of `BasepullConfiguration` would mean one scheduled process per indexed table, which is also legitimate and gives per-table schedules for free.
+
+### 4. Customizer composition
+
+Confirmed: `QTableMetaData.withCustomizer` throws `IllegalArgumentException` on an occupied slot at the 4.1 tag (`QTableMetaData.java:651-654`), same at 4.0.0. Options: `MultiCustomizer.of(a, b)` / `MultiCustomizer.addTableCustomizer(existing, c)` (`MultiCustomizer.java:66-83`), or instance-level `QInstance.withTableCustomizer(role, codeRef)` (`QInstance.java:1927-1945`), which runs for every table on insert/update (`InsertAction.java:356-362`, `UpdateAction.java:303-309`) but not on delete (`DeleteAction.java` has no `getTableCustomizers` call). For per-table targeting, `MultiCustomizer` composition is the right customizer-based answer; but the better 4.1 answer is the `RecordChangeListenerInterface` (F-03), which is instance-level, filters by `appliesTo(tableName, type)`, covers delete, and never fails the write.
+
+### 5. Transactions and consistency
+
+- Customizers run after the backend write and before the action returns, inside the caller's transaction if one was passed (`InsertAction.java:286-292`, `UpdateAction.java:268`, `DeleteAction.java:370-390`); the `RecordChangeListenerInterface` javadoc states this explicitly (`:32-34`). ETL/bulk processes commit after the actions (`StreamedETLExecuteStep.java:204-207`) and roll back on exception (`:215-218`), so a rolled-back bulk insert is indexed today. The javalin single-record endpoints pass no transaction (`QJavalinImplementation.java:689,726,957` set up sessions only; no `openTransaction`), and the RDBMS module then uses a fresh autocommit connection (`RDBMSInsertAction.java:73-80`; only `RDBMSTransaction` sets autocommit false, `RDBMSTransaction.java:55`), so those are effectively committed before the customizer runs.
+- Fix available only in 4.1: `QBackendTransaction.addAfterCommitCallback` (`QBackendTransaction.java:110-113`), honoured by `RDBMSTransaction.commit` (`:128`), discarded on rollback (`:89-96`, `RDBMSTransaction.java:114`).
+- Update customizer receives `Optional<List<QRecord>> oldRecordList` (`UpdateAction.java:289`); records are the sparse updated records, hence the qbit's re-read (`QuickSearchPostUpdateCustomizer.java:132-155`). Delete customizer receives the old records minus those in error (`DeleteAction.java:376-390`), so primary keys are present. The listener API gives the same: UPDATE `oldRecords` paired by primary key (`UpdateAction.java:498-537`), DELETE `records` = stored records before delete (`RecordChangeEvent.java:35,41`).
+- `QContext`/`QSession` exist inside customizers and listeners: javalin calls `setupSession` then `QContext.init(qInstance, session, null, input)` (`QJavalinImplementation.java:495,573`); the scheduler runs under the system session (`QScheduleManager.java:163`). The re-read in the update path therefore runs as the writing user; for indexing that is correct (index what the user could write), but for a permission-aware re-read on behalf of other readers it is not a substitute for F-06.
+
+### 6. Permissions
+
+Siblings mostly leave rules null and let the enricher apply the instance default (`QInstanceEnricher.java:544-575` → `NOT_PROTECTED` unless the host sets `defaultPermissionRules`); workflows sets `NOT_PROTECTED` explicitly on its processes; qqq-all layers its own rules on the webhook tables/processes and asserts them (`FullQBitsTest.java:67-77`). `RecordSearchAction` filters by table READ permission and queries through `QueryAction` so record locks apply (`RecordSearchAction.java:103-113,168-177`). There is no provider/SPI: the action is a concrete class with no code-reference indirection, and the javalin executor instantiates it directly (`RecordSearchExecutor.java:51`). Smallest SPI sketched in F-11.
+
+### 7. Application launcher and qqq-all
+
+`QApplicationLauncher` (`JAV/QApplicationLauncher.java:106-184`) starts the server (which builds the instance), the scheduler if anything is scheduled, then runtime services; failures stop what started and throw. Two things this qbit should support: a `QRuntimeServiceInterface` for connect/ensure-index/close with a fail-fast or degraded mode (F-09, F-10), and health reporting the host can surface. qqq-all's core profile excludes the qbit entirely (`CoreQBitsTest.java:42`) and its README says "Quick search is disabled in core because it requires OpenSearch" (`ALL/main/.../README.md:14`); the full profile's unit test builds the instance with an unreachable OpenSearch (`FullProfileTest.java:123-124`), so the degraded mode must remain the default. qqq-all also runs with `withFailOnMetaDataProducerError(true)` (`QqqAllApplication.java:98`), which only affects `MetaDataProducerHelper`; the qbit's own `produce` must throw a `QException` with a clear message when fail-fast is requested.
+
+### 8. Build and release
+
+Parent `v2.0.0` pins `qqq-bom-pom` 4.0.0 and `junit-bom` 6.0.1 and manages only the compiler plugin (`qbit-bom v2.0.0:pom.xml:54-55,62-63,70-90`); no 2.1.0 tag exists yet. The qbit's jacoco `check` has no `prepare-agent` and no version (`pom.xml:156-181`), so coverage is never measured and the plugin version floats with the repository metadata; the local JUnit 5.10.0 pins (`pom.xml:69-77,118-122,147-150`) override the parent's 6.0.1. The `qqq-snapshot` profile is correct for RC checks and should be removed or re-pointed when parent 2.1.0 ships. The CI orb versions differ (0.5.2 in the qbit, 0.5.0 in the parent's own CI) but that is harmless.
+
+## Minimal migration plan (ordered)
+
+1. Producer contract (F-01, F-12): make `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`; add `getQBitMetaData()` (artifactId `quick-search`, version from a `quick-search.properties` resource filtered by Maven), `getNamespace()`; move table/process/app building into package-scanned component producers (or `@QMetaDataProducingEntity` on the two entities plus small process/app producers); move customizer registration and client bootstrap into `postProduceActions`. Keep `withConfig`, `getConfig` and a `produce(QInstance)` override that adds to the instance and returns an empty multi-output, so `qqq-all DemoQBits.java:161` keeps compiling; mark them `@Deprecated`.
+2. Config surface (F-05, F-06, F-09): add `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`, `permissionRules` (app/tables/processes), `startupMode` (`DEGRADED` default, `FAIL_FAST`), `tableMetaDataCustomizer` setter, `getDefaultBackendNameForTables()` = `backendName`. Deprecate `enableScheduledProcesses` in favour of `enableIndexingProcesses`. Validation stays list-based and null-safe; add a `QInstanceValidatorPluginInterface` for late checks (F-07).
+3. Real-time path (F-02, F-03, F-04): add `QuickSearchRecordChangeListener implements RecordChangeListenerInterface`, registered via `qInstance.withRecordChangeListener` in `postProduceActions`; publish after commit through `event.getTransaction().addAfterCommitCallback(...)` when a transaction is present. Keep the three customizers only as a documented 4.0 fallback (`enableRealTimeIndexing` + a `legacyCustomizers` flag), composing with `MultiCustomizer` when a slot is occupied; delete them in 1.1.
+4. Runtime state (F-08, F-10): add `QuickSearchRuntimeService implements QRuntimeServiceInterface` (connect, `ensureIndexExists`, close) registered via `qInstance.withRuntimeService`; resolve the config from `SourceQBitAware.getSourceQBitConfig()` in steps and the listener; keep `QuickSearchQBitContext` as a deprecated facade over the instance-bound state for one release.
+5. Scheduling and basepull (F-05, F-14): attach `QScheduleMetaData` when `schedulerName` is set; capture the basepull run-start timestamp before the first query and store it minus an overlap; add primary-key `orderBy`/keyset paging in `querySourceTableBatch`.
+6. Permission-aware search (F-06): filter table configs by `PermissionsHelper.hasTablePermission(input, table, READ)` and pass the allowed `sourceTable` set to the OpenSearch query; verify hits for tables with record security locks via `QueryAction`; align `QuickSearchResult` with core `RecordSearchResult` (add `tableLabel`) ahead of the SPI proposal (F-11).
+7. Build (F-13): decide jacoco (bind `prepare-agent` or remove), drop the JUnit 5.10.0 pins, re-pin to `qbit-build-parent` 2.1.0 when released and retire the `qqq-snapshot` profile's BOM import; bump to 0.3.0 for the compat release, 1.0.0 after qqq-all removes its manual `addQBit`.
+8. Downstream: qqq-all PR removing `DemoQBits.java:162-163`, passing `withSchedulerName("demoScheduler")`, and asserting `instance.getQBits()` contains `quick-search` from the qbit itself; propose the `RecordSearchProviderInterface` SPI in qqq (F-11).

docs/audits/2026-10-04/02-opensearch-aws-compat.md

@@ -0,0 +1,160 @@
+# Audit 02: OpenSearch and AWS compatibility of qbit-quick-search
+
+Date: 2026-10-04. Repo reviewed read-only at `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` (branch `feature/GH-6-repin-and-drift-fix`). Compile experiments were run on a copy under the scratchpad, never in the repo.
+
+Files reviewed: `src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java` (CLIENT), `OpenSearchDocument.java`, `BulkIndexResult.java`, `QuickSearchQBitConfig.java` (CONFIG), `processes/IndexingUtils.java`, `actions/QuickSearchAction.java`, `pom.xml`, and the three `*IntegrationTest.java` classes.
+
+## 0. Resolved dependency facts (from `mvn dependency:tree` in the repo)
+
+| Artifact | Resolved | Note |
+| --- | --- | --- |
+| org.opensearch.client:opensearch-java | 2.10.0 | Pinned in pom.xml line 24. Its own POM declares httpclient5 5.3.1, httpcore5 5.2.4, jackson 2.17.0, and a runtime dependency on opensearch-rest-client 2.12.0 (https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/2.10.0/opensearch-java-2.10.0.pom) |
+| org.opensearch.client:opensearch-rest-client | 2.12.0 (runtime) | Pulled only because the 2.x client still ships the deprecated RestClientTransport; the 3.x client POM no longer lists it (https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/3.9.0/opensearch-java-3.9.0.pom) |
+| org.apache.httpcomponents.client5:httpclient5 | 5.6.3 | Managed by the qqq BOM |
+| org.apache.httpcomponents.core5:httpcore5 / httpcore5-h2 | 5.4.3 | Managed by the qqq BOM |
+| com.fasterxml.jackson.core:* | 2.21.5 | Managed by the qqq BOM |
+| software.amazon.awssdk:* (auth, regions, apache-client, netty-nio-client, ...) | 2.41.10 | Already on the classpath, but only transitively via qqq-backend-core -> quicksight. Not something this library should rely on without declaring it |
+
+Maven Central: latest opensearch-java is 3.10.0 (`<latest>3.10.0</latest>` in https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/maven-metadata.xml); the 2.x line ended at 2.26.0. The OpenSearch Java client docs now install 3.10.0 (https://docs.opensearch.org/latest/clients/java/).
+
+## 1. Research findings (external)
+
+### 1.1 opensearch-java client
+
+* Compatibility matrix: client 2.x.0 supports OpenSearch 1.x to 2.x; client 3.x.0 supports 1.x to 3.x. JDK 8/11/17/21 for both lines. (https://github.com/opensearch-project/opensearch-java/blob/main/COMPATIBILITY.md)
+* `ApacheHttpClient5Transport` is the default and recommended transport; `RestClientTransport` and `org.opensearch.client.RestClient` are deprecated and will be removed. (https://docs.opensearch.org/latest/clients/java/)
+* OpenSearch 3.0 brings HTTP/2 and `ApacheHttpClient5Transport` switches to HTTP/2 when available; `HttpVersionPolicy` can be forced through `setHttpClientConfigCallback`. (https://github.com/opensearch-project/opensearch-java/blob/main/USER_GUIDE.md)
+* 3.0.0 breaking changes (https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/CHANGELOG.md and https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/UPGRADING.md): more conservative URL path encoding, `searchAfter` and `Hit.sort` now `FieldValue`, `listAllPit` -> `getAllPits`, several type corrections, removal of ES-only namespaces (`features`, `shutdown`, `termsEnum`, `indices.upgrade`). Nothing in the transport builders or `JacksonJsonpMapper` changed. One undocumented change does hit this repo: `DeleteByQueryRequest.Builder.refresh(...)` takes the `Refresh` enum instead of `boolean` (see empirical test below).
+* Empirical compile check (scratch copy, `opensearch.version` set to 3.10.0): exactly one compile error, CLIENT line 427 `.refresh(true)` ("boolean cannot be converted to org.opensearch.client.opensearch._types.Refresh"). After changing it to `Refresh.True`, `mvn test-compile` and all 233 unit tests pass. Resolved tree with 3.10.0: httpcore5 5.4.3, httpclient5 5.6.3 (BOM), plus a new `commons-logging:1.3.6` transitive. The 3.x client POM is built against httpclient5 5.6 / httpcore5 5.4.2, so the BOM versions are in range.
+* httpcore5/httpclient5 constraint with the current 2.10.0 client: it was built against httpclient5 5.3.1 / httpcore5 5.2.4 but runs on 5.6.3 / 5.4.3 here; the repo already carries a workaround for the 5.6 content-compression behaviour (CLIENT lines 124 to 130). Moving to the 3.x client removes that version skew.
+* Elasticsearch-compatible endpoints: the compatibility matrix lists only OpenSearch. A maintainer answer on the forum says clients generally only bridge adjacent majors (ES 7.10 <-> OpenSearch 1.x) and that 2.x clients against ES 7.10 hit renamed APIs. (https://forum.opensearch.org/t/opensearch-api-compatibility-with-elastic-search/17647). Legacy Elasticsearch 7.10 domains on AWS should be declared out of scope.
+* SigV4: `AwsSdk2Transport(SdkHttpClient, endpointWithoutScheme, "es" | "aoss", Region, AwsSdk2TransportOptions)`; credentials from the default provider chain; `aws-crt-client` is the recommended SdkHttpClient and the AWS SDK `ApacheHttpClient` is discouraged (it cannot send bodies on GET/DELETE and `AwsSdk2Transport` throws `TransportException` for such calls). (https://docs.opensearch.org/latest/clients/java/#connecting-to-amazon-opensearch-service and https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md). The AWS SDK artifacts are compileOnly in the client, so the application must declare them (release notes for #628 in https://github.com/opensearch-project/opensearch-java/releases/tag/v3.0.0).
+
+### 1.2 Amazon OpenSearch Service (managed domains)
+
+* Engine versions offered today: OpenSearch 3.5, 3.3, 3.1, 2.19, 2.17, 2.15, 2.13, 2.11, 2.9, 2.7, 2.5, 2.3, 1.3 to 1.0, plus legacy Elasticsearch up to 7.10. 2.3 to 2.9 left standard support in Nov 2025; 2.11 to 2.17 end Nov 2027. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html)
+* Endpoint and TLS: "OpenSearch Service only accepts connections over port 80 (HTTP) or 443 (HTTPS)" (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html); TLS 1.2 required, 1.3 recommended (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/infrastructure-security.html). No custom port; endpoints are `https://search-<domain>-<id>.<region>.es.amazonaws.com` (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/createupdatedomains.html).
+* FGAC: master user is either an IAM principal or an internal-database user. With an internal master user "you can use HTTP basic authentication (as well as IAM credentials)". If the domain access policy names IAM principals, clients must sign with SigV4; "You can't sign a request with a username and password and IAM credentials", so AWS recommends an access policy that does not require signed requests when using basic auth. Master user roles are `all_access` and `security_manager`. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html)
+* Signing service name for domains is `es`; for Serverless it is `aoss` and `x-amz-content-sha256` is required. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-comparison.html)
+* Supported operations: the per-version operation lists for managed domains include `/_bulk`, `/_delete_by_query`, `/_refresh`, `/_reindex`, `/_update_by_query` and all index-path operations, so every call this library makes is available on a managed domain. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html)
+* Request payload limits: 10 MiB on t2/t3 and `*.large` instance types, 100 MiB on xlarge and above. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html)
+* Permissions on a FGAC domain follow the security plugin model (section 1.4).
+
+### 1.3 Amazon OpenSearch Serverless (AOSS)
+
+* Supported API table (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html): `aoss:CreateIndex` (PUT index), `aoss:DescribeIndex` (HEAD index, GET mappings/settings), `aoss:WriteDocument` (POST _bulk, DELETE index/_doc/id, PUT index/_doc/id "for search collection types only", POST index/_doc), `aoss:ReadDocument` (GET/POST _search, _count, _msearch, PIT). `_delete_by_query`, `_update_by_query`, `_reindex` and `_refresh` do not appear anywhere in the table. Aliases exist under `aoss:CreateCollectionItems` / `UpdateCollectionItems`.
+* Collection types: Search, Vector search, Time series. Custom document IDs and upserts are only allowed for search collections. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-overview.html and the Python note in https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html)
+* Settings: "The number of shards, number of intervals, and refresh interval are not modifiable and are handled by OpenSearch Serverless"; refresh interval is approximately 10 seconds for search and time series collections. (serverless-overview.html)
+* Clients must sign with service `aoss`, include `x-amz-content-sha256`, and "must be compatible with OpenSearch 3.x" (serverless-overview.html; the older serverless-clients.html page still says 2.x). The OpenSearch Java docs note AOSS does not support the `refresh` parameter and does not serve the root endpoint (https://docs.opensearch.org/latest/clients/java/#connecting-to-amazon-opensearch-serverless).
+* Data access policy minimum for an ingesting principal: `aoss:CreateIndex`, `aoss:WriteDocument`, `aoss:UpdateIndex` on `index/<collection>/<index>`; this library also needs `aoss:DescribeIndex` (exists check) and `aoss:ReadDocument` (search). (serverless-clients.html)
+* Quotas: 1000 indexes per collection (https://docs.aws.amazon.com/general/latest/gr/opensearch-service.html). No per-request payload limit is documented on that page.
+* Which of this library's calls fail on AOSS: the connection itself (basic auth transport, no SigV4), `deleteDocumentsForTable` (`_delete_by_query`, CLIENT line 392), `deleteDocumentsIndexedBefore` (`_delete_by_query` with `refresh=true`, CLIENT line 424), `refreshIndex` (`_refresh`, CLIENT line 579), and all indexing with custom `_id` unless the collection is of type Search. `ensureIndexExists`, bulk index/delete, single delete, and search with highlight and from/size are within the supported surface.
+
+### 1.4 Self-managed OpenSearch with the security plugin
+
+* Demo config and Docker image: since 2.12 a custom admin password in `OPENSEARCH_INITIAL_ADMIN_PASSWORD` is required or the node does not start; the password must be 8 to 100 chars, mixed classes, and rated "strong" by zxcvbn. `DISABLE_SECURITY_PLUGIN=true` disables the plugin; the demo config installs self-signed certificates. (https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/ and https://docs.opensearch.org/latest/security/configuration/demo-configuration/)
+* TLS with self-signed certs: the official Java client example builds an `SSLContext` (trust-all for demo, or a truststore) and installs it via `ClientTlsStrategyBuilder` + `PoolingAsyncClientConnectionManager` inside `setHttpClientConfigCallback`; the RestClient transport variant relies on the JVM truststore system properties. (https://docs.opensearch.org/latest/clients/java/#connecting-to-opensearch)
+* Auth backends: HTTP basic against the internal user database; client certificate (mTLS) via `plugins.security.ssl.http.clientauth_mode` and a `clientcert` auth domain mapped to roles by CN (https://docs.opensearch.org/latest/security/authentication-backends/client-auth/); JWT bearer tokens (HMAC, RSA, ECDSA, JWKS since 3.3) in the `Authorization` header (https://docs.opensearch.org/latest/security/authentication-backends/jwt/); API tokens with permissions bound to the token were introduced in OpenSearch 3.7 (https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-3.7.0.md).
+* Permissions for this library's operations (default action groups, https://docs.opensearch.org/latest/security/access-control/default-action-groups/): index-level `create_index` (`indices:admin/create`, `indices:admin/mapping/put`) for `ensureIndexExists`; `read` or `search` for `_search`; `write` (`indices:data/write*`, which covers index, bulk items, delete, and `delete/byquery`) and `delete`; `indices:admin/refresh*` is not in any default group except `manage` (`indices:admin/*`) and must be added for `refreshIndex`; cluster-level `cluster_composite_ops` for `indices:data/write/bulk`. The exists check (HEAD index) also needs an `indices:admin/*` read permission; verify the exact action name against the permissions reference when writing the README role.
+* Versions: 2.x is in maintenance (latest 2.19.6, maintained until 4.0 GA); 3.x is current (3.9.0 released 2026-09-29; 3.10.0 scheduled Nov 2026). (https://opensearch.org/releases/)
+
+### 1.5 Testcontainers
+
+* Docker Hub has `opensearchproject/opensearch` tags 2.19.0 to 2.19.6 and 3.0.0 to 3.9.0 (https://hub.docker.com/v2/repositories/opensearchproject/opensearch/tags).
+* The official module `org.opensearch:opensearch-testcontainers` is at 4.1.0 (requires Testcontainers 2.0.0+ and JDK 17+); 3.0.x works with Testcontainers 1.20.0+ and JDK 21. `OpenSearchContainer` disables security by default and offers `withSecurityEnabled()` with HTTPS plus generated credentials. (https://github.com/opensearch-project/opensearch-testcontainers/blob/main/README.md and https://repo1.maven.org/maven2/org/opensearch/opensearch-testcontainers/maven-metadata.xml)
+
+### 1.6 Server-side behaviours relevant to the mapping and query design
+
+* `index.max_result_window` default 10000 caps `from + size` (https://docs.opensearch.org/latest/install-and-configure/configuring-opensearch/index-settings/).
+* Search analyzer resolution order: query `analyzer`, field `search_analyzer`, index `default_search`, field `analyzer`, `standard`. The docs' edge n-gram example sets `analyzer: edge_ngram_analyzer` with `search_analyzer: standard` so the query is not n-grammed. (https://docs.opensearch.org/latest/analyzers/search-analyzers/)
+* Edge n-gram filter: `max_gram` too low means long search terms do not match; the docs suggest a `truncate` filter as search analyzer to mitigate. (https://docs.opensearch.org/latest/analyzers/token-filters/edge-ngram/)
+* Dynamic mapping: strings become `text` + `keyword` subfield, or `date` when `date_detection` (default on) matches `strict_date_optional_time`; JSON numbers become `long`/`float`; `index.mapping.total_fields.limit` defaults to 1000; `flat_object` or `dynamic_templates` are the recommended defences. (https://docs.opensearch.org/latest/mappings/ and https://docs.opensearch.org/latest/mappings/mapping-explosion/)
+
+## 2. Compatibility matrix
+
+| Target | Works today? | What breaks | Change needed |
+| --- | --- | --- | --- |
+| OpenSearch 2.x self-managed, security disabled (the integration-test setup) | Yes | Nothing; this is the only tested configuration (image 2.11.0) | Test against 2.19.x as well (F10) |
+| OpenSearch 2.x self-managed, security plugin, basic auth over TLS | Partly | Demo or private-CA certificates fail the JVM default trust check; no truststore or insecure option exists (CONFIG exposes only `useSsl`). Only basic auth is wired; no client-cert, JWT or API-token path | Add TLS trust configuration (F1), document required roles (F13); optional auth-header or SSLContext hooks (F3) |
+| OpenSearch 3.x self-managed | Not supported by the pinned client | opensearch-java 2.10.0 is documented for 1.x to 2.x only; the server negotiates HTTP/2 with the Apache 5 transport | Bump to opensearch-java 3.10.0; one-line change at CLIENT line 427 (F2); add 3.x to the IT matrix (F10) |
+| AWS managed domain 2.x or 3.x, FGAC with internal-database master user, HTTP basic auth | Yes for 2.x (3.x after F2) | Needs `opensearchPort = 443`, `useSsl = true`, and a domain access policy that does not require signed requests. AWS certificates are publicly trusted so TLS works. Every API the library calls is on the managed-domain supported list. Payload limit 10 MiB on small instances with count-only batching (F8); no timeouts (F9) | F2 for 3.x domains, F8, F9, README guidance |
+| AWS managed domain, IAM master user or IAM principals in the access policy (SigV4) | No | Client only does basic auth; AWS rejects unsigned requests when IAM principals are in the policy; no `AwsSdk2Transport` path, region, or service-name configuration | Add an optional SigV4 transport mode: `AwsSdk2Transport` with service `es`, `Region`, default credential chain, `aws-crt-client`; declare the AWS SDK artifacts as optional dependencies (F3) |
+| Amazon OpenSearch Serverless (AOSS) | No | Transport (SigV4 `aoss`, `x-amz-content-sha256`); `_delete_by_query` used by FullReindexStep and ReconcileIndexStep; `_refresh` used by ReconcileIndexStep; custom `_id` only on Search collections; refresh lag ~10 s; client must be 3.x | Declare unsupported for 1.0. Supporting it later means: SigV4 transport (F3), replace delete-by-query with search + bulk delete, replace refresh with a bounded wait, and require Search collections (F4) |
+| Legacy Elasticsearch 7.10 domains or other ES-compatible endpoints | Untested, not documented | Client matrix lists only OpenSearch; renamed APIs across the 7.10 to 2.x gap | Declare out of scope |
+
+## 3. Findings
+
+Severity: High = breaks a target platform or returns wrong results; Medium = operational risk or avoidable failure; Low = hygiene.
+
+| ID | Severity | Finding | Evidence | Recommended change | 1.0 blocker |
+| --- | --- | --- | --- | --- | --- |
+| F1 | High | No TLS trust configuration. `useSsl` only flips the scheme; there is no truststore path, PEM CA, or insecure toggle, so self-managed clusters with demo or private-CA certificates fail the handshake. Hostname verification cannot be relaxed either | CLIENT lines 98 to 139; CONFIG lines 54, 420 to 444; official pattern in https://docs.opensearch.org/latest/clients/java/#connecting-to-opensearch | Add `trustStorePath`/`trustStorePassword` (or `caCertificatePem`) and an explicit `insecureSkipTlsVerify` flag; build an `SSLContext` and install it through `ClientTlsStrategyBuilder` + `PoolingAsyncClientConnectionManager` in the existing `setHttpClientConfigCallback`. Log a warning when verification is disabled | Yes |
+| F2 | High | opensearch-java pinned to 2.10.0 (2.x line, documented for OpenSearch 1.x to 2.x). AWS already offers 3.1 to 3.5 and 3.x is the current community line. Empirical check: bumping to 3.10.0 yields a single compile error at CLIENT line 427 (`refresh(true)` must become `refresh(Refresh.True)`); 233 unit tests pass afterwards. 3.x also drops the transitive `opensearch-rest-client` and matches the BOM's httpclient5 5.6 / httpcore5 5.4 | pom.xml line 24; COMPATIBILITY.md; CHANGELOG/UPGRADING links in section 1.1; scratch compile log | Bump `opensearch.version` to 3.10.0, fix line 427, re-run ITs; keep the `disableContentCompression()` workaround until proven unnecessary | Yes |
+| F3 | High (for AWS IAM) | Only HTTP basic auth is supported. No SigV4 (`AwsSdk2Transport`), no region or service name, no bearer-token or client-certificate hooks. An IAM master user or any IAM principal in the domain access policy makes the domain unreachable | CLIENT lines 109 to 137; CONFIG lines 52, 53; fgac.html; clients/java docs | Introduce an `authMode` (BASIC, AWS_SIGV4, NONE) plus `awsRegion`, `awsServiceName` (default `es`), optional credentials provider; build `AwsSdk2Transport` with `AwsCrtHttpClient`. Declare `software.amazon.awssdk:aws-crt-client` and `auth` as `<optional>true</optional>` rather than relying on the quicksight transitive. Optionally expose a `Consumer<HttpAsyncClientBuilder>` hook for JWT/API-token headers and mTLS | No (declare unsupported in README) |
+| F4 | High (for AOSS) | AOSS cannot be targeted: SigV4 `aoss` missing; `_delete_by_query` (CLIENT 392, 424) and `_refresh` (CLIENT 579) are not in the AOSS supported-operations table; custom `_id` requires a Search collection; `refresh=true` parameter unsupported | serverless-genref.html; serverless-overview.html; CLIENT lines 388 to 445, 575 to 585 | Declare AOSS unsupported for 1.0. For a later release: F3, implement table wipe and stale-document removal as search (PIT or `search_after`) + bulk delete, replace `refreshIndex` with a polling wait, document the Search-collection requirement and the `aoss:CreateIndex/DescribeIndex/WriteDocument/ReadDocument/UpdateIndex` data-access policy | No |
+| F5 | High | Edge n-gram analyzer (min 2, max 20) is set as the field `analyzer` with no `search_analyzer`, so the query string is also n-grammed. A search for "search" becomes se, sea, sear, searc, search and matches every document containing any token starting with "se", inflating hits and flattening relevance. Tokens longer than 20 chars only index their first 20 grams, and single-character terms never match | CLIENT lines 173 to 192 (`searchableText` property), 472 to 475 (must clause on `searchableText`); search-analyzers doc | Add a second analyzer (`standard` tokenizer + `lowercase`) and set it as `search_analyzer` on `searchableText`; consider `truncate` to 20 in the search analyzer or `preserve_original` so long terms still match; keep `min_gram 2` but document it. Requires index recreation (bump an index-version suffix or document a full reindex) | Yes |
+| F6 | High | `fieldValues` is a dynamic `object` that receives raw `QRecord` values (`Integer`, `BigDecimal`, `Instant`, `LocalDate`, `Boolean`, `String`). Dynamic mapping turns them into `long`/`float`/`date`/`boolean`; the first table to index a field name fixes its type for the whole index, so a `status` that is a string in one table and an integer in another causes `mapper_parsing_exception` on the second table. The boost `multi_match` clauses target `fieldValues.<field>` without `lenient: true`, so a non-numeric or non-date search term against a numeric or date `fieldValues` field makes the whole search request fail. Field count also grows with every configured field across all tables toward `index.mapping.total_fields.limit` (1000) | IndexingUtils lines 144 to 156; CLIENT lines 192, 525 to 565; mappings docs (date detection on by default; dynamic rules); mapping-explosion doc | Map `fieldValues` with a `dynamic_templates` rule that maps every `fieldValues.*` string to `text` (edge n-gram analyzer + standard search analyzer) and set `date_detection: false`; convert values to strings in `IndexingUtils` (`getValueString`) so types can never conflict; set `lenient(true)` on the boost `multi_match` queries; consider `flat_object` if per-field boosting is not needed | Yes |
+| F7 | Medium | `limit` and `offset` are unbounded; `from + size > 10000` returns a 400 (`index.max_result_window`) and large `size` values are accepted verbatim | QuickSearchAction lines 46 to 79; CLIENT 497 to 502; index-settings doc | Cap `limit` (for example 100) and reject or clamp `offset + limit > 10000` with a clear error; document that deep paging is not supported | No |
+| F8 | Medium | Bulk batching is count-based only (`bulkBatchSize` default 500) with no byte cap. On AWS small instance types the HTTP payload limit is 10 MiB; wide records with large text fields can exceed it and the whole batch fails with 413 | CLIENT lines 241 to 274; CONFIG line 58; AWS limits.html | Flush a batch when its serialized size approaches a configurable byte limit (default 5 MiB) in addition to the count; surface 413 clearly | No |
+| F9 | Medium | No connect, response or connection-request timeouts and no pool sizing; a hung node blocks the caller indefinitely; single host only, no failover list | CLIENT lines 105 to 139; no `RequestConfig`, `setConnectionManager`, or timeout call anywhere in `src/main/java` | Add `connectTimeoutMs`, `responseTimeoutMs`, `maxConnections` to CONFIG and apply via `setDefaultRequestConfig` and `PoolingAsyncClientConnectionManagerBuilder`; accept a list of hosts (`ApacheHttpClient5TransportBuilder.builder(HttpHost...)`) | Yes |
+| F10 | Medium | Integration tests pin `opensearchproject/opensearch:2.11.0` (Oct 2023). Nothing exercises 2.19.x or any 3.x, although AWS offers 2.19 and 3.1 to 3.5 and the community line is 3.9 | OpenSearchIntegrationTest line 76; IndexDriftIntegrationTest line 81; ConfigDrivenIndexingIntegrationTest line 76; releases page; Docker Hub tags | Parameterize the image via a system property (`-Dopensearch.image=...`) and run two CI jobs: `2.19.6` and `3.9.0`. Optionally adopt `org.opensearch:opensearch-testcontainers` 3.0.2 (Testcontainers 1.20+, JDK 21) now and 4.1.0 after moving to Testcontainers 2.x | Yes |
+| F11 | Low | ITs disable security via `plugins.security.disabled=true` while also passing `OPENSEARCH_INITIAL_ADMIN_PASSWORD=Admin123!`. Harmless today, but if security is ever enabled that password is unlikely to pass the zxcvbn "strong" check required since 2.12, and the documented image switch is `DISABLE_SECURITY_PLUGIN=true` | the three IT classes lines 79 to 85; demo-configuration doc | Use `DISABLE_SECURITY_PLUGIN=true` (and `DISABLE_INSTALL_DEMO_CONFIG=true`); drop the password, or when adding a secured IT, generate a strong one and connect over HTTPS with F1 in place | No |
+| F12 | Low | `opensearch-rest-client:2.12.0` and `httpcore5` 5.2.x come in transitively from the 2.10.0 client even though `RestClientTransport` is never used; version skew with the BOM's httpclient5 5.6.3 already required the compression workaround | dependency tree; CLIENT lines 124 to 130 | Resolved by F2; otherwise exclude `opensearch-rest-client` | No |
+| F13 | Low | No documented least-privilege role for the security plugin or AWS FGAC | none in repo | README: index pattern `<opensearchIndexName>`, groups `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*` and an `indices:admin/*` read for the exists check; cluster `cluster_composite_ops`. For AWS FGAC map the same to the basic-auth user; for AOSS list the five `aoss:*` permissions | No |
+| F14 | Low | Composite `_id` is `sourceTable + ":" + recordId`; `recordId` is mapped `keyword` (correct for term filters and mixed PK types). A table name containing ":" would be ambiguous, and `_id` is limited to 512 bytes | OpenSearchDocument lines 184 to 187; CLIENT line 188 | Validate table names against ":" in CONFIG; keep `keyword` | No |
+| F15 | Info | Against 3.x servers the Apache 5 transport negotiates HTTP/2 automatically; nothing in the code forces a version | USER_GUIDE.md HTTP/2 note | No change; mention in README; `HttpVersionPolicy.FORCE_HTTP_1` is available if a proxy misbehaves | No |
+
+## 4. Recommended "supported platforms" statement for the 1.0 README
+
+Supported (tested in CI):
+
+* Self-managed OpenSearch 2.19.x and 3.x (latest minor), reached over HTTP or HTTPS with a certificate the JVM trusts or a configured truststore, with the security plugin disabled or with HTTP basic authentication against the internal user database. Required role: `create_index`, `read`, `write`, `delete` on the quick-search index plus `indices:admin/refresh*`, and the cluster-level `cluster_composite_ops` group.
+* Amazon OpenSearch Service managed domains running OpenSearch 2.x or 3.x with fine-grained access control, an internal-database master user (or a mapped internal user), HTTP basic authentication, `opensearchPort = 443`, `useSsl = true`, and a domain access policy that does not require signed requests. Keep `bulkBatchSize` small enough for the instance type's HTTP payload limit (10 MiB on t3 and `*.large`).
+
+Not supported in 1.0:
+
+* IAM / SigV4 authentication to Amazon OpenSearch Service (IAM master user or IAM principals in the access policy).
+* Amazon OpenSearch Serverless (requires SigV4 with service `aoss` and does not offer `_delete_by_query` or `_refresh`, which the full-reindex and reconcile processes use).
+* Legacy Elasticsearch 7.10 domains and other Elasticsearch-compatible endpoints.
+* OpenSearch 1.x.
+
+Pre-conditions for the statement above: F1, F2, F5, F6, F9 and F10 must land first; the statement should name the exact image tags the CI matrix runs.
+
+## 5. Source list
+
+* https://github.com/opensearch-project/opensearch-java/blob/main/COMPATIBILITY.md
+* https://docs.opensearch.org/latest/clients/java/
+* https://github.com/opensearch-project/opensearch-java/blob/main/USER_GUIDE.md
+* https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md
+* https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/CHANGELOG.md
+* https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/UPGRADING.md
+* https://github.com/opensearch-project/opensearch-java/releases/tag/v3.0.0
+* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/maven-metadata.xml
+* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/2.10.0/opensearch-java-2.10.0.pom
+* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/3.9.0/opensearch-java-3.9.0.pom
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/infrastructure-security.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-overview.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-comparison.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html
+* https://docs.aws.amazon.com/general/latest/gr/opensearch-service.html
+* https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+* https://docs.opensearch.org/latest/security/configuration/demo-configuration/
+* https://docs.opensearch.org/latest/security/access-control/default-action-groups/
+* https://docs.opensearch.org/latest/security/authentication-backends/client-auth/
+* https://docs.opensearch.org/latest/security/authentication-backends/jwt/
+* https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-3.7.0.md
+* https://opensearch.org/releases/
+* https://hub.docker.com/v2/repositories/opensearchproject/opensearch/tags
+* https://github.com/opensearch-project/opensearch-testcontainers/blob/main/README.md
+* https://docs.opensearch.org/latest/install-and-configure/configuring-opensearch/index-settings/
+* https://docs.opensearch.org/latest/analyzers/search-analyzers/
+* https://docs.opensearch.org/latest/analyzers/token-filters/edge-ngram/
+* https://docs.opensearch.org/latest/mappings/
+* https://docs.opensearch.org/latest/mappings/mapping-explosion/
+* https://forum.opensearch.org/t/opensearch-api-compatibility-with-elastic-search/17647

docs/audits/2026-10-04/03-authentication-security.md

@@ -0,0 +1,156 @@
+# Audit 03: Authentication, secrets handling, and connection security
+
+Repo: `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` at `7cd94ae` (branch `feature/GH-6-repin-and-drift-fix`), read-only.
+QQQ reference: `/Users/james.maes/Git.Local/QRunIO/qqq` at tag `v4.1.0-RC.1`.
+Resolved runtime versions (from `mvn -B dependency:tree`): opensearch-java 2.10.0, opensearch-rest-client 2.12.0 (runtime), httpclient5 5.6.3, httpcore5 5.4.3, jackson 2.21.5, log4j 2.25.5, AWS SDK v2 2.41.10 (via qqq-backend-core), AWS SDK v1 1.12.797 (via qqq-backend-core).
+
+Paths below are relative to `src/main/java/com/kingsrook/qbits/quicksearch/` unless stated otherwise.
+
+## Findings
+
+| ID | Severity | Finding | Evidence | Recommended change | 1.0 blocker |
+|----|----------|---------|----------|--------------------|-------------|
+| A1 | Critical | `QuickSearchAction` returns `tableName`, `recordId`, `recordLabel` and a `highlightSnippet` (fragments of `searchableText`, which is the concatenation of every configured field value) for every indexed table, with no check of the caller's session, table permission, or record security locks. The caller-supplied `tableName` filter is also unchecked. Any authenticated (or anonymous, if the instance allows it) caller can enumerate labels and field values across all indexed tables. | `actions/QuickSearchAction.java:81-121` (reads client and all `discoveredTables` from the static context, maps every hit straight to output); `opensearch/QuickSearchOpenSearchClient.java:466-510` (only filter is the optional caller-supplied `sourceTable` term); `processes/IndexingUtils.java:83,136-165` (searchableText and fieldValues built from all configured fields) | See section "A1 fix" below: make `QuickSearchInput` extend `AbstractActionInput`, compute the set of tables the session may read with `PermissionsHelper.hasTablePermission(input, table, READ)`, reject or ignore a requested `tableName` outside that set, pass the allowed set into the client as a `terms` filter on `sourceTable`, then post-filter hits per table through `QueryAction` by primary key so record security locks apply. Add negative tests. | Yes |
+| A2 | High | HTTP basic credentials are sent in cleartext when `useSsl` is false (the default), and nothing in `validate()` objects to username/password combined with plain HTTP. The README example and the qqq-all `DemoQBits` default `OPENSEARCH_SSL` to false. | `QuickSearchQBitConfig.java:54` (`useSsl = false`), `:103-112` (only checks that username and password are set together); `opensearch/QuickSearchOpenSearchClient.java:98-99,113-119`; `README.md:303`; qqq-all `DemoQBits.java:156-158` | In `validate()`, add an error when `authMode` is BASIC (or username/password set) and TLS is off, unless an explicit `allowPlaintextCredentials` (dev only) is true. For 1.0 change the default scheme to HTTPS (see config sketch) so that opting into plaintext is deliberate. | Yes |
+| A3 | High | The password lives as a plain `String` on a `Serializable` `QBitConfig`, and the downstream host attaches that same object to `QBitMetaData` which is stored in `QInstance.getQBits()`. Anything that serializes the `QInstance` (for example `QInstanceAdapter.qInstanceToJson`, used by dev tooling and tests) or any code that walks `getQBits()` can read the password through the public getter. No `@JsonIgnore`, no redaction, no `toString()` override (the default `Object.toString()` happens not to leak it, but only by accident). | `QuickSearchQBitConfig.java:52-53,389-392`; qqq `QBitMetaData.java` (field `QBitConfig config`, `getConfig()`); qqq `QInstance.java:114,1800-1836`; qqq `adapters/QInstanceAdapter.java:46,57`; qqq-all `DemoQBits.java:157,162-163`; qqq-all `FullQBitsTest.java:78` (casts the config back out of `getQBits()`) | Do not keep the raw secret on the metadata object. Options, cheapest first: (1) mark `getOpensearchPassword()` (and any token/secret getter) `@JsonIgnore` and add a `toString()` that redacts; (2) accept `${env.X}` / `${prop.X}` references and resolve them with `QMetaDataVariableInterpreter` only inside the client constructor, so the config object stores the reference string, not the value (this is exactly how `RDBMSBackendMetaData.enrich()`, `MongoDBBackendMetaData.enrich()` and `S3BackendMetaData.enrich()` handle `password`/`secretKey`); (3) longer term, resolve credentials through a `QCodeReference` provider so no secret ever sits on metadata. Do (1) and (2) for 1.0. | Yes |
+| A4 | Medium | Only two auth modes exist: none and HTTP basic. There is no AWS SigV4 support, which blocks Amazon OpenSearch Service domains using IAM (and OpenSearch Serverless entirely), and there is no way to inject a bearer token, API-key header, or client certificate without forking the client. | `opensearch/QuickSearchOpenSearchClient.java:94-147` (hard-wires `ApacheHttpClient5TransportBuilder` + `BasicCredentialsProvider`) | Add an `authMode` enum (NONE, BASIC, AWS_SIGV4) and a transport-customizer SPI for the rest. `AwsSdk2Transport` ships in opensearch-java 2.10.0 and the AWS SDK v2 core modules are already on the compile classpath via qqq-backend-core, so SigV4 is mostly wiring (details in section 1). | Yes (SigV4); No (bearer, API key, mTLS via SPI) |
+| A5 | Medium | TLS trust is not configurable. The transport uses `SSLContext.getDefault()` (JVM trust store) and the httpclient5 default hostname verifier, so certificates are verified correctly by default, but a private CA, a custom trust store, or client-certificate material can only be supplied through JVM system properties. There is no explicit, loud "insecure" switch for local development, which pushes developers toward global `-Djavax.net.ssl.trustStore` hacks or disabling the security plugin. | opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` bytecode: `ClientTlsStrategyBuilder.create().setSslContext(SSLContext.getDefault())`; `opensearch/QuickSearchOpenSearchClient.java:105-137` never sets a TLS strategy | Add `tls.caCertificatePath` (PEM) or `tls.trustStorePath` + `tls.trustStorePassword` (reference-interpreted), `tls.hostnameVerification` (default true), and `tls.insecureSkipVerify` (default false; refuse unless host is loopback or `allowInsecureInProduction` is set; log a WARN at startup). Build an `SSLContext` via `SSLContextBuilder.loadTrustMaterial(...)` and pass it through `ClientTlsStrategyBuilder` on a `PoolingAsyncClientConnectionManager`. | No (strongly recommended) |
+| A6 | Medium | The three produced processes (`quickSearchBasepullIndex`, `quickSearchFullReindex`, `quickSearchReconcileIndex`), the two operational tables, and the `quickSearchAdmin` app carry no `QPermissionRules`, so they inherit `QInstance.defaultPermissionRules` (QQQ default is `NOT_PROTECTED`). In a host that has not tightened the instance default, any user can trigger a full reindex (which wipes the index first, a search blackout) or read/edit the run tables. | `QuickSearchQBitProducer.java:457-549` (tables and processes built without `withPermissionRules`), `:246-260` (app); `grep PermissionRules src/main` returns nothing; qqq `QInstance.java:140` (`defaultPermissionRules = QPermissionRules.defaultInstance()`); qqq `QJavalinProcessHandler.java:404,418` (middleware enforces whatever rules the process declares) | Set explicit rules in the producer: processes and app `HAS_ACCESS_PERMISSION`, tables `READ_WRITE_PERMISSIONS`, with a stable `permissionBaseName` (for example `quickSearchAdmin`) so one grant covers the admin surface. Make it overridable via an `adminPermissionRules` config field; never default to `NOT_PROTECTED`. | Yes |
+| A7 | Medium | Transport timeouts are the opensearch-java defaults: connect 1 s, response 30 s, max 10 connections per route / 30 total, automatic httpclient retries disabled. None are configurable. 1 s connect is tight for cross-region or cold AWS endpoints; 30 s response is marginal for `deleteByQuery(... refresh=true)` on large tables (`deleteDocumentsIndexedBefore`, `deleteDocumentsForTable`) and for 500-document bulk requests on a slow cluster. There is no explicit user-agent. | opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` constants `DEFAULT_CONNECT_TIMEOUT_MILLIS=1000`, `DEFAULT_RESPONSE_TIMEOUT_MILLIS=30000`, `DEFAULT_MAX_CONN_PER_ROUTE=10`, `DEFAULT_MAX_CONN_TOTAL=30`, and `HttpAsyncClientBuilder.disableAutomaticRetries()` in its `createHttpClient`; `opensearch/QuickSearchOpenSearchClient.java:105-137` sets no `RequestConfigCallback` | Add `connectTimeoutMillis` (default 5000), `responseTimeoutMillis` (default 60000), `maxConnections` (default 30) to config; apply via `setRequestConfigCallback` and `PoolingAsyncClientConnectionManagerBuilder`. Set a default header `User-Agent: qbit-quick-search/<version>` so cluster-side logs can attribute traffic. Keep retries off at the HTTP layer; the indexing steps already record per-item failures. | No |
+| A8 | Medium | The OpenSearch client is never closed on host shutdown. `QuickSearchOpenSearchClient` is `Closeable` and `SynchronousIndexEventPublisher.close()` calls `client.close()`, but nothing invokes `publisher.close()` or `client.close()` outside tests; the static `QuickSearchQBitContext` has no lifecycle hook. The pooled connections and the httpclient5 I/O reactor thread leak until JVM exit, and a hot-reload of the `QInstance` (which re-runs `produce`) creates a second transport while the first stays open. | `QuickSearchQBitContext.java:34-37,152-158`; `QuickSearchQBitProducer.java:159-203`; `publisher/SynchronousIndexEventPublisher.java:179-181`; `opensearch/QuickSearchOpenSearchClient.java:595-612` | In `produce`, close any previously stored client/publisher before replacing them, and register a `Runtime.getRuntime().addShutdownHook` (or the QQQ instance shutdown hook if the host exposes one) that calls `QuickSearchQBitContext.shutdown()`. | No |
+| A9 | Low | Search terms are logged at DEBUG (`"term", normalizedTerm`). Search terms are frequently PII (names, emails, order numbers). Not a secret leak, but worth knowing before enabling DEBUG in production. | `actions/QuickSearchAction.java:89` | Log term length or a hash at DEBUG; log the raw term only at TRACE. | No |
+| A10 | Low | `QuickSearchQBitConfig implements QBitConfig` (which `extends Serializable`) but holds a non-`Serializable` `IndexEventPublisher` and a `List<Class<?>>`. Java serialization of the config (which QQQ permits by contract) would throw `NotSerializableException` when a custom publisher is set. Not a security issue, but it is the same contract that A3 relies on. | `QuickSearchQBitConfig.java:45,60,63`; qqq `QBitConfig.java` (`public interface QBitConfig extends Serializable`) | Mark `indexEventPublisher` `transient` or replace it with a `QCodeReference`. | No |
+| A11 | Low | `limit` and `offset` are unbounded. A caller can request `limit=100000`; OpenSearch rejects `from+size > index.max_result_window` (10000) with an error that surfaces as a `QException`, and smaller but large pages pull `fieldValues` for every hit into heap. Combined with A1 this makes bulk enumeration trivial. | `actions/QuickSearchAction.java:78-79` | Clamp `limit` to a `maxLimit` (default 100) and `offset + limit` to 10000, and document it. | No |
+| A12 | Info | No committed production credentials. Test fixtures use placeholders (`"user"`, `"secret"`, `"pass"`, `"admin"`) and the Testcontainers `OPENSEARCH_INITIAL_ADMIN_PASSWORD=Admin123!` with `plugins.security.disabled=true`, so the password is never used. No `.env`, properties, or YAML with secrets in the tree; `.gitignore` excludes `.env` and `.env.local`. The README example hard-codes `"secretPassword"` as a literal, which teaches the wrong pattern. | `src/test/.../QuickSearchQBitConfigTest.java:229-278,509-555`; `src/test/.../QuickSearchOpenSearchClientTest.java:89-90`; `src/test/.../OpenSearchIntegrationTest.java:75-81` (and the two other integration tests); `README.md:115-125` | Change the README example to `.withOpensearchPassword("${env.OPENSEARCH_PASSWORD}")` once A3 lands. | No |
+| A13 | Info | Logging and exception messages do not include the host, port, username or password. `validate()` emits only field names, `backendName` and table names; the client's `QException`s include `indexName`, document ids, the search term, and the underlying `e.getMessage()` (which for connection failures from httpclient5 can include `host:port`, acceptable). | `QuickSearchQBitConfig.java:73-159`; `opensearch/QuickSearchOpenSearchClient.java:145,199,222,351,376,401,443,508,583`; `QuickSearchQBitProducer.java:121,166,180` | Keep as is; add the `toString()` redaction from A3 so a future `LOG.info("config", config)` cannot regress this. | No |
+| A14 | Info | Supply chain: no concrete CVE affects the resolved runtime set. httpclient5 5.6.3 is past the fix for CVE-2025-27820 (PSL/hostname-verification bypass in 5.4.0 to 5.4.2, fixed 5.4.3; see https://nvd.nist.gov/vuln/detail/CVE-2025-27820 and https://github.com/advisories/GHSA-73m2-qfq3-56cx). jackson-core 2.21.5 is past CVE-2025-52999 (nesting-depth DoS, fixed 2.15.0; https://nvd.nist.gov/vuln/detail/CVE-2025-52999). log4j 2.25.5 has no open advisories. No published advisory was found for opensearch-java 2.10.0 or opensearch-rest-client 2.12.0 (searched NVD and GitHub advisories on 2026-10-04). Note, not a CVE: qqq-backend-core still drags in AWS SDK for Java v1 1.12.797, which reached end of support on 2025-12-31 (https://aws.amazon.com/blogs/developer/announcing-end-of-support-for-aws-sdk-for-java-v1-x-on-december-31-2025/); that is a qqq concern, not this repo's, and this QBit should use SDK v2 only. | `pom.xml:22` (`opensearch.version 2.10.0`), `:74-83` (httpclient5 and jackson unpinned, versions from the qqq BOM); `mvn -B dependency:tree` output | Nothing to pin. Consider moving opensearch-java to the 2.x line that matches the target cluster (the client's `opensearchVersion` is 2.12.0) when the next qqq BOM bump happens; keep httpclient5 and jackson BOM-managed as ADR-0007 already requires. | No |
+
+## 1. Auth modes: today versus 1.0
+
+| Mode | Supported today | What it needs | Notes |
+|------|-----------------|---------------|-------|
+| None (local dev, security plugin disabled) | Yes (omit username/password) | Nothing | Should stay the default only for `localhost`; see A2. |
+| HTTP basic (security plugin internal user, or Amazon OpenSearch fine-grained access control with an internal master user) | Yes | Nothing new; fix A2 (require TLS) and A3 (`${env.}` references) | Current implementation uses `BasicCredentialsProvider` scoped to the host, which is correct. Consider pre-emptive auth (`BasicScheme` in a `HttpClientContext`) to save one 401 round trip per connection; optional. |
+| AWS IAM SigV4, service `es` (Amazon OpenSearch Service domain with IAM) | No | `org.opensearch.client.transport.aws.AwsSdk2Transport` (already inside opensearch-java 2.10.0); `software.amazon.awssdk:sdk-core`, `auth`, `regions`, `http-client-spi` (already compile-scope via qqq-backend-core at 2.41.10); one `SdkHttpClient` implementation declared explicitly, `software.amazon.awssdk:apache-client` (currently only transitive runtime scope) or `url-connection-client`; `software.amazon.awssdk:sts` if assume-role is wanted (not currently on the classpath) | `new AwsSdk2Transport(httpClient, endpointHost, "es", Region.of(region), AwsSdk2TransportOptions.builder().setCredentials(provider).build())`. Default provider is `DefaultCredentialsProvider` (env, profile, container, instance role). Assume-role: `StsAssumeRoleCredentialsProvider`. Reference: https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md and https://docs.opensearch.org/latest/clients/java/ |
+| AWS IAM SigV4, service `aoss` (OpenSearch Serverless) | No | Same as above with signing service name `"aoss"` | Serverless rejects some operations this QBit uses; verify `refreshIndex()` and `deleteByQuery(... refresh=true)` against the supported-operations list before claiming support (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html). Reference for `aoss`: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html |
+| Bearer / JWT token | No | `ApacheHttpClient5TransportBuilder.setDefaultHeaders(new BasicHeader("Authorization", "Bearer " + token))` for a static token; a `Supplier<String>` plus an `HttpRequestInterceptor` for rotating tokens | The security plugin supports JWT as an HTTP authenticator (https://docs.opensearch.org/latest/security/authentication-backends/authc-index/). Static tokens fit a header field; rotation needs the SPI below. |
+| API key header | No | Same mechanism as bearer, custom header name and value | Not a native OpenSearch security-plugin concept (OpenSearch has no Elasticsearch-style API keys); treat as "arbitrary static header", covered by the SPI. |
+| Client certificate mTLS | No | `SSLContextBuilder.loadKeyMaterial(keyStore, keyPassword)` plus `loadTrustMaterial`, passed through `ClientTlsStrategyBuilder` to a `PoolingAsyncClientConnectionManager` set on the `HttpAsyncClientBuilder` | Security plugin side: `plugins.security.ssl.http.clientauth_mode: OPTIONAL` or `REQUIRE` (https://docs.opensearch.org/latest/security/authentication-backends/client-auth/). Fits naturally under the `tls` block (`keyStorePath`, `keyStorePassword`). |
+
+Recommended minimum for 1.0: `authMode` enum with three values, NONE, BASIC, AWS_SIGV4 (with `awsRegion`, `awsServiceName` defaulting to `es`, optional `awsAssumeRoleArn`), plus one escape hatch, `transportCustomizer` as a `QCodeReference` to an interface such as
+
+    public interface OpenSearchTransportCustomizer
+    {
+       default ApacheHttpClient5TransportBuilder customizeTransport(ApacheHttpClient5TransportBuilder b) { return b; }
+       default HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder b) { return b; }
+    }
+
+That covers local, self-managed with the security plugin, and both AWS flavours natively, and lets bearer, API key and mTLS be added by hosts without a fork. Do not add BEARER, API_KEY or MTLS enum values in 1.0; they would each need a token-refresh or key-store story that the SPI defers.
+
+The AWS path is a different `OpenSearchTransport` implementation, so `QuickSearchOpenSearchClient` should build an `OpenSearchTransport` from a small factory keyed on `authMode` and hold the interface type rather than `ApacheHttpClient5Transport` (`opensearch/QuickSearchOpenSearchClient.java:77`).
+
+Least privilege for whatever identity is used: the QBit needs, on its one index, `indices:admin/create`, `indices:admin/exists` (or `indices:admin/get`), `indices:admin/refresh`, `indices:data/write/index`, `indices:data/write/bulk*`, `indices:data/write/delete`, `indices:data/write/delete/byquery`, and `indices:data/read/search`. It needs no cluster-level privileges. Document a role template; the index holds copies of field values from every indexed table, so the index itself should be treated as sensitive data.
+
+## 2. TLS
+
+- Can trust be configured: no. The transport is built with `SSLContext.getDefault()` and the httpclient5 default hostname verifier (confirmed from the opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` bytecode). Certificates are verified against the JVM trust store by default, hostname verification is on, and there is no code path to weaken or extend either.
+- Custom CA or trust store: only through `-Djavax.net.ssl.trustStore` system properties on the host JVM, which is global and awkward in containers. Add the `tls` block from the sketch.
+- Insecure flag for dev: none exists. This is good for safety but in practice developers running OpenSearch with its default self-signed demo certs will reach for a global workaround. Provide `tls.insecureSkipVerify` with guard rails (refuse for non-loopback hosts unless an explicit second flag is set; WARN at startup) rather than leaving it to ad hoc JVM flags.
+- `useSsl` default false: acceptable only for a local, no-auth cluster. It is not acceptable combined with credentials (A2). For 1.0 prefer a single `url` (or `scheme`) field defaulting to `https`, and make `validate()` reject BASIC or AWS_SIGV4 over `http` unless `allowPlaintextCredentials` is set. AWS endpoints are HTTPS only, so AWS_SIGV4 should force `https`.
+
+## 3. Secrets handling
+
+- Storage: `opensearchPassword` is a plain `String` on a `Serializable` config (`QuickSearchQBitConfig.java:53`). `QuickSearchQBitProducer.produce` stores the same object in the static `QuickSearchQBitContext` (`QuickSearchQBitProducer.java:200`), and the downstream qqq-all host attaches it to `QBitMetaData` and `QInstance.addQBit` (`DemoQBits.java:162-163`). The password therefore lives in three places for the JVM lifetime.
+- Exposure through QQQ metadata endpoints: not today. `MetaDataAction` builds `MetaDataOutput` from `QFrontendTableMetaData`, `QFrontendProcessMetaData`, `QFrontendAppMetaData`, widgets, `environmentValues`, and supplemental metadata; it never touches `QInstance.getQBits()` or backend metadata, so `/metaData` does not carry QBit configs. Two caveats: `QInstanceAdapter.qInstanceToJson` (qqq `adapters/QInstanceAdapter.java:46,57`) serializes the whole `QInstance` including `qBits`, and Jackson will emit `opensearchPassword` through its public getter; and `MetaDataOutput.environmentValues` is sent to the frontend and is populated from every `QQQ_ENV_*` environment variable (qqq `QInstance.java:190-216`), so an operator must never name an OpenSearch secret `QQQ_ENV_...`. Document that.
+- Logging: no `LOG` call prints host, port, username or password (full list in A13). `validate()` and `produce()` print only field names, `backendName`, class names and table names. There is no `toString()` on the config, so an accidental `LOG.info("...", config)` would print `QuickSearchQBitConfig@hash` today; add an explicit redacting `toString()` so that stays true if someone adds one.
+- Recommendation (ordered): (1) interpret `opensearchUsername`, `opensearchPassword` and any future token or store-password fields with `QMetaDataVariableInterpreter.interpret()` at client-construction time only, so hosts write `"${env.OPENSEARCH_PASSWORD}"` and the metadata object never holds the value; this is the established QQQ pattern (`RDBMSBackendMetaData.enrich()`, `MongoDBBackendMetaData.enrich()`, `S3BackendMetaData.enrich()`); `${prop.X}` and `${literal.X}` come for free, and `??` null-coalescing lets `"${env.X}??${prop.X}"` work. (2) `@JsonIgnore` on the secret getters and a redacting `toString()`. (3) For 1.0 documentation, state that the raw password form is deprecated and will be rejected (or at least warned about) in a later release.
+
+## 4. QQQ-side authorization
+
+### A1 fix (search)
+
+Exact leak: `QuickSearchAction.execute` reads every discovered table from `QuickSearchQBitContext` and returns `tableName`, `recordId`, `recordLabel` and `highlightSnippet` for every hit (`actions/QuickSearchAction.java:81-121`). The snippet is cut from `searchableText`, which `IndexingUtils.buildSearchableText` builds from all configured fields (`processes/IndexingUtils.java:83,136`). So a user with read permission on `order` but not `customer` who searches "smith" receives customer labels and fragments of customer field values (emails, phone numbers, whatever was configured). The caller-supplied `tableName` filter is passed straight into the query (`QuickSearchOpenSearchClient.java:481-487`) so it can also be used to target one table deliberately.
+
+Minimal fix, in order:
+
+1. Make `QuickSearchInput extends AbstractActionInput` (`actions/QuickSearchInput.java:25` is a plain class today). `PermissionsHelper.hasTablePermission(AbstractActionInput, String, TablePermissionSubType)` needs it, and it gives the action the session via `QContext`.
+2. Compute `allowedTables = discoveredTables.filter(t -> PermissionsHelper.hasTablePermission(input, t.getTableName(), TablePermissionSubType.READ))`. QQQ 4.1's own `RecordSearchAction` does exactly this check (qqq `actions/tables/RecordSearchAction.java:110`); mirror it, including the `Capability.TABLE_QUERY` check.
+3. If `input.tableName` is set and not in `allowedTables`, return the empty output (do not throw; do not reveal that the table exists). If `allowedTables` is empty, return empty without calling OpenSearch.
+4. Change `QuickSearchOpenSearchClient.search` to take `Collection<String> allowedTables` and always add `filter: { terms: { sourceTable: allowedTables } }` to the bool query, in addition to the optional single-table term. Only build boost `should` clauses for allowed tables.
+5. Optional but recommended (initiative D3 already decided yes): group hits by `sourceTable`, run one `QueryAction` per table with `primaryKeyField IN (ids)` and `shouldOmitHiddenFields`, keep only hits whose id came back. `QueryAction` itself does not check table permission (it only calls `ActionHelper.validateSession`, qqq `QueryAction.java:127`), so step 2 is still required; what the re-read adds is record security locks (`RecordSecurityLock` handling in `JoinsContext`), which is why `RecordSearchAction` goes through `QueryAction` too (qqq `RecordSearchAction.java:66,177`). Over-fetch (for example `limit * 2`, capped) and document that `totalHits` is an upper bound.
+6. Tests: a session without READ on table B searching a term that matches rows in A and B gets only A hits; a session with a record security lock sees only its own rows after step 5; a requested `tableName` the session cannot read yields empty.
+
+### Processes, tables and app
+
+Today none of the produced metadata declares permission rules (A6). For 1.0 the producer should set, by default and overridable through config:
+
+- `quickSearchBasepullIndex`, `quickSearchFullReindex`, `quickSearchReconcileIndex`: `QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName("quickSearchAdmin")`. Full reindex deletes the table's documents before rebuilding (search blackout), so it must not be runnable by arbitrary users.
+- `quickSearchIndex`, `quickSearchIndexRun`: `READ_WRITE_PERMISSIONS` (operators may need to flip `enabled` or edit intervals; everyone else should not see run error messages, which can include backend exception text).
+- `quickSearchAdmin` app: `HAS_ACCESS_PERMISSION` with the same base name so a single `quickSearchAdmin.hasAccess` grant lights up the whole admin surface.
+- Scheduled runs: when D4 lands (`schedulerName` required), basepull runs under the scheduler's system session, not a user's, so the process permission only gates manual triggering. Say so in the README.
+
+The customizers run inside the acting user's transaction and session; they index what that user inserted or updated and delete what that user deleted, so they need no extra checks. `QuickSearchPostUpdateCustomizer` re-reads via `QueryAction` in the same session (`customizers/QuickSearchPostUpdateCustomizer.java:154`), which is correct.
+
+## 5. Transport hygiene
+
+| Aspect | Current value | Source | Assessment |
+|--------|---------------|--------|------------|
+| Connect timeout | 1000 ms | opensearch-java builder default | Too tight for AWS or cross-AZ; make configurable, default 5 s |
+| Response (socket) timeout | 30 000 ms | builder default | Bounded, good; configurable, default 60 s for bulk and delete-by-query |
+| Connection-request (pool wait) timeout | httpclient5 `RequestConfig.DEFAULT` (3 min) | not set | Bounded; fine |
+| Max connections | 10 per route / 30 total | builder default | Fine for one host; expose `maxConnections` |
+| HTTP-level retries | disabled (`disableAutomaticRetries()`) | builder | Good; indexing steps handle failures per item |
+| Node failover | single `HttpHost`, dead-host denylist with backoff | `QuickSearchOpenSearchClient.java:99`; transport `DeadHostState` | OK for one endpoint; allow a list of hosts for self-managed clusters |
+| Compression | client-side decompression disabled, transport handles it | `QuickSearchOpenSearchClient.java:122-131` | Correct workaround for httpclient5 5.6 |
+| User-Agent | opensearch-java default | not set | Add `qbit-quick-search/<version>` |
+| Close on shutdown | never called in production | A8 | Fix |
+| Keep-alive / idle eviction | httpclient5 defaults | not set | Acceptable |
+
+## 6. Supply chain
+
+See A14. Resolved compile/runtime set has no concrete published CVE at the audit date. The one cited fix the stack already has is CVE-2025-27820 in httpclient5 5.4.0 to 5.4.2 (resolved 5.6.3 is clear). Keep httpclient5 and jackson BOM-managed (ADR-0007). `h2 2.2.220`, Testcontainers 1.21.4, JUnit, AssertJ and Mockito are test scope only.
+
+## Recommended 1.0 auth configuration
+
+Field names on `QuickSearchQBitConfig` (fluent `withX` setters as today). All `String` values that can hold secrets or paths are run through `QMetaDataVariableInterpreter.interpret()` inside the client factory, never at `validate()` or `produce()` time, so the config object stores references.
+
+    // Connection
+    opensearchUrl            String   "https://search.example.com:443"  (preferred; replaces host/port/useSsl)
+    opensearchHost/Port      String/Integer  (kept for compatibility; deprecated)
+    useSsl                   Boolean  (kept; ignored when opensearchUrl is set)
+    opensearchHosts          List<String>  optional, same scheme, for self-managed multi-node clusters
+    opensearchIndexName      String   (unchanged)
+
+    // Authentication
+    authMode                 enum OpenSearchAuthMode { NONE, BASIC, AWS_SIGV4 }   default NONE
+    opensearchUsername       String   BASIC only; "${env.OPENSEARCH_USER}"
+    opensearchPassword       String   BASIC only; "${env.OPENSEARCH_PASSWORD}"; @JsonIgnore getter; redacted in toString()
+    awsRegion                String   AWS_SIGV4; "${env.AWS_REGION}" if omitted
+    awsServiceName           String   AWS_SIGV4; "es" (default) or "aoss"
+    awsAssumeRoleArn         String   AWS_SIGV4, optional; uses StsAssumeRoleCredentialsProvider when set
+    allowPlaintextCredentials Boolean default false; validate() errors on BASIC over http unless true
+
+    // TLS
+    tls.caCertificatePath    String   PEM bundle to trust in addition to or instead of the JVM store
+    tls.trustStorePath       String   JKS/PKCS12 alternative to caCertificatePath
+    tls.trustStorePassword   String   reference-interpreted
+    tls.keyStorePath         String   client certificate for mTLS (optional)
+    tls.keyStorePassword     String   reference-interpreted
+    tls.hostnameVerification Boolean  default true
+    tls.insecureSkipVerify   Boolean  default false; validate() rejects for non-loopback hosts unless allowInsecureInProduction is true; WARN at startup
+
+    // Transport
+    connectTimeoutMillis     Integer  default 5000
+    responseTimeoutMillis    Integer  default 60000
+    maxConnections           Integer  default 30
+    transportCustomizer      QCodeReference<OpenSearchTransportCustomizer>  escape hatch for bearer, API key, custom interceptors
+
+    // Authorization inside QQQ
+    searchRequiresTableReadPermission  Boolean default true (no switch to turn it off in 1.0; listed so the behaviour is documented)
+    applyRecordSecurityLocks           Boolean default true; when true, hits are re-read through QueryAction by primary key
+    adminPermissionRules     QPermissionRules  default HAS_ACCESS_PERMISSION, base name "quickSearchAdmin", applied to the app and the three processes
+    tablePermissionRules     QPermissionRules  default READ_WRITE_PERMISSIONS, applied to quickSearchIndex and quickSearchIndexRun
+
+`validate()` additions: `authMode` consistency (BASIC needs both username and password; AWS_SIGV4 needs a resolvable region and forbids username/password; NONE with username/password set is an error, not a silent fallback), TLS consistency (trust store needs password or must be PKCS12 without one; `insecureSkipVerify` rules above), and the plaintext-credentials rule. Validation messages must never echo values, only field names, which is already the convention in the file.
+
+Deprecation path: keep `withOpensearchHost/Port/UseSsl/Username/Password` working for 0.x hosts; infer `authMode = BASIC` when username is set and `authMode` is null, and log a one-line WARN recommending `${env.}` references when a password value does not start with `${`.

docs/audits/2026-10-04/04-feature-completeness.md

@@ -0,0 +1,213 @@
+# Audit 04: Feature completeness for Quick Search 1.0
+
+Scope: `qbit-quick-search` at `feature/GH-6-repin-and-drift-fix` (content-identical to `develop` 000e5bb), 29 main classes, 29 test classes (233 unit tests, 12 integration tests). Compared against the draft PRD `docs/initiatives/quick-search-1.0.md`, qqq-all `DemoQBits.java`, Voyage, and QQQ `v4.1.0-RC.1` `RecordSearchAction`.
+
+Paths below are relative to `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1/src/main/java/com/kingsrook/qbits/quicksearch/` unless stated. Core paths are relative to `/Users/james.maes/Git.Local/QRunIO/qqq` at tag `v4.1.0-RC.1`.
+
+Verdict in one line: the QBit has a working happy path (index, bulk, search, reconcile) but is missing most of what makes a search feature trustworthy in a multi-table, multi-user, long-running host: no scheduling, no permissions, no timeouts, no mapping/config drift handling, no admin UI affordances, a single global static context, and a search query shape that was never tuned. The PRD covers roughly half of the gaps found here.
+
+## 1. Area-by-area table
+
+| # | Area | Status | Evidence | What 1.0 needs |
+|---|---|---|---|---|
+| 1a | Real-time insert/update/delete customizers | Partial | `customizers/QuickSearchPostInsertCustomizer.java:66-105`, `...PostUpdateCustomizer.java:81-155` (re-reads full record in the update's transaction), `...PostDeleteCustomizer.java:67-118`. All three swallow every exception at WARN (`:99-102`, `:117-120`, `:112-115`) and return records unchanged. Registration throws when the slot is occupied (`QuickSearchQBitProducer.java:571-579`; core `QTableMetaData.java:651-653` throws `IllegalArgumentException`). | `MultiCustomizer` composition (PRD E1). Decide and document the failure contract (swallow vs. fail write), plus a failed-event record so swallowed failures are not lost (see 4). Honour `quickSearchIndex.enabled` (see 3). |
+| 1b | Bulk inserts with many records | Partial | Insert customizer builds one `IndexEvent` per record and the publisher bulk-indexes in `bulkBatchSize` chunks (`publisher/SynchronousIndexEventPublisher.java:93-127`, `opensearch/QuickSearchOpenSearchClient.java:252-272`). Whole batch runs inside the host's insert call; no size-based (bytes) batching, no async. A 50k-row bulk load blocks the insert for the full OpenSearch round trips. | Document that bulk loads should disable real-time indexing or rely on basepull; add byte-size cap per bulk request (OpenSearch `http.max_content_length` default 100 MB). |
+| 1c | Null/blank searchable fields | Complete | `processes/IndexingUtils.java:96-99` skips null/blank in text; `:150-156` omits null from `fieldValues`; `:138-141` returns null doc when PK null. Unit tested (`IndexingUtilsTest`). | Nothing. |
+| 1d | Very large text fields | Missing | No truncation or length cap anywhere (`IndexingUtils.buildSearchableText`); `recordLabel`/`searchableText` mapped as plain `text` (`QuickSearchOpenSearchClient.java:189-190`). A CLOB field is sent whole; edge-ngram 2..20 on a 1 MB field explodes token count. Highlight returns default fragments only. | Per-field `maxLength` in `SearchableFieldConfig`/`@QuickSearchField` with a sane default (e.g. 10k chars), documented. |
+| 1e | Tables without a `modifyDate`-style field | Missing | Default `basepullTimestampField = "modifyDate"` (`annotations/QuickSearchable.java:54`, `SearchableTableConfig.java:34`). `QuickSearchQBitConfig.validate` (`:73-157`) never checks that the timestamp field or the searchable fields exist on the table. Basepull then fails per table at query time, caught at `BasepullIndexStep.java:128-132` and logged. qqq-all's `customer` table config does not declare one (`qqq-all DemoQBits.java:159-160`). | Validate field existence at produce (fail fast); allow `basepullTimestampField = null` meaning "no incremental basepull, reconcile only". |
+| 1f | Non-integer primary keys | Partial | `recordId` is a String everywhere (`OpenSearchDocument.java:35`, `QuickSearchResult.java:28`); keyset paging uses `GREATER_THAN` on the PK (`ReconcileIndexStep.java:222-238`), which works for strings/UUIDs only if the backend sorts them consistently. Untested. | Integration test with a STRING PK table; document ordering assumption. |
+| 1g | Composite keys | Missing | Single `primaryKeyField` model (`QuickSearchableTableConfig.java:25`). | Out of scope for 1.0; state it in README. |
+| 1h | Table removed from config (orphaned documents) | Missing | Reconcile iterates `getDiscoveredTables()` only (`ReconcileIndexStep.java:98`); documents for a table no longer configured are never deleted and are still returned by an unfiltered search (`QuickSearchOpenSearchClient.java:481-487` filter only when `tableName` given). `quickSearchIndex` rows for removed tables stay `ACTIVE`. | Reconcile (or a startup check) should delete-by-query every `sourceTable` not in the discovered set and mark the index row `ORPHANED`/disabled. |
+| 1i | Table renamed / prefix changed | Missing | `sourceTable` is the QQQ table name; `tableNamePrefix` applies only to the two operational tables and processes (`QuickSearchQBitConfig.applyPrefix`). Renaming a source table leaves old documents behind (same as 1h). Changing `opensearchIndexName` silently starts an empty index (`ensureIndexExists` creates it, `:173-192`) with no reindex trigger. | Same mechanism as 1h plus an "index is empty but source has rows" warning in the admin table. |
+| 1j | Config changes to fields/weights/analyzer | Missing | `searchableFieldsJson` is written once when the index row is created (`AbstractIndexingStep.java:167-175`) and never compared again. `ensureIndexExists` returns early if the index exists (`QuickSearchOpenSearchClient.java:167-171`), so a pre-existing index with a different analyzer or mapping is used as-is. Weights are query-time boosts on `fieldValues.*` (`:526-565`), so weight changes need no reindex, but field additions do. | Compare current config to `searchableFieldsJson` at produce/basepull and set `status = NEEDS_REINDEX`; verify the live mapping's analyzer settings and refuse to start (or warn loudly) on mismatch. |
+| 1k | Index name collisions on a shared cluster | Missing | One flat index per config, name is whatever the host passes (`QuickSearchQBitConfig.java:51`). Two apps with the same default would share documents; there is no `appId`/environment field in the document or filter. `fieldValues` is a single dynamic object shared by every table (`:192`): two tables with the same field name but different types (e.g. `code` INTEGER in one, STRING in another) produce `mapper_parsing_exception` bulk-item failures for the second table. | Document required index naming per environment; namespace `fieldValues` per table (`fieldValues.<table>.<field>`) or map all values as `keyword`/`text` strings to avoid type conflicts. |
+| 1l | Basepull catch-up | Partial | Works per enabled row (`BasepullIndexStep.java:74-134`). Defects: never scheduled (no `withSchedule` anywhere; `QuickSearchQBitProducer.java:510-518`); `lastBasepullTime` is set to `Instant.now()` after the run (`:268-279`), not the run's start, so rows modified during the run but after their page was read are skipped forever; `GREATER_THAN` on a seconds-precision DB column vs. millisecond `Instant` misses same-second rows; limit/skip paging without order-by (`AbstractIndexingStep.java:358-359`) is nondeterministic under concurrent writes. | PRD E2 covers scheduling and ordered paging. Add: capture `runStart` before querying and use it (minus a small overlap) as the new watermark; use `GREATER_THAN_OR_EQUALS` with the overlap. |
+| 1m | Full reindex | Partial | Wipes the table's documents first (`FullReindexStep.java:152`) causing a search blackout; limit/skip paging without order-by (`:162`); does not honour `enabled`; `recordCount` written only on success (`:191-204`). | PRD E2 (alias swap) covers the blackout. |
+| 1n | Reconcile | Complete for its design | Keyset paging (`ReconcileIndexStep.java:222-238`), refresh then delete-stale by `indexedAt` (`:191-193`), aborts stale-delete on any failure (`:195-198`). Relies on cross-node clock sync (class Javadoc `:62-63`). Not scheduled. Does not handle orphaned tables (1h). | Schedule it (PRD E2); add orphan-table sweep. |
+| 2a | Term normalisation and short/empty terms | Partial | `IndexingUtils.normalizeSearchText` lowercases and collapses whitespace (`:58-66`); blank returns empty output (`QuickSearchAction.java:67-74`). No minimum length: a 1-char term yields zero edge-ngram tokens (min_gram 2) so it silently returns nothing; no maximum length (core caps at 100). | Enforce min 2 / max 100 characters with a clear message, matching core. |
+| 2b | Multi-word queries | Missing (untuned) | `multi_match` on `searchableText` with no `operator`, no `minimum_should_match`, no `type` (`QuickSearchOpenSearchClient.java:472-475`). Default is OR, so "blue widget" matches every document containing "bl", "blu", "wi", ... and ranks by ngram overlap. | Set `operator = AND` (or `minimum_should_match = "2<75%"`) and type `best_fields` or `cross_fields`; cover with an integration test using two-word terms. |
+| 2c | Phrase matching | Missing | No `match_phrase` clause; quoted input is passed through as-is. | Optional for 1.0; document absence. |
+| 2d | Prefix/typeahead behaviour | Partial (defective) | `searchableText` uses the edge-ngram analyzer for both indexing and search; no `search_analyzer` is set (`:190`). The query term is therefore also edge-ngrammed: "order" becomes or/ord/orde/order, so "or" matches as strongly as the full word and relevance degrades with term length. The boost should-clauses hit `fieldValues.*`, which are dynamically mapped with the standard analyzer, so field weights only apply to whole-token matches, never to prefix matches. | Set `search_analyzer: standard` (lowercase only) on `searchableText`; either map `fieldValues.*` with the same edge-ngram analyzer via a dynamic template or drop field-level boosting into per-field sub-fields of `searchableText`. |
+| 2e | Fuzziness | Missing | None configured. | 1.x (PRD explicitly out of scope). |
+| 2f | Per-table result limits and grouping | Missing | Single flat ranked list; `tableName` filter accepts one table (`QuickSearchInput.java:29`). One hot table swamps the others. Core has `limitPerTable` (default 5, max 25). | Add `tableNames` (list) and `limitPerTable` (use an OpenSearch `terms` aggregation with `top_hits`, or `collapse`); needed to map onto core `RecordSearchInput`. |
+| 2g | Result fields | Partial | `QuickSearchResult` has `tableName, recordId(String), recordLabel, score, highlightSnippet` (`actions/QuickSearchResult.java:27-31`). No `tableLabel`, no typed `recordId`, no route/URL hint, no display fields, no `relation` on total. Core `RecordSearchResult` has `tableLabel` and `Serializable recordId`. | Add `tableLabel` (from `QInstance.getTable(name).getLabel()`), typed `recordId`, and optionally the matched field name. |
+| 2h | Max page depth | Missing | `limit` has no upper bound (`QuickSearchAction.java:78`); `from + size > 10000` hits `index.max_result_window` and surfaces as a raw QException. | Cap `limit` (e.g. 100) and `offset + limit` (e.g. 10000); return a clean error. |
+| 2i | Highlight quality | Partial | Highlight only on `searchableText` with defaults (`QuickSearchOpenSearchClient.java:491-492`); fragments joined with "..." (`QuickSearchAction.java:104-109`). Snippets include the `fieldName: ` prefix when `includeLabel` is set (`IndexingUtils.java:106-108`), exposing internal field names; no `pre/post_tags` control; nothing stops a snippet from a field the user may not see (PRD E3 mentions this). | Configurable tags and fragment size; strip label prefixes or use the QQQ field label. |
+| 2j | Sorting options | Missing | Score only. | Fine for 1.0; document. |
+| 2k | Total hit accuracy | Missing | `track_total_hits` not set, so `totalHits` caps at 10,000 and `hasMore` is computed from the capped value (`QuickSearchAction.java:123-124`); `relation` is ignored. | Set `track_total_hits` to a bounded number and expose `totalHitsIsLowerBound`. |
+| 2l | Permissions / session | Missing | No `PermissionsHelper`, no `QContext` use anywhere in `actions/` or `opensearch/`. Core `RecordSearchAction.java` checks `TablePermissionSubType.READ` and `Capability.TABLE_QUERY` per table and personalises fields. | PRD E3. |
+| 3a | `quickSearchIndex` fields | Partial | `id, tableName, enabled, basepullIntervalMinutes, basepullTimestampField, searchableFieldsJson, lastBasepullTime, lastFullReindexTime, recordCount, status, createDate, modifyDate` (`QuickSearchQBitProducer.java:456-471`). `status` is always "ACTIVE" (`AbstractIndexingStep.java:175`) and never updated; no `lastError`, `lastRunStatus`, `documentCount` (from OpenSearch), `lastReconcileTime`, `indexName`. No PVS, no sections, no `isRequired`, no `isEditable=false` on system-managed fields, no labels beyond defaults. | Add `lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`; make tracking fields read-only; add field sections and a tableName PVS. |
+| 3b | `quickSearchIndexRun` fields | Partial | `quickSearchIndexId, runType, status, startTime, endTime, recordsProcessed, recordsIndexed, errorCount, errorMessage` (`:482-497`). `errorMessage` is a plain STRING holding only the first error (`BasepullIndexStep.java:254-257`); no per-item error table; `quickSearchIndexId` has no PVS/join so the admin UI shows an integer. | Join/PVS to `quickSearchIndex`; store error list (JSON or child table); set `errorMessage` max length. |
+| 3c | Run records for every path | Partial | Written for basepull, full reindex, reconcile (`createRunRecord`/`completeRunRecord`). Not written for real-time customizer failures (they only WARN) nor for "skipped, not due" basepull iterations. Nothing is scheduled, so there are no scheduled runs today. | A failed-event table or a `REALTIME_FAILURE` run row per failed publish (see 4c). |
+| 3d | Retention/cleanup of run history | Missing | No purge; unbounded growth once scheduled (every 5 minutes per table). | Scheduled purge (e.g. keep 30 days or N rows per index) in the same scheduled process. |
+| 3e | `enabled` honoured at runtime | Partial | Only basepull filters `enabled = true` (`BasepullIndexStep.java:89-92`). Customizers, full reindex, reconcile and search ignore it, so disabling a table in the admin UI still indexes it on every write and still returns its rows. | Customizers and processes consult the index row (cached, refreshed per run); search excludes disabled tables via a `terms` filter. |
+| 3f | Status reporting / counts | Partial | `recordCount` updated only on full reindex/reconcile success; no live document count; no health field. | See 3a. |
+| 3g | Admin widgets or dashboards | Missing | App is two tables plus three processes (`QuickSearchQBitProducer.java:247-258`); no widgets, no icon. | A small dashboard widget (per-table status, last run, failures) is a 1.x nicety; an icon and labels are 1.0 polish. |
+| 3h | Process input screens | Missing | All three processes are a single backend step with no frontend step and no input `QFieldMetaData` (`:510-549`), yet full reindex and reconcile read `tableName` from input (`FullReindexStep.java:73`, `ReconcileIndexStep.java:93`). From the admin UI there is no way to supply it, so UI-triggered runs always hit every table. README's "or through the QQQ admin UI" is therefore only half true. | Add a frontend step with an optional `tableName` field backed by a PVS of discovered tables, plus a summary screen showing counts (reconcile already emits `recordsIndexed`/`documentsRemoved`). |
+| 3i | Labels/help text | Partial | Table and process labels set; no field labels/help, no app icon/section. | Polish pass. |
+| 4a | Logging | Partial | QLogger with structured pairs; failures at WARN. Basepull logs "not due" at INFO per table per run (`BasepullIndexStep.java:118-120`), which is noisy every 5 minutes. | Downgrade to DEBUG; add one INFO summary per run. |
+| 4b | Metrics / health check | Missing | No counters, no timers, no `ping`/cluster-health call; the client constructor never connects (`QuickSearchOpenSearchClient.java:94-147`), so "client created" says nothing about reachability. | Expose a health method (indices.exists or cluster health) and surface it in the admin table or a QQQ health endpoint; count publish failures. |
+| 4c | OpenSearch down at startup vs. runtime | Partial (hazardous) | Startup: client build rarely fails; `ensureIndexExists` failure is WARN-and-continue (`QuickSearchQBitProducer.java:174-181`), publisher is still created (`:192-195`), host boots "healthy". Runtime: every host insert/update/delete calls OpenSearch synchronously; failures are swallowed (customizers) so the host write succeeds and the index silently diverges; `QuickSearchAction` throws. No timeouts are configured on the transport (no `setConnectTimeout`/`setResponseTimeout`/`RequestConfig` anywhere), so a black-holed host blocks each write for the OS connect timeout (tens of seconds to minutes) and a slow cluster blocks indefinitely. | Fail-fast at produce with opt-out (PRD E1). Add explicit connect/response timeouts in config (e.g. 2s/10s). Record failed events (see 4e). |
+| 4d | Retry/backoff | Missing | None; a transient 429/503 on bulk is a hard failure. | Bounded retry with jitter on bulk 429/5xx inside the client (sync path), documented. |
+| 4e | Dead-letter / failed-event capture | Missing | Failed real-time events vanish after a WARN. Basepull only recovers rows whose `modifyDate` moves; a failed delete is recovered only by reconcile. | A `quickSearchFailedEvent` table (table, recordId, action, error, attempts) replayed by the scheduled process; this is the cheapest correct answer without an async queue. |
+| 4f | Circuit breaking | Missing | None. | Simple breaker in `SynchronousIndexEventPublisher` (open after N consecutive failures, half-open after T seconds; skip and record events while open). With timeouts (4c) this is what keeps a down cluster from slowing every host write. |
+| 5a | Index name per environment | Partial | Host supplies the name; qqq-all reads `OPENSEARCH_INDEX` (`DemoQBits.java:155`). No validation of OpenSearch naming rules (lowercase, no spaces). | Validate name; document per-environment convention. |
+| 5b | Analyzer customisation | Missing | Hard-coded `standard` tokenizer + `lowercase` + `edge_ngram(2,20)` (`QuickSearchOpenSearchClient.java:176-185`). | Expose `minGram/maxGram` and an optional analyzer-settings hook (see 6c). |
+| 5c | Refresh policy | Missing | Bulk requests use default (near-real-time, ~1s); only tests call `refreshIndex`. Hosts that read-after-write will not see the row. | `refreshPolicy` config (none/wait_for) for the real-time path; document the default. |
+| 5d | `sourceBatchSize` vs `bulkBatchSize` | Complete but undocumented | Source page size for QQQ queries (`:1000`) vs documents per bulk request (`:500`) (`QuickSearchQBitConfig.java:58-59`). README table describes both correctly. | Note interaction (one source page = ceil(1000/500) bulk calls). |
+| 5e | Timeouts | Missing | See 4c. | Add. |
+| 5f | Multiple QBit instances / multiple indexes | Missing | Static `QuickSearchQBitContext` (`:34-37`); a second `produce()` overwrites the first; customizers and steps resolve state statically. | PRD E1 (instance-scoped holder). |
+| 5g | Excluded tables | Missing | No way to exclude, only to include; `enabledByDefault=false` still registers customizers and creates the row. | Fine if `enabled` is honoured (3e). |
+| 5h | Per-table record label strategy | Partial | `recordLabelFormat` only on config-driven tables (`SearchableTableConfig.java:36-37`); annotation path uses `QRecord.getRecordLabel()` which is null unless the query generated it (`IndexingUtils.java:142`). Customizers receive records from Insert/Update actions where `recordLabel` is typically not populated, so real-time-indexed documents often have a null `recordLabel`. Core falls back to `QValueFormatter.formatRecordLabel`. | Fall back to `QValueFormatter.formatRecordLabel(table, record)` in `buildDocument`; allow label format on the annotation too. |
+| 5i | Field type handling (dates, decimals, booleans, PVS labels) | Missing | Raw values only: `getValueString` for text (`IndexingUtils.java:94`) and raw `Object` into `fieldValues` (`:150-154`). Dates index as ISO instants, booleans as "true", PVS fields as their raw id. No display values, no PVS translation. Core sets `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` on its queries. | Index display values (translate PVS, format dates) into `searchableText` and keep raw values in `fieldValues`; batch processes already go through `QueryAction` so this is a flag; customizers need a re-read with display values (update already re-reads). |
+| 5j | Flag semantics | Partial | `enableScheduledProcesses=false` removes all three processes including the manual full reindex and reconcile (`QuickSearchQBitProducer.java:228-232`), so a host cannot have on-demand processes without the basepull. | Split into `enableBasepullProcess` and `enableMaintenanceProcesses`, or always register processes and gate only the schedule. |
+| 6a | `IndexEventPublisher` SPI | Partial | `publishIndexEvents`, `publishDeleteEvents`, `close()` (`publisher/IndexEventPublisher.java:40-60`). `IndexEvent` carries `tableName, recordId, action, QRecord` (`IndexEvent.java:30-33`): the full record snapshot, not a built document, so an async consumer can rebuild the document later via `IndexingUtils.buildDocument(record, tableConfig)` only if it has the table config (currently only through the static context). There is no event timestamp, sequence or record version, so a queue consumer cannot order an INDEX after a later DELETE. `close()` is never invoked by the producer or any shutdown hook. The publisher receives no `QSession`/tenant information. | Add `occurredAt` (Instant) and an optional `version` to `IndexEvent`; pass the table config map (or a resolver) into the publisher constructor contract; register `close()` with the host lifecycle; document ordering expectations. |
+| 6b | Document builder hook | Missing | `IndexingUtils.buildDocument` is static and called directly from publisher and steps. | `DocumentBuilder` interface with a default implementation, settable on `QuickSearchQBitConfig` (lets hosts add computed text, strip PII, index display values). |
+| 6c | Custom analyzer hook | Missing | Hard-coded in `ensureIndexExists`. | `IndexSettingsCustomizer` (consumer of `CreateIndexRequest.Builder`) or at least `minGram/maxGram/searchAnalyzer` config. |
+| 6d | Result post-processing hook | Missing | `QuickSearchAction.execute` maps hits inline (`:95-121`). | `ResultCustomizer` to add route hints or filter; also where permission filtering would plug in. |
+| 7a | Unit coverage | Partial | 233 unit tests across all packages; happy paths and null/empty guards well covered (see test list in section 4). JaCoCo `check` is configured (`pom.xml:188-212`) but with no `prepare-agent` execution, so the 0.70/0.90 gates never run (PRD notes this). | Bind `prepare-agent`; keep gates. |
+| 7b | Integration coverage | Partial | 12 tests in `OpenSearchIntegrationTest`, `IndexDriftIntegrationTest`, `ConfigDrivenIndexingIntegrationTest`. Every search uses the single whole-word term "widget" or "zzzznonexistent"; no prefix, multi-word, boost-ordering, highlight-content, pagination-beyond-first-page-correctness, basepull, or scheduling test. | See section 4 list. |
+| 7c | README promises not met | Partial | "Scheduled basepull" (README:14, 30, 287) is not scheduled. "Prefix indexed text with the field label" (README:171, 331) prefixes the field *name* (`IndexingUtils.java:108`). "Or through the QQQ admin UI" for per-table reindex (README:254, 271) has no input screen. Dependency snippet shows `0.2.0-SNAPSHOT` (README:79). qbit-crm/qbit-wms companion section (README:333-341) describes integrations that do not exist. "70%+ instruction coverage (JaCoCo enforced)" (README:362) is not enforced. "Admin Observability" (README:20) is two raw tables. | PRD E5 rewrite; add the items above to its checklist. |
+| 7d | CHANGELOG / migration notes | Missing | No `CHANGELOG.md` at repo root; no migration doc. | PRD E5. |
+| 7e | Configuration reference accuracy | Partial | README table (291-312) matches `QuickSearchQBitConfig` fields but omits the `enableScheduledProcesses` side effect (5j), the `indexEventPublisher` lifecycle, and says nothing about timeouts, refresh, or required cluster privileges. | Fix with E5. |
+| 7f | Javadoc on public API | Mostly complete | Flower-box Javadoc on nearly every public member. Gaps: `IndexingUtils.buildDocument` documents `fieldWeights` as "reserved for future use" (`:130`) although weights drive query boosts; `QuickSearchQBitProducer` public constants (`:54-57`) undocumented; no `package-info.java`; `IndexEventPublisher` does not state thread-safety or ordering guarantees. | Small pass with E5. |
+
+## 2. Comparison with QQQ 4.1 core `RecordSearchAction`
+
+Source: `qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/RecordSearchAction.java`, `model/actions/tables/search/RecordSearch{Input,Output,Result}.java`, `QTableMetaData.searchFields` (`QTableMetaData.java:100, 2013-2060`), Javalin `RecordSearchExecutor` (passes `QInputSource.USER`).
+
+Core has, Quick Search lacks:
+
+| Capability | Core | Quick Search |
+|---|---|---|
+| Table-level READ permission check | `PermissionsHelper.hasTablePermission(..., READ)` per table; unauthorised tables skipped silently (no probing) | None |
+| Capability check (`TABLE_QUERY`) | Yes | N/A (reads from index) but no equivalent gate |
+| Field personalisation; hidden/masked fields excluded | `TableMetaDataPersonalizerAction`, `getIsHidden`, `needsMasked` | None; whatever is configured is indexed and highlighted |
+| Record-level security locks | Inherited from `QueryAction` | None |
+| Per-table limit | `limitPerTable` default 5, max 25 | Global `limit`, no max |
+| Multi-table filter | `tableNames` list | Single `tableName` |
+| Term validation | Blank -> `QBadRequestException`; max 100 chars | Blank -> empty output; no max |
+| Query timeout | 30s per table | None |
+| Read-replica hint | `QueryHint.MAY_USE_READ_ONLY_BACKEND` | N/A |
+| Result shape | `tableName, tableLabel, Serializable recordId, recordLabel` | `tableName, String recordId, recordLabel, score, highlightSnippet`; no `tableLabel` |
+| Record label fallback | `QValueFormatter.formatRecordLabel` | Null when the record had no label |
+| Display values / PVS translation | `shouldGenerateDisplayValues`, `shouldTranslatePossibleValues` | Raw values |
+| Deterministic ordering | Order by PK | Score (fine) |
+| Integer term matching | Integral fields match numeric terms exactly | Numbers indexed as text tokens; "42" prefix-matches "4200" |
+| Metadata declaration | `QTableMetaData.withSearchFields(...)` is first-class metadata the Next UI can read | Separate annotation/config model; `searchFields` on the table is ignored |
+| UI surface | `POST /qqq/v1/search` and the Next UI search box | None |
+
+Quick Search has, core lacks:
+
+| Capability | Quick Search | Core |
+|---|---|---|
+| Relevance scoring and field weights | BM25 with per-field boosts | None (PK order) |
+| Prefix/typeahead | Edge-ngram (with the defect in 2d) | `CONTAINS` (LIKE `%term%`), which is a full scan on large tables |
+| Highlights | Yes | No |
+| Pagination and total | `offset/limit`, `totalHits`, `hasMore` | Top-N per table only |
+| Cross-table ranking | Single ranked list | Per-table lists concatenated |
+| Performance isolation | Query hits OpenSearch, not the OLTP backend | One LIKE query per searchable table per keystroke |
+| Operational history | Run tables | None |
+| Async-capable transport | `IndexEventPublisher` SPI | N/A |
+
+Integration consequence (PRD E4/D1): to back the Next UI, a `QuickSearchRecordSearchProvider` must accept `RecordSearchInput` (`tableNames`, `limitPerTable`, `inputSource`) and return `RecordSearchResult` (`tableLabel`, typed `recordId`). Quick Search today cannot satisfy `limitPerTable` or `tableNames` without the changes in 2f, and cannot honour the permission semantics core users will assume without E3. Also, a host that sets `QTableMetaData.searchFields` for core and `@QuickSearchable` for the QBit now maintains two field lists; 1.0 should at least default the QBit's field list from `searchFields` when present.
+
+## 3. Prioritised gap list
+
+Severity: P0 = blocks a credible 1.0; P1 = should ship in 1.0; P2 = 1.0 if cheap, else 1.x; P3 = 1.x.
+
+| ID | Sev | Gap | Evidence | Recommended scope |
+|---|---|---|---|---|
+| G01 | P0 | No connect/response timeouts on the OpenSearch transport; a down or black-holed cluster blocks every host write for the OS timeout, indefinitely for a hung response | `QuickSearchOpenSearchClient.java:94-147`; grep for timeout in `src/main` returns nothing | 1.0. Add `connectTimeoutMs`/`responseTimeoutMs` config with conservative defaults. Not in PRD. |
+| G02 | P0 | Search ignores permissions, hidden/masked fields, record locks | `actions/QuickSearchAction.java`, no `PermissionsHelper` in repo; core does all three | 1.0 (PRD E3). Extend E3 to exclude hidden/masked fields at index time, not just at search time. |
+| G03 | P0 | Basepull and reconcile never scheduled | No `withSchedule`; `QuickSearchQBitProducer.java:510-518` | 1.0 (PRD E2, D4). |
+| G04 | P0 | Edge-ngram analyzer applied at query time too (no `search_analyzer`); multi_match defaults to OR with no `minimum_should_match`; field boosts never apply to prefix matches | `QuickSearchOpenSearchClient.java:190, 472-475, 526-565` | 1.0. Set `search_analyzer`, `operator=AND`, dynamic template for `fieldValues.*`. Requires a reindex (mapping change) so it must land before 1.0's first index. Not in PRD. |
+| G05 | P0 | Startup is best-effort; host boots healthy with search broken | `QuickSearchQBitProducer.java:160-181` | 1.0 (PRD E1). |
+| G06 | P1 | Real-time failures are swallowed with no record; no retry, no breaker, no dead-letter | `customizers/*:99-120`; `SynchronousIndexEventPublisher` | 1.0: failed-event table replayed by the scheduled process, plus bounded retry on bulk 429/5xx. Breaker can be 1.x if timeouts (G01) ship. Not in PRD. |
+| G07 | P1 | `quickSearchIndex.enabled` ignored by customizers, full reindex, reconcile and search | Only `BasepullIndexStep.java:89-92` filters on it | 1.0. Not in PRD. |
+| G08 | P1 | Basepull watermark set after the run; same-second rows and rows modified mid-run are skipped forever | `BasepullIndexStep.java:268-279`, `:206-209` | 1.0 alongside E2 ordered paging; capture start time, apply overlap. Not in PRD (E2 mentions only ordering). |
+| G09 | P1 | `fieldValues` shared dynamic object across tables causes mapping type conflicts when two tables share a field name with different types | `QuickSearchOpenSearchClient.java:192`; `IndexingUtils.java:150-154` | 1.0 (mapping change, same reindex as G04): namespace per table or map as keyword. Not in PRD. |
+| G10 | P1 | Static JVM-global context; one QBit instance per JVM | `QuickSearchQBitContext.java:34-37` | 1.0 (PRD E1). |
+| G11 | P1 | Customizer slot collision fails produce | `QuickSearchQBitProducer.java:571-579`; core `QTableMetaData.java:651-653` | 1.0 (PRD E1). |
+| G12 | P1 | Full reindex blackout; unordered paging | `FullReindexStep.java:152, 162` | 1.0 (PRD E2, D5). |
+| G13 | P1 | No validation that searchable fields and the basepull timestamp field exist on the table; silent PK fallback to "id" for tables missing from the QInstance | `QuickSearchQBitConfig.java:73-157`; `QuickSearchQBitProducer.java:349, 418` | 1.0 (fold into E1 fail-fast validation). Partly in PRD ("fail-fast config validation") but field existence is not named. |
+| G14 | P1 | Orphaned documents when a table is removed from config or renamed; reconcile only sweeps configured tables | `ReconcileIndexStep.java:98`; `QuickSearchOpenSearchClient.java:481-487` | 1.0 (small addition to reconcile). Not in PRD. |
+| G15 | P1 | No config-drift detection: `searchableFieldsJson` never compared; existing index mapping/analyzer never verified | `AbstractIndexingStep.java:167-175`; `QuickSearchOpenSearchClient.java:167-171` | 1.0: compare and set `NEEDS_REINDEX`; verify analyzer on existing index and warn. Not in PRD. |
+| G16 | P1 | Result model lacks `tableLabel`, typed `recordId`, `tableNames`/`limitPerTable` input; cannot map onto core `RecordSearchResult` cleanly | `actions/QuickSearchInput.java:28-31`, `QuickSearchResult.java:27-31` | 1.0 (needed for PRD E4 even if the SPI slips). PRD E4 names the mapping but not the input-side gaps. |
+| G17 | P1 | Processes have no input screen; `tableName` cannot be supplied from the admin UI | `QuickSearchQBitProducer.java:510-549`; `FullReindexStep.java:73` | 1.0: frontend step with optional tableName PVS. Not in PRD. |
+| G18 | P1 | `totalHits` capped at 10,000 silently; `limit`/`offset` unbounded | `QuickSearchAction.java:78-79, 123-124` | 1.0: bound limit/offset, set `track_total_hits`, expose lower-bound flag. Not in PRD. |
+| G19 | P1 | Raw values indexed; PVS ids, booleans, dates not searchable by their display form; `recordLabel` often null on real-time path | `IndexingUtils.java:94, 142, 150-154` | 1.0: index display values, fallback to `QValueFormatter.formatRecordLabel`. Not in PRD. |
+| G20 | P1 | `enableScheduledProcesses=false` also removes manual full reindex and reconcile | `QuickSearchQBitProducer.java:228-232` | 1.0: split the flag (backwards compatible: keep old name as alias). Not in PRD. |
+| G21 | P2 | Run history grows unbounded once scheduled; no purge | No purge code | 1.0 if trivial (one delete-by-filter in the scheduled step), else 1.1. Not in PRD. |
+| G22 | P2 | `quickSearchIndex` lacks `lastRunStatus`, `lastError`, `documentCount`, `lastReconcileTime`; `status` never changes; no PVS/joins/sections | `QuickSearchQBitProducer.java:456-497`; `AbstractIndexingStep.java:175` | 1.0 polish with E5; schema additions are cheap before 1.0 and painful after. Not in PRD. |
+| G23 | P2 | Health check and metrics absent | No ping/health call | 1.0: a health method surfaced in the admin table; metrics 1.x. Not in PRD. |
+| G24 | P2 | No term min/max length; 1-char term returns nothing silently | `QuickSearchAction.java:67-74` | 1.0 (trivial, align with core 100-char cap). Not in PRD. |
+| G25 | P2 | Large text fields unbounded | `IndexingUtils.buildSearchableText` | 1.0: per-field `maxLength` default. Not in PRD. |
+| G26 | P2 | `IndexEvent` has no timestamp/version; publisher `close()` never called; publisher cannot resolve table configs without the static context | `publisher/IndexEvent.java:30-33`; `IndexEventPublisher.java:60` | 1.0 if the SPI is to be stable at 1.0 (PRD says "the SPI stays"): add `occurredAt`, pass config resolver, wire `close()`. Otherwise mark SPI experimental in 1.0. Not in PRD. |
+| G27 | P2 | No document-builder, analyzer or result hooks | `IndexingUtils` static; `ensureIndexExists` hard-coded | 1.1 unless needed for E3 (a result hook is the natural place for permission post-filtering). Not in PRD. |
+| G28 | P2 | Refresh policy not configurable; read-after-write misses | Bulk requests default | 1.1; document the 1s default in 1.0. Not in PRD. |
+| G29 | P2 | Highlight snippets expose field names via `includeLabel` prefix; README says "label" | `IndexingUtils.java:106-108` | 1.0 docs fix at least; use QQQ field label in 1.0 if cheap. |
+| G30 | P3 | Fuzziness, phrase queries, synonyms, language analyzers, sort options | Not present | 1.x (PRD out of scope). |
+| G31 | P3 | Composite keys, multi-index layouts | Not present | 1.x (PRD out of scope). |
+| G32 | P1 | Tests: JaCoCo gate inert; integration suite exercises one whole-word term only; no basepull, scheduling, permission, typeahead, multi-word, boost-order, non-integer PK, OpenSearch-down, mapping-conflict tests | `pom.xml:188-212`; test list in section 4 | 1.0 (PRD E1 covers the agent; add the concrete tests below). |
+| G33 | P1 | Docs: README promises (scheduled basepull, admin UI per-table reindex, coverage enforced, qbit-crm/wms integrations, field "label"), no CHANGELOG or migration notes | `README.md:14, 30, 79, 171, 254, 271, 287, 333-341, 362` | 1.0 (PRD E5). |
+
+## 4. Test inventory and concrete untested behaviours
+
+Unit (233): `QuickSearchQBitConfigTest` (34), `QuickSearchQBitProducerTest` (23), `IndexingUtilsTest` (19), `QuickSearchActionTest` (16), `QuickSearchOpenSearchClientTest` (13, constructor/unavailable-server only), `SynchronousIndexEventPublisherTest` (12), `BasepullIndexStepTest` (9), `QuickSearchPostUpdateCustomizerTest` (9), `QuickSearchQBitContextTest` (9), `FullReindexStepTest` (8), `QuickSearchPostDeleteCustomizerTest` (8), `BulkIndexResultTest` (8), `QuickSearchableTableConfigTest` (8), `QuickSearchDTOsTest` (7), `QuickSearchAnnotationsTest` (7), `ReconcileIndexStepTest` (6), `QuickSearchPostInsertCustomizerTest` (6), `SearchableFieldConfigTest` (6), `SearchableTableConfigTest` (6), `IndexEventTest` (4), `OpenSearchDocumentTest` (4), `QuickSearchIndexTest` (4), `QuickSearchIndexRunTest` (4), `RequiresDockerConditionTest` (3).
+
+Integration (12, Testcontainers OpenSearch 2.11): `OpenSearchIntegrationTest` (index created, full reindex, search returns, pagination, no results), `ConfigDrivenIndexingIntegrationTest` (discovery, full reindex, label format, no results), `IndexDriftIntegrationTest` (partial update keeps fields, delete with missing doc, reconcile repairs orphan/missing/stale).
+
+Concretely untested:
+
+1. Query shape: no test asserts the OpenSearch request (operator, boosts, highlight, from/size). `QuickSearchOpenSearchClient.search` is only exercised end-to-end with "widget".
+2. Typeahead: README's "ord finds order" is never tested; nor that "or" does not outrank "order".
+3. Multi-word terms and `minimum_should_match` behaviour.
+4. Field weight ordering (a weight-3 field match ranks above a weight-1 match).
+5. Highlight fragment content against a real cluster.
+6. Basepull against a real cluster; watermark semantics; timestamp filter with seconds-precision columns; rows modified mid-run.
+7. Any scheduling (nothing to test yet).
+8. Permission denial (nothing to test yet).
+9. Non-integer and string primary keys through reconcile keyset paging.
+10. OpenSearch down at runtime during a host insert (only a mocked publisher-throws test exists); produce with an unreachable host beyond `ensureIndexExists` failing.
+11. Pre-existing index with a different mapping/analyzer.
+12. Two tables sharing a field name with different types (mapping conflict).
+13. `totalHits` above 10,000; `offset + limit` above `max_result_window`.
+14. `enableScheduledProcesses=false` behaviour; `tableNamePrefix` applied to processes/app end to end.
+15. Duplicate `@QuickSearchable(tableName)` across two entity classes (only annotation-vs-config duplicates are checked, `QuickSearchQBitProducer.java:144`).
+16. Large text fields; `includeLabel` text appearing in snippets.
+17. Publisher `close()` lifecycle.
+18. `enabled=false` row: customizers/search still index and return (would currently fail if written as a negative test, which is the point).
+
+## 5. Items the draft PRD does not mention
+
+The PRD's five epics cover: 4.1 contract and fail-fast (E1), scheduling, ordered paging, alias-swap reindex, per-item run errors (E2), permissions (E3), core SPI and result mapping (E4), docs and release (E5). It does not mention:
+
+1. Transport timeouts (G01). The single largest operational risk; without it, "OpenSearch down" slows every host write.
+2. Search query quality: `search_analyzer`, `operator`/`minimum_should_match`, boosts not applying to prefixes (G04). This changes the mapping and should be bundled with the alias-swap work so 1.0 starts with the right index.
+3. `fieldValues` cross-table mapping conflicts (G09). Same reindex.
+4. Honouring `enabled` outside basepull (G07).
+5. Basepull watermark correctness (G08). E2 says "order-by primary key" but not the start-time watermark or overlap.
+6. Failed-event capture, retry, circuit breaker (G06). E2's "run records capture per-item errors" covers batch runs only; real-time failures still vanish.
+7. Orphaned documents for removed/renamed tables (G14).
+8. Config/mapping drift detection and `NEEDS_REINDEX` status (G15).
+9. Validation that configured fields and the timestamp field exist on the table; `basepullTimestampField = null` for tables without one (G13). E1 says "fail-fast config validation" generically.
+10. Process input screens with a table PVS (G17). Without it, the "Or through the QQQ admin UI" README promise stays false after E5.
+11. `totalHits` cap, `limit`/`offset` bounds, term length bounds (G18, G24).
+12. Display values and PVS translation at index time; `recordLabel` fallback (G19). Core does both; a UI backed by Quick Search will show different labels than core for the same record.
+13. Input-side parity with core (`tableNames`, `limitPerTable`) and `tableLabel` on results (G16). E4 mentions only the output mapping.
+14. The `enableScheduledProcesses` flag removing manual processes (G20).
+15. Run-history retention (G21) and the admin table's missing status/error/count fields (G22). Schema changes are cheap before 1.0.
+16. Health check (G23).
+17. `IndexEvent` ordering metadata and publisher lifecycle (G26). The PRD keeps the SPI but does not say whether it is stable; as it stands an async implementation cannot be correct.
+18. Large-field truncation (G25) and refresh policy (G28).
+19. Defaulting the QBit's field list from `QTableMetaData.searchFields` so hosts do not maintain two lists.
+20. Voyage as a consumer: the local Voyage checkout (`/Users/james.maes/Git.Local/QRunIO/Voyage`, develop, main, circleci-project-setup) contains no reference to Quick Search or OpenSearch on any ref; the PRD's "Voyage (pins 0.2.0)" stakeholder row should be re-verified or dropped. qqq-all is the only confirmed consumer (`DemoQBits.java:151-163`, config-driven `customer` name/email, hand-registered `QBitMetaData` 0.2.1-SNAPSHOT, no `schedulerName` while the SFTP QBit on the same line uses `withSchedulerName("demoScheduler")`).
+
+## 6. Suggested adjustments to the epics
+
+- E1: add field-existence validation, OpenSearch naming validation, timeouts in config, split of `enableScheduledProcesses`.
+- E2: add start-time watermark with overlap, orphan-table sweep in reconcile, config-drift detection, run-history purge, failed-event table replayed by the scheduled run. Bundle the mapping changes (search_analyzer, operator, fieldValues namespacing, display values) into the alias-swap work so the first 1.0 index is right.
+- E3: add index-time exclusion of hidden/masked fields; honour `enabled` in search.
+- E4: add `tableNames`, `limitPerTable`, `tableLabel`, typed `recordId`, term bounds; default fields from `searchFields`.
+- E5: add process input screens, admin table fields, health method, README corrections listed in 7c, and an explicit "SPI stability" statement for `IndexEventPublisher`.

docs/audits/2026-10-04/05-correctness-failures.md

@@ -0,0 +1,91 @@
+# 05. Correctness and silent-failure review: qbit-quick-search
+
+Scope: every file under `src/main/java/com/kingsrook/qbits/quicksearch` and `src/test/java`, read in full, cross-checked against QQQ `v4.1.0-RC.1` (`InsertAction`, `UpdateAction`, `DeleteAction`, `QueryAction`, `QueryInput`, `QRecord`, `ValueUtils`, `TableCustomizerInterface`, `RDBMSQueryAction`, `RDBMSInsertAction`, `RDBMSUpdateAction`, `RDBMSDeleteAction`, `RDBMSTransaction`, `QInstance`, `ProcessLockUtils`) and the opensearch-java 2.10.0 jar in the local Maven repo. Read-only; no repo file was modified.
+
+Paths below are relative to `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1/src/main/java/com/kingsrook/qbits/quicksearch/` unless prefixed with `qqq:` (relative to `/Users/james.maes/Git.Local/QRunIO/qqq/` at tag `v4.1.0-RC.1`).
+
+## Framework facts this review depends on (verified)
+
+- POST_INSERT / POST_UPDATE / POST_DELETE customizers run synchronously, in the calling thread, immediately after the backend DML and before `execute()` returns (`qqq:qqq-backend-core/.../actions/tables/InsertAction.java:286`, `UpdateAction.java:268`, `DeleteAction.java:370-390`).
+- When the caller passes a `QBackendTransaction`, the RDBMS module runs the DML on that transaction's connection (`qqq:qqq-backend-module-rdbms/.../RDBMSInsertAction.java:73-79`, `RDBMSUpdateAction.java:85-91`, `RDBMSDeleteAction.java:87-93`). `RDBMSTransaction` sets `autoCommit=false` (`RDBMSTransaction.java:55`) and only the caller's later `commit()` (`:87`) or `rollback()` (`:137`) ends it. So the customizers run inside an open transaction.
+- A customizer exception never propagates to the caller; QQQ catches it and attaches a `QWarningMessage("An error occurred after the insert/update/delete: ...")` to each output record (`InsertAction.java:344-350`, `UpdateAction.java:291-297`, `DeleteAction.java:418-426`).
+- Post-delete receives the pre-fetched old records minus those with errors (`DeleteAction.java:376`); post-update receives the (possibly sparse) updated records plus `Optional<oldRecordList>`.
+- RDBMS paging emits `LIMIT n OFFSET m` and emits `ORDER BY` only when the filter has order-bys (`qqq:qqq-backend-module-rdbms/.../RDBMSQueryAction.java:340-354`). There is no default ordering, so unordered paging is not stable.
+- `QueryAction` sets `QRecord.recordLabel` only when `queryInput.getShouldGenerateDisplayValues()` is true (`qqq:.../QueryAction.java:503-506`); the default is `false` (`qqq:.../QueryInput.java:56`).
+- `QRecord.getValueString` is `ValueUtils.getValueAsString`, which returns `String.valueOf(value)` for Instant, LocalDate, BigDecimal, Boolean, and the possible-value id (not the label) for enums (`qqq:.../ValueUtils.java:67-90`).
+- `QInstance.addTable` throws `IllegalArgumentException` on a duplicate name (`qqq:.../QInstance.java addTable`).
+- `org.opensearch.client.opensearch._types.OpenSearchException extends java.lang.RuntimeException` (javap on opensearch-java 2.10.0). The client wrapper only catches `IOException`.
+
+## Findings
+
+| ID | Severity | Finding | Failure scenario | Evidence | Recommended fix | 1.0 blocker |
+|----|----------|---------|------------------|----------|-----------------|-------------|
+| F01 | Critical | Real-time indexing happens inside the caller's open transaction, before commit. | Host code inserts an order inside a `QBackendTransaction`, the post-insert customizer indexes it, the host then rolls back: OpenSearch now holds a document for a row that never existed, and it is only removed by the next reconcile (basepull cannot see deletes). Post-update is worse: it re-reads the row through the same transaction (`setTransaction`) and indexes uncommitted values. | `customizers/QuickSearchPostInsertCustomizer.java:97`; `customizers/QuickSearchPostUpdateCustomizer.java:114,151`; `customizers/QuickSearchPostDeleteCustomizer.java:109`; `qqq:InsertAction.java:286`; `qqq:RDBMSInsertAction.java:73-79`; `qqq:RDBMSTransaction.java:55,87` | QQQ has no post-commit hook, so: (a) when `input.getTransaction() != null`, buffer events keyed by the transaction object and flush them from a wrapper the host calls after `commit()` (ship a `QuickSearchTransactionListener` helper), or at minimum (b) document loudly that real-time indexing is pre-commit and that reconcile is the correction path; and make reconcile the recommended scheduled job. | Yes (fix or document; today the README implies the index tracks the table) |
+| F02 | Critical | Real-time indexing failures are fully silent to the host. | OpenSearch is down for ten minutes; every insert/update/delete in that window logs one WARN line and succeeds normally; no run record, no counter, no record warning. QQQ would have attached a `QWarningMessage` to the returned records if the exception had been allowed to escape, but the customizers swallow it first. The index silently drifts until someone runs reconcile. | `customizers/QuickSearchPostInsertCustomizer.java:99-102`; `QuickSearchPostUpdateCustomizer.java:117-120`; `QuickSearchPostDeleteCustomizer.java:112-115`; `publisher/SynchronousIndexEventPublisher.java:160-171` (throwIfAnyFailed is caught by the customizers and goes nowhere); `qqq:InsertAction.java:344-350` | Remove the blanket catch in the three customizers (QQQ already guarantees the write is not failed; the host gets a warning on the records). Additionally record the failure durably: increment a failure counter on the `quickSearchIndex` row (new `realTimeErrorCount`/`lastRealTimeError`), or write the failed (table, id) pairs to a small dead-letter table that basepull/reconcile drains. | Yes |
+| F03 | High | Basepull and FullReindex page with `LIMIT/OFFSET` and no `ORDER BY`. | While FullReindex pages through 1M rows, a concurrent insert or delete shifts the physical order: a row moves from page 101 to page 100 (already read) and is never indexed, or is read twice. Because FullReindex first deletes every document for the table, a skipped row disappears from search until the next full run. Postgres/MySQL do not guarantee a stable order for repeated unordered `LIMIT/OFFSET` queries even without writes. | `processes/AbstractIndexingStep.java:358-359`; `processes/BasepullIndexStep.java:218-262`; `processes/FullReindexStep.java:158-185`; `qqq:RDBMSQueryAction.java:340-354` | Replace offset paging with the keyset loop already written in `processes/ReconcileIndexStep.java:222-238` (order by PK ascending, `PK > lastKey`). For basepull, keyset on PK with the timestamp criterion added. Delete `querySourceTableBatch` once both callers migrate. | Yes |
+| F04 | High | Basepull window is anchored at the end of the run and compared against the application clock, so records modified during the run are lost. | Run starts at 10:00:00, pages for 90 s; a row updated and committed at 10:00:30 is on a page already read with its old values, or not yet visible; at 10:01:30 `lastBasepullTime = now()`; the next run filters `modifyDate > 10:01:30` and never sees the 10:00:30 change. Any DB/app clock skew widens or flips the gap. `GREATER_THAN` also drops rows whose `modifyDate` equals the boundary exactly. | `processes/BasepullIndexStep.java:198-209` (filter), `:268-279` (`lastBasepullTime = Instant.now()` after the loop); `processes/FullReindexStep.java:196` (same end-of-run stamp) | Capture `windowStart = Instant.now()` before the first query and persist that (as `ReconcileIndexStep.java:142,274` already does), subtract a configurable overlap (default: the basepull interval, or 5 minutes) to absorb skew and long transactions, and use `GREATER_THAN_OR_EQUALS`. Re-indexing is idempotent, so overlap is cheap. | Yes |
+| F05 | High | No document versioning; last writer to OpenSearch wins, not last committer. | Thread A updates row 7 (v1) and, in its post-update, reads v1 back; thread B updates row 7 (v2), commits, indexes v2; thread A's bulk arrives last and overwrites with v1. Same race between FullReindex/Reconcile reading an old page and a real-time write that lands between the read and the bulk. Also applies to delete-then-reinsert of the same id across threads. | `opensearch/QuickSearchOpenSearchClient.java:263-267` (`IndexOperation` with no `version`/`versionType`/`ifSeqNo`); `processes/IndexingUtils.java:164` (`indexedAt = now()` is informational only) | Use external versioning: `versionType(External)` with `version = modifyDate.toEpochMilli()` (or a DB version column) so stale writes are rejected with 409 and counted, not applied. Short of that, document that basepull/reconcile is the convergence mechanism (true only once F04 is fixed). | No, if F04 is fixed and the limitation is documented |
+| F06 | High | Search-time analysis uses the edge_ngram analyzer, so query terms are n-grammed too. | Mapping sets `analyzer("quick_search_analyzer")` with no `search_analyzer`, so OpenSearch analyzes the query with the same edge_ngram filter. Searching `stella` becomes `st OR ste OR stel OR stell OR stella` and matches every document containing any token that starts with `st`. Scores still favour the full term, but `totalHits`, `hasMore`, and pages 2+ are mostly noise. | `opensearch/QuickSearchOpenSearchClient.java:177-190` | Add a second analyzer (`standard` tokenizer + `lowercase`) and set it as `searchAnalyzer` on `searchableText`. Requires index recreation (bump the index name or add a mapping-version check in `ensureIndexExists`, which currently returns early if the index exists at `:165-171` and never reconciles mappings). | Yes |
+| F07 | High (behaviour of OpenSearch assumed from its documented default `lenient=false`; verify with the Docker test) | Boost clauses query `fieldValues.<field>` for every configured field, and `fieldValues` is a dynamic object; a non-string searchable field breaks every search. | A table marks `quantity` (Integer) or `createDate` (Instant) as a `@QuickSearchField`. The first indexed document maps `fieldValues.quantity` as `long`. The next search for `widget` issues `multi_match` against that field with a non-numeric term; OpenSearch returns `query_shard_exception: failed to create query ... For input string: "widget"` and the whole bool query (not just the should clause) fails. Separately, dynamic mapping is first-value-wins: a `sku` whose first value looks like a date maps as `date`, and later SKUs fail per-item with `mapper_parsing_exception`. | `opensearch/QuickSearchOpenSearchClient.java:192` (dynamic `object`), `:536-561` (per-field multi_match, no `lenient`); `processes/IndexingUtils.java:150-155` (raw `Object` values copied into `fieldValues`) | Either set `.lenient(true)` on the should-clause multi_match queries, or define a `dynamic_templates` rule mapping everything under `fieldValues.*` as `text` with the search analyzer, or stringify values when building `fieldValues`. | Yes |
+| F08 | High | `recordLabel` is null for every annotation-configured table. | `buildDocument` copies `record.getRecordLabel()`, but the records come from `QueryAction` calls that never set `shouldGenerateDisplayValues`, and from `InsertAction` output, neither of which populates the label. Only tables with `recordLabelFormat` (config-driven path) get a label. Search results for annotation-based tables show no label, which is the primary display field. | `processes/IndexingUtils.java:142`; `processes/AbstractIndexingStep.java:361-365`; `processes/ReconcileIndexStep.java:233-237`; `customizers/QuickSearchPostUpdateCustomizer.java:149-154`; `qqq:QueryAction.java:503-506`; `qqq:QueryInput.java:56` | In `buildDocument`, when `recordLabelFormat` is absent, call `QValueFormatter.formatRecordLabel(QContext.getQInstance().getTable(tableName), record)`; or set `shouldGenerateDisplayValues(true)` on every source query and translate possible values (also fixes F13). | Yes |
+| F09 | High | FullReindex wipes the table's documents first; a mid-run failure leaves a partially empty index with no automatic recovery. | `deleteDocumentsForTable` runs, then OpenSearch returns 429 on batch 40 of 1000: `OpenSearchException` (unchecked) escapes `indexDocuments`, the run is marked FAILED, 96 percent of the table is unsearchable until someone reruns. Also `deleteByQuery` is called with default `conflicts=abort` and no `refresh`, so a real-time write racing the delete raises `version_conflict_engine_exception` and fails the run after a partial wipe. The FAILED run record from the bulk-item path carries `errorMessage = null`. | `processes/FullReindexStep.java:152` (wipe), `:160-185` (loop), `:210` (null errorMessage on FAILED), `:215-220`; `opensearch/QuickSearchOpenSearchClient.java:388-403` (no `conflicts(Proceed)`, no `refresh`) | Prefer the reconcile algorithm (index into place, then delete stale) for the "full" process too, or build into a fresh index and swap an alias. At minimum: pass the first bulk error into `completeRunRecord`, set `conflicts(Proceed)`, and document the blackout. | Yes for the errorMessage and conflicts fixes; the wipe design should be documented or replaced before 1.0 |
+| F10 | Medium | `OpenSearchException` (unchecked) bypasses every `catch(IOException)` in the client and reaches `QuickSearchAction` callers raw. | Someone deletes the index (`index_not_found_exception`), or a client sends `offset=20000` (`max_result_window`), or F07 fires: `QuickSearchAction.execute` throws a `RuntimeException` rather than the declared `QException`; UI/API layers that only handle `QException` surface a 500 with the raw OpenSearch JSON. In the steps and customizers the `catch(Exception)` blocks do catch it, so only the search path is exposed. | `opensearch/QuickSearchOpenSearchClient.java:197,220,349,374,399,441,506,581` (all `catch(IOException)`); `actions/QuickSearchAction.java:91` | Catch `OpenSearchException` alongside `IOException` in every client method and wrap in `QException` with `error().reason()`. | Yes (small) |
+| F11 | Medium | Search input has no upper bound on `limit` or `offset`. | `limit=100000` builds a 100k-hit response (memory and `max_result_window` error); `offset+limit > 10000` is rejected by OpenSearch and surfaces as F10. `totalHits` is also capped at 10000 by `track_total_hits` default, so `hasMore` is wrong beyond that. | `actions/QuickSearchAction.java:78-79`; `opensearch/QuickSearchOpenSearchClient.java:497-502` | Clamp `limit` to a configurable max (e.g. 100) and reject or clamp `offset` so `from + size <= 10000`; set `trackTotalHits(true)` only if exact counts matter. | Yes (small) |
+| F12 | Medium | `quickSearchIndex.enabled` is honoured only by basepull; the row's `basepullTimestampField` is never read. | An admin disables a table in the UI; the next FullReindex or Reconcile still indexes it. An admin edits `basepullTimestampField` on the row; basepull keeps using the annotation value. | `processes/FullReindexStep.java:90-96`; `processes/ReconcileIndexStep.java:98-106`; `processes/BasepullIndexStep.java:113` (read) vs `:207` (uses `tableConfig.getBasepullTimestampField()`) | Check `enabled` in all three steps (or make the field read-only and document). Pick one source of truth for the timestamp field and remove the other. | No (document) |
+| F13 | Medium | Basepull on a table whose config was removed but whose row and QQQ table still exist silently scans everything and indexes nothing. | `getTableConfig` returns null; the filter stays null (full scan), `buildDocument` is skipped for every row, the run completes `COMPLETED` with `recordsProcessed=N, recordsIndexed=0`, and `lastBasepullTime` advances. | `processes/BasepullIndexStep.java:184,202-209,236-240,268` | If `tableConfig == null`, mark the run FAILED (or the row `status=INACTIVE`) and skip. | No |
+| F14 | Medium | `ensureIndexRowExists` assumes a unique constraint on `tableName` that the produced metadata does not declare. | Two replicas run the first basepull together; both see no row, both insert; without a DB unique index there is no exception, the recheck is never reached, and the table now has two rows. Basepull then processes the table twice per cycle (two run records, two `lastBasepullTime` stamps); FullReindex and Reconcile use `get(0)` arbitrarily. | `processes/AbstractIndexingStep.java:143-195`; `QuickSearchQBitProducer.java:457-475` (no `withUniqueKey`) | Add `.withUniqueKey(new UniqueKey("tableName"))` to the table metadata so QQQ's pre-insert unique check fires, and ship DDL with a unique index. | Yes (metadata only) |
+| F15 | Medium | Per-table basepull failures are logged at WARN and the process still reports success. | OpenSearch rejects one table's bulk every run; the scheduler sees a green process forever. | `processes/BasepullIndexStep.java:124-132` | Collect failures and throw (or set an output value and a FAILED status on the step) after the loop; expose counts via `output.addValue` as Reconcile does. | Yes (small) |
+| F16 | Medium | `produce()` tolerates OpenSearch being unreachable and starts with `client=null, publisher=null`. | Misconfigured host/port in production: the app boots healthy, every write logs "publisher is null", every search throws "client is not initialized". Nothing in the operational tables records this. | `QuickSearchQBitProducer.java:159-167,172-182,187-195`; `customizers/*.java:74-79` | Add `failFastOnStartup` (default true) to the config, or at least retry client creation lazily on first use and surface a health check. | No (document) |
+| F17 | Medium | Static singleton context: re-produce leaks transports, and two QInstances in one JVM share one context. | `produce()` called twice (tests, qqq-all, hot reload): the first `ApacheHttpClient5Transport` and its IO reactor threads are never closed (only `clear()` nulls references; main code never calls `publisher.close()` or `client.close()` outside tests). A second QInstance overwrites `config/client/discoveredTables`, so instance A's customizers index into instance B's index. Calling `produce()` twice on the same QInstance throws from `addTable` after the static context has already been replaced. | `QuickSearchQBitContext.java:34-37,152-158`; `QuickSearchQBitProducer.java:200-215`; `qqq:QInstance.java addTable` | Close the previous client when replacing it; register a shutdown hook or expose `QuickSearchQBit.shutdown()`; document the one-QInstance-per-JVM constraint; longer term key the context by `QInstance` identity. | No (document; close-on-replace is trivial and worth doing) |
+| F18 | Medium | `String.format(recordLabelFormat, ...)` with `""` substituted for null throws on non-`%s` specifiers and poisons the whole batch. | `recordLabelFormat="%s #%d"` and `orderNumber` null: `IllegalFormatConversionException` (unchecked) escapes `buildDocument`; in `SynchronousIndexEventPublisher.publishIndexEvents` this happens before any bulk call, so every document in the batch (all tables) is dropped and the customizer logs one WARN. In the steps it fails the table's run. | `processes/IndexingUtils.java:194-200`; `publisher/SynchronousIndexEventPublisher.java:113-120` | Validate the format at `produce()` time against the field types; wrap per-record `buildDocument` in try/catch, count as a failure, continue. | No |
+| F19 | Medium | Indexed text is raw `String.valueOf`: PVS ids instead of labels, ISO instants, `true/false`, unbounded length, no HTML stripping. | A `status` PVS field indexes `3` not `Shipped`, so users cannot search by status label; a `notes` CLOB of 5 MB is sent whole (500 per bulk can exceed `http.max_content_length` and fail the batch with a 413 that surfaces as F10 in the steps). | `processes/IndexingUtils.java:94,150-155`; `qqq:ValueUtils.java:67-90` | Query with `shouldGenerateDisplayValues(true)` and `shouldTranslatePossibleValues(true)` and index `getDisplayValue(field)`; truncate each value to a configurable max (e.g. 32 KB); strip tags for `HTML` adorned fields. | No |
+| F20 | Low | Public `buildSearchableText` dereferences `includeLabels` without a null check. | Any external caller passing `null` for `includeLabels` gets an NPE; the table-config overload guards it, the 6-arg overload does not. | `processes/IndexingUtils.java:83-115` (`:106`) vs `:190` | Null-guard inside `buildSearchableText`. | No |
+| F21 | Low | Document ids are `sourceTable:recordId`, not namespaced by `tableNamePrefix` or instance. | Two apps pointed at the same `opensearchIndexName` with a table named `order` overwrite each other's documents, and either app's FullReindex/Reconcile deletes the other's documents (`deleteDocumentsForTable`, `deleteDocumentsIndexedBefore` filter on `sourceTable` only). Changing `tableNamePrefix` does not change ids (good: no orphan churn). A table name containing `:` can collide with another (table, id) pair only in contrived cases. | `opensearch/OpenSearchDocument.java:184-186`; `opensearch/QuickSearchOpenSearchClient.java:392-397,428-437` | Document one index per QInstance (validate at produce time if the index already holds a different app's marker document), or include an app/prefix token in `sourceTable`. | No |
+| F22 | Low | Reconcile's orphan detection depends on `indexedAt` written by whichever node indexed the document, using that node's clock. | Node B's clock is 3 minutes behind node A; node A starts reconcile at T; node B inserts a row at T+1 min and indexes it with `indexedAt = T-2 min`; reconcile deletes it as an orphan. Basepull repairs it only if its `modifyDate` (DB clock) is `> reconcileStart` and the row still exists. The comment at `:62-63` acknowledges the assumption. The orphan delete itself uses `delete_by_query` with `conflicts(Proceed)` and `refresh`, so there is no 10000-document cap and no scroll needed. | `processes/ReconcileIndexStep.java:142,192`; `opensearch/QuickSearchOpenSearchClient.java:420-445`; `processes/IndexingUtils.java:164` | Replace the time cutoff with a per-run token (`reconcileRunId` stored on each document; delete `sourceTable = X AND reconcileRunId != current`), or subtract a skew margin from the cutoff. | No |
+| F23 | Low | `RUNNING` run records are never swept; `completeRunRecord` failures are WARN-only. | JVM killed mid-run (or DB blip at completion): the row stays `RUNNING` indefinitely; operators cannot tell a hung run from a dead one. | `processes/AbstractIndexingStep.java:264-292,310-334` | On step start, mark `RUNNING` rows older than N hours as `ABANDONED`; or add a heartbeat column. | No |
+| F24 | Low | FullReindex aborts remaining tables on the first table failure; basepull continues. Inconsistent semantics. | A reindex-all with 5 tables fails on table 2; tables 3 to 5 are untouched and no run records exist for them. | `processes/FullReindexStep.java:90-96,215-220` vs `processes/BasepullIndexStep.java:124-132` | Pick one policy (continue and aggregate is friendlier for a reindex-all) and apply it to all three steps. | No |
+| F25 | Low | `normalizeSearchText` lowercases client-side even though both index and (intended) search analyzers lowercase; harmless but misleading once F06 is fixed. The highlight join and `totalHits` fallback are correct: `fragments` emptiness is checked before `String.join("...")`, and when `total()` is null `totalHits = results.size()` so `hasMore` is false. | None. | `actions/QuickSearchAction.java:76,102-110,123-124` | Leave the whitespace collapsing, drop the lowercase, or keep as belt-and-braces with a comment. | No |
+
+### Items checked and found acceptable
+
+- `ReconcileIndexStep` keyset paging works for non-numeric keys and gaps: both the `ORDER BY` and the `GREATER_THAN` comparison are evaluated by the database, so string collation is consistent; a null primary key fails fast (`processes/ReconcileIndexStep.java:178-185`). Not verified for `byte[]` (binary UUID) keys.
+- `ReconcileIndexStep` when the orphan delete throws after a successful re-index: run FAILED, index row untouched, documents are all current; the next run simply repeats. Acceptable.
+- `ReconcileIndexStep` race with a concurrent real-time update (stale page wins, F05) is repaired by the next basepull because `lastBasepullTime` is set to `reconcileStart` (`:274`), provided F04's overlap is in place.
+- `deleteDocuments` treats `not_found` as success (bulk delete items with status 404 carry no `error`), matching the comment at `opensearch/QuickSearchOpenSearchClient.java:283-284`.
+- `BulkIndexResult` accounting per item is correct; partial bulk application is reported, not hidden, on the batch path.
+- Post-delete correctly skips records with errors and null ids (`customizers/QuickSearchPostDeleteCustomizer.java:95-99`); post-update correctly re-reads the full row rather than indexing the sparse update (`QuickSearchPostUpdateCustomizer.java:132-155`), though through the open transaction (F01).
+
+## Missing tests
+
+Unit tests use `MemoryBackendModule` with a mocked `QuickSearchOpenSearchClient`; integration tests use the memory backend plus a Testcontainers OpenSearch. No test uses an RDBMS backend or a `QBackendTransaction`, so none of the transaction, ordering, or SQL-paging hazards are exercised.
+
+Covered today (for calibration): post-update sparse-record re-read (`QuickSearchPostUpdateCustomizerTest.testUpdateAction_partialUpdate_reindexesFullRecord`, `IndexDriftIntegrationTest.testPartialUpdate_unchangedFieldsStaySearchable`); publisher throws on item failures (`SynchronousIndexEventPublisherTest.testPublish*_itemFailures_throwsQException`); customizers swallow publisher exceptions (`*CustomizerTest.*_publisherThrows_returnsRecordsWithoutThrowing`, which is testing the F02 behaviour as desired); reconcile keyset paging, stale delete skipped on errors, run FAILED (`ReconcileIndexStepTest`, `IndexDriftIntegrationTest.testReconcile_repairsOrphanMissingAndStaleDocuments`); basepull due/not-due and FAILED on client throw (`BasepullIndexStepTest`); full reindex wipe-then-index ordering and FAILED on throw (`FullReindexStepTest`); search mapping, defaults, hasMore (`QuickSearchActionTest`); config-driven `recordLabelFormat` label (`ConfigDrivenIndexingIntegrationTest.testSearchResults_recordLabelFormatApplied`).
+
+Missing, by finding:
+
+1. F01: `QuickSearchPostInsertCustomizerTest` (or a new `TransactionIntegrationTest` against H2 via `qqq-backend-module-rdbms`): insert inside a `QBackendTransaction`, roll back, assert no `publishIndexEvents` call (today it will be called, so this test documents the defect until fixed). Same for update and delete.
+2. F02: `QuickSearchPostInsertCustomizerTest`: when the publisher throws, assert the returned records carry a warning (after the fix) or that a failure counter/dead-letter row was written; today's `*_publisherThrows_returnsRecordsWithoutThrowing` tests lock in the silent behaviour and should be inverted.
+3. F03: `FullReindexStepTest` and `BasepullIndexStepTest`: capture every `QueryInput` passed to the source table and assert each filter has an `ORDER BY` on the primary key and uses a keyset criterion (no `skip`). Integration variant: insert a row from another thread between pages (use a `Answer` on the mocked client to trigger the insert during `indexDocuments`) and assert every id was indexed exactly once.
+4. F04: `BasepullIndexStepTest.testLastBasepullTime_isCapturedBeforeQuerying`: insert a record with `modifyDate` between the step's start and end (same trick: insert from within the mocked `indexDocuments`), run twice, assert it is indexed on the second run. `testBasepullFilter_isGreaterThanOrEquals_withOverlap`: assert criterion operator and that the bound is `lastBasepullTime - overlap`.
+5. F05: `QuickSearchOpenSearchClientTest` or `OpenSearchIntegrationTest.testStaleWriteRejected`: index v2 of a document, then v1 with an older version, assert v2 survives (requires the external-version fix).
+6. F06: `OpenSearchIntegrationTest.testSearch_doesNotMatchOnSharedPrefixOnly`: index "stella" and "stone", search "stella", assert `totalHits == 1`. Also `testEnsureIndexExists_upgradesMappingWhenVersionChanges`.
+7. F07: `OpenSearchIntegrationTest.testSearch_tableWithIntegerAndDateSearchableFields_doesNotError`: add an `INTEGER` and a `DATE_TIME` field to a searchable table, reindex, search a word. `testIndex_dynamicMappingConflictOnFieldValues_reportedPerItem` for the first-value-wins mapping case.
+8. F08: `OpenSearchIntegrationTest.testSearchResults_annotationTable_haveRecordLabel`: assert `getRecordLabel()` is not null for the annotation-based `testEntity` table (currently it would fail). `IndexingUtilsTest.testBuildDocument_withoutFormat_usesTableRecordLabelFields`.
+9. F09: `FullReindexStepTest.testBulkItemFailures_runRecordHasErrorMessage` (currently `errorMessage` is null on that path). `QuickSearchOpenSearchClientTest`/integration: `deleteDocumentsForTable` proceeds on version conflicts.
+10. F10: `QuickSearchOpenSearchClientTest.testSearch_indexMissing_throwsQException` (start a container, delete the index, search); `QuickSearchActionTest.testOpenSearchException_isWrappedAsQException` using a mocked client that throws `OpenSearchException`.
+11. F11: `QuickSearchActionTest.testLimit_aboveMax_isClamped`, `testOffsetPlusLimit_aboveMaxResultWindow_isRejectedOrClamped`.
+12. F12: `FullReindexStepTest.testDisabledIndexRow_isSkipped`, `ReconcileIndexStepTest.testDisabledIndexRow_isSkipped`, `BasepullIndexStepTest.testRowTimestampField_isUsedOverAnnotation` (or the inverse, once a source of truth is chosen).
+13. F13: `BasepullIndexStepTest.testIndexRowWithoutTableConfig_runMarkedFailedAndNoScan`.
+14. F14: `QuickSearchQBitProducerTest.testIndexTable_hasUniqueKeyOnTableName`; `BasepullIndexStepTest.testEnsureIndexRowExists_concurrentFirstRun_createsOneRow` (two threads).
+15. F15: `BasepullIndexStepTest.testOneTableFails_stepReportsFailure` (assert the step throws or sets an output value after processing the other tables).
+16. F16: `QuickSearchQBitProducerTest.testProduce_openSearchUnreachable_failsFastWhenConfigured`.
+17. F17: `QuickSearchQBitProducerTest.testProduce_twice_closesPreviousClient` (spy on client), `testProduce_sameQInstanceTwice_doesNotLeaveContextHalfReplaced`.
+18. F18: `IndexingUtilsTest.testBuildDocument_recordLabelFormatWithNumericSpecifierAndNullValue_doesNotThrow`; `SynchronousIndexEventPublisherTest.testOneBadRecord_doesNotDropWholeBatch`.
+19. F19: `IndexingUtilsTest.testBuildSearchableText_possibleValueField_usesLabel`, `testBuildSearchableText_instantAndBigDecimal_formatting`, `testBuildSearchableText_veryLongValue_truncated`.
+20. F20: `IndexingUtilsTest.testBuildSearchableText_nullIncludeLabels_doesNotThrow`.
+21. F22: `ReconcileIndexStepTest.testDocumentIndexedByLaggingClock_duringRun_notDeleted` (requires the run-token design).
+22. F23: `AbstractIndexingStep` sweeper test: a `RUNNING` run older than the threshold is marked `ABANDONED` on the next step start.
+
+## Summary of blockers for 1.0
+
+Must fix or explicitly document before 1.0: F01 (pre-commit indexing), F02 (silent real-time failures), F03 (unordered paging), F04 (basepull window), F06 (search analyzer), F07 (non-text searchable fields break search), F08 (null record labels), F09 (errorMessage and conflicts on full reindex), F10 (unchecked OpenSearchException), F11 (limit/offset bounds), F14 (unique key on tableName), F15 (basepull hides failures). Everything else can ship with a documented limitation.

docs/audits/2026-10-04/README.md

@@ -0,0 +1,108 @@
+# Deep-dive audit: QBit Quick Search for 1.0 on QQQ 4.1
+
+Date: 2026-10-04. Code: `develop` `000e5bb` (identical to `feature/GH-6-repin-and-drift-fix`), 0.2.1-SNAPSHOT. References: QQQ `v4.1.0-RC.1`, opensearch-java 2.10.0, OpenSearch and AWS documentation as of this date. Five parallel code-grounded reviews; the raw reports with file:line evidence are the numbered files beside this one. Every claim below was spot-checked against source where it changed a decision; one agent claim (that Voyage does not use the QBit) was wrong and is corrected here.
+
+## Verdict
+
+The QBit compiles and passes its 233 unit and 12 integration tests on QQQ 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1 with no source change, so it is **API-compatible with QQQ 4.1 today**. It is **not feature complete or production-safe for a 1.0**: the index can hold uncommitted writes, failures are silent, basepull loses records and is never scheduled, search leaks data across tables, the search analyzer inflates matches, the OpenSearch client line is end-of-life, and only HTTP basic auth over an optional TLS connection is supported. None of these are hard; most are a few days each. The draft PRD covered about half of them. The PRD is updated alongside this audit.
+
+| Dimension | Score (1 to 10) | One line |
+|---|---|---|
+| QQQ 4.1 API compatibility | 9 | Builds and tests green on RC.1 (see section 1 for the contract gaps that remain) |
+| QQQ 4.1 QBit contract | 3 | Plain producer, no `QBitMetaData`, no schedule, no permission rules, static context, slot collisions |
+| Index trustworthiness | 3 | Pre-commit indexing, silent failures, lossy basepull window, unordered paging, wipe-first reindex |
+| Search quality and safety | 3 | No permissions, query terms edge-ngrammed, OR semantics, non-text fields break search, unbounded paging |
+| OpenSearch and AWS compatibility | 4 | 2.x self-managed and AWS FGAC basic auth work; 3.x needs a client bump; IAM SigV4 and Serverless unsupported |
+| Authentication and secrets | 4 | Basic only, plaintext credentials allowed, password on serializable metadata, no TLS trust config |
+| Operability | 4 | No timeouts, no health check, no failed-event capture, admin tables thin, processes without input screens |
+| Tests and docs | 5 | Good unit coverage of happy paths; JaCoCo gate inert; integration suite tests one word; README over-promises |
+
+## 1. QQQ 4.1 integration readiness
+
+Full detail with file:line evidence in `01-qqq-4.1-integration.md`. The QBit compiles and tests green on 4.1.0-RC.1, but it uses none of the 4.1 contract and misses three 4.1 APIs that solve its hardest problems. All three are absent from 4.0.0 (verified by tag), so a 1.0 that adopts them requires qqq 4.1 (decision D10).
+
+| Topic | Today | 4.1 answer | Refs |
+|---|---|---|---|
+| Producer contract | Plain class; `produce(QInstance)` adds tables, processes and app directly; no `QBitMetaData`; qqq-all stamps one by hand with version `0.2.1-SNAPSHOT` and its test asserts artifactId `quick-search` | `QBitMetaDataProducer<QuickSearchQBitConfig>` (3 of 4 sibling QBits use it); `getQBitMetaData()` with the real version from a Maven-filtered resource; component producers or `@QMetaDataProducingEntity`; registration in `postProduceActions`; keep `withConfig()` and a `produce(QInstance)` override that adds to the instance so qqq-all and Voyage keep compiling | 01 F-01, F-12 |
+| Real-time hook and transactions | Three per-table POST_* customizers; run inside the caller's open transaction before commit; `withCustomizer` throws `IllegalArgumentException` on an occupied slot | `RecordChangeListenerInterface` (instance-level, `appliesTo(table, type)`, covers INSERT/UPDATE/DELETE, cannot fail the write, carries old records and the transaction) registered with `QInstance.withRecordChangeListener`; publish through `QBackendTransaction.addAfterCommitCallback` when a transaction is present (discarded on rollback; `RDBMSTransaction` runs them after JDBC commit); reference implementation `qqq-esb` `EsbRecordChangeListener`. `MultiCustomizer` only if customizers are kept as a 4.0 fallback; instance-level `withTableCustomizer` is unsuitable because `DeleteAction` never consults it | 01 F-02, F-03, F-04 |
+| Scheduling | No `QScheduleMetaData` or `BasepullConfiguration`; `QApplicationLauncher` starts the scheduler only if some process has a schedule; validator requires repeat-or-cron plus an existing scheduler name | `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression` on the config; attach schedules only when `schedulerName` is set; keep the per-table `quickSearchIndex` watermark (core `BasepullConfiguration` is keyed per process, not per table) but anchor it at run start minus an overlap | 01 F-05 |
+| Permissions | No `PermissionsHelper` in search; no `QPermissionRules` on produced metadata (enricher default `NOT_PROTECTED`) | Mirror core `RecordSearchAction` (table READ filter, `QueryAction` for locks); `permissionRules` config defaulting to `HAS_ACCESS_PERMISSION` for processes and app, `READ_INSERT_EDIT_DELETE_PERMISSIONS` for the operational tables | 01 F-06 |
+| Validation timing | Backend lookup works for the qqq-all ordering; package scanning would instantiate the producer with a null config and NPE; `QBitConfig.validate` swallows thrown exceptions (only collected errors fail); `QInstanceValidator` has no qbit checks | `isEnabled()` false when config is null; keep list-based validate null-safe; a `QInstanceValidatorPluginInterface` for late checks (backend, searchable tables, scheduler) | 01 F-07 |
+| Runtime state and lifecycle | Static `QuickSearchQBitContext`; OpenSearch connect and index creation happen during metadata production; nothing closes the transport | `QRuntimeServiceInterface` (`start` connects and ensures the index, `stop` closes), registered with `qInstance.withRuntimeService`; config resolved from `SourceQBitAware.getSourceQBitConfig()` on produced tables and processes (the webhooks pattern); `startupMode` `DEGRADED` or `FAIL_FAST`. Note: qqq-all's `FullQBitsTest` builds the full instance with OpenSearch at `127.0.0.1:1`, so either `DEGRADED` stays the default or qqq-all's test changes | 01 F-08, F-09, F-10 |
+| Core search overlap | `RecordSearchExecutor` hard-wires `new RecordSearchAction()`; no SPI | Propose `QInstance.withRecordSearchProvider(QCodeReference)` plus `RecordSearchProviderInterface { appliesTo(table); search(input, table, term, limit) }`, consulted after core's permission check | 01 F-11 |
+| Build | JaCoCo `check` has no `prepare-agent` and no version; JUnit pinned 5.10.0 while parent manages `junit-bom` 6.0.1; parent 2.1.0 not yet tagged | Bind the agent with an explicit version; drop the JUnit pins; re-pin to parent 2.1.0 when released and retire the snapshot profile's BOM import | 01 F-13 |
+
+Ordered migration plan (from report 01): 1 contract and compat shims; 2 config surface; 3 record-change listener with after-commit publish; 4 runtime service and `SourceQBitAware` config resolution; 5 schedules, watermark, keyset paging; 6 permission-aware search; 7 build and parent 2.1.0 re-pin; 8 qqq-all follow-up (remove the manual `addQBit`, pass `withSchedulerName("demoScheduler")`) and the qqq SPI proposal.
+
+## Status (2026-10-04, branch feature/GH-1-0-release-readiness)
+
+All 25 blockers below are implemented on the release branch (commits 90cd09d through the admin-screen commit). Verification: 307 unit tests and 18 Testcontainers integration tests green against qqq 4.1.0-RC.1 and OpenSearch 2.19.6, JaCoCo 85% instruction / 96% class. Remaining before the 1.0.0 tag: parent 2.1.0 re-pin when published, qqq-all follow-up (remove its manual `addQBit`, set `schedulerName`, `startupMode DEGRADED` for its test), CI image matrix job, and the qqq core search SPI proposal (1.1).
+
+## 2. Blockers for 1.0, deduplicated across reports
+
+Severity: Critical = wrong or leaked data; High = breaks a target platform or a core promise; Medium = operational risk. IDs reference the raw reports (A = auth, C = compatibility, F = correctness, G = features).
+
+| # | Sev | Blocker | Evidence | Fix in brief | Refs |
+|---|---|---|---|---|---|
+| B1 | Critical | Real-time indexing runs inside the host's open transaction, before commit; a rollback leaves a ghost document, and post-update re-reads uncommitted values through the same transaction (javalin single-record endpoints pass no transaction and autocommit, so this bites ETL, bulk and transactional host code) | `customizers/*` :97, :114, :151, :109; qqq `InsertAction.java:286`, `RDBMSTransaction.java:55,87`, `StreamedETLExecuteStep.java:204-218` | Replace the customizers with a `RecordChangeListenerInterface` (4.1) and publish through `QBackendTransaction.addAfterCommitCallback` (4.1, discarded on rollback) when a transaction is present; publish immediately otherwise | F01, 01 F-02, F-03 |
+| B2 | Critical | Real-time indexing failures are fully silent: customizers catch everything and log WARN; nothing reaches the host records or any table | `customizers/*` :99-120; `SynchronousIndexEventPublisher.java:160-171` | The record-change listener cannot fail the write by design; record failed (table, id, action) rows in a failed-event table that the scheduled run replays, and surface counts on `quickSearchIndex` | F02, G06 |
+| B3 | Critical | Search returns labels and highlight snippets from every indexed table with no session, table-permission or record-lock check; the `tableName` filter is caller-controlled | `actions/QuickSearchAction.java:81-121`; `QuickSearchOpenSearchClient.java:481-487` | `QuickSearchInput extends AbstractActionInput`; filter tables with `PermissionsHelper.hasTablePermission(READ)` as core `RecordSearchAction.java:110` does; `terms` filter on `sourceTable`; re-read hits via `QueryAction` by primary key for record locks | A1, G02 |
+| B4 | High | Basepull and full reindex page with limit/skip and no order-by; concurrent writes skip or duplicate rows, and full reindex wiped first so skipped rows vanish | `AbstractIndexingStep.java:358-359`; qqq `RDBMSQueryAction.java:340-354` | Reuse the keyset loop already in `ReconcileIndexStep.java:222-238` | F03, G12 |
+| B5 | High | Basepull watermark is `Instant.now()` at the end of the run with a strict greater-than, so rows modified during the run or in the boundary second are lost forever | `BasepullIndexStep.java:198-209, 268-279` | Capture run start before the first query, persist that minus an overlap, use greater-than-or-equal; reconcile already does this at `:142, :274` | F04, G08 |
+| B6 | High | Basepull and reconcile are never scheduled; README calls basepull "scheduled" | no `withSchedule` anywhere; `QuickSearchQBitProducer.java:510-518` | `QScheduleMetaData` on both processes with a `schedulerName` config (decision D4) | G03 |
+| B7 | High | `searchableText` has the edge-ngram analyzer but no `search_analyzer`, so query terms are n-grammed too: "stella" matches anything starting with "st"; `multi_match` defaults to OR with no `minimum_should_match` | `QuickSearchOpenSearchClient.java:177-190, 472-475` | Add a standard+lowercase search analyzer, `operator=AND` or `minimum_should_match`; mapping change, so bundle with the index-versioning work | F06, C5, G04 |
+| B8 | High | `fieldValues` is one dynamic object shared by all tables with raw typed values; same-named fields of different types cause `mapper_parsing_exception`, and boost clauses without `lenient` make a text term against a numeric field fail the whole search | `QuickSearchOpenSearchClient.java:192, 536-561`; `IndexingUtils.java:150-155` | `dynamic_templates` mapping `fieldValues.*` to text, `date_detection: false`, stringify values, `lenient(true)` | F07, C6, G09 |
+| B9 | High | `recordLabel` is null for every annotation-configured table because no source query sets `shouldGenerateDisplayValues` and insert output has no label; PVS fields index ids, not labels | `IndexingUtils.java:94, 142`; qqq `QueryAction.java:503-506` | Fall back to `QValueFormatter.formatRecordLabel`; query with display values and PVS translation | F08, F19, G19 |
+| B10 | High | Full reindex wipes the table's documents first; a mid-run failure leaves a partial index with no recovery; delete-by-query uses `conflicts=abort`; FAILED runs from the bulk path have a null error message | `FullReindexStep.java:152, 160-185, 210`; `QuickSearchOpenSearchClient.java:388-403` | Build into a fresh index and swap an alias (decision D5) or reuse the reconcile algorithm; `conflicts(Proceed)`; pass the first bulk error into the run record | F09, G12 |
+| B11 | High | No TLS trust configuration (no CA, trust store, or guarded insecure flag); `useSsl` defaults false and basic credentials over plain HTTP pass validation | `QuickSearchOpenSearchClient.java:98-139`; `QuickSearchQBitConfig.java:54, 103-112` | `tls` block (CA path, trust store, hostname verification, guarded `insecureSkipVerify`); reject credentials over http unless `allowPlaintextCredentials`; prefer a single `opensearchUrl` defaulting to https | C1, A2, A5 |
+| B12 | High | Password is a plain String on a Serializable config that hosts attach to `QBitMetaData`; `QInstanceAdapter.qInstanceToJson` would emit it | `QuickSearchQBitConfig.java:52-53`; qqq-all `DemoQBits.java:162-163` | Interpret `${env.X}` references with `QMetaDataVariableInterpreter` at client construction (the RDBMS/Mongo/S3 backend pattern); `@JsonIgnore` secret getters; redacting `toString()` | A3 |
+| B13 | High | opensearch-java pinned at 2.10.0, the end-of-life 2.x line documented for OpenSearch 1.x to 2.x; AWS already offers 3.1 to 3.5 | `pom.xml:24`; compatibility matrix | Bump to 3.10.0; the only compile break is `.refresh(true)` to `.refresh(Refresh.True)` at `QuickSearchOpenSearchClient.java:427` (verified: 233 tests pass after) | C2 |
+| B14 | High | Startup is best-effort: an unreachable cluster leaves `client=null`, the host boots healthy, every write logs and every search throws; connect and index creation are side effects inside metadata production | `QuickSearchQBitProducer.java:159-195` | Move connect, `ensureIndexExists` and close into a `QRuntimeServiceInterface` (4.1) with `startupMode` `FAIL_FAST` or `DEGRADED`; qqq-all's `FullQBitsTest` currently relies on degraded behaviour, so the default is a decision (PRD D11) | F16, G05, 01 F-09, F-10 |
+| B15 | Medium | No connect/response timeouts or pool sizing exposed; opensearch-java defaults are 1 s connect and 30 s response; a hung node blocks every host write | `QuickSearchOpenSearchClient.java:105-139` | `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `User-Agent`; multi-host list | C9, A7, G01 |
+| B16 | Medium | Processes, operational tables and the admin app declare no `QPermissionRules`, so they inherit the instance default (`NOT_PROTECTED` unless the host tightened it); anyone can trigger a full reindex | `QuickSearchQBitProducer.java:457-549` | `HAS_ACCESS_PERMISSION` on processes and app with base name `quickSearchAdmin`; `READ_WRITE_PERMISSIONS` on tables; overridable | A6 |
+| B17 | Medium | `limit` and `offset` unbounded; `from + size > 10000` fails with a raw error; `totalHits` silently capped at 10000; no term length bounds | `QuickSearchAction.java:67-79, 123-124` | Clamp limit (100) and window (10000), bound term length (2 to 100) matching core, `track_total_hits` bounded with a lower-bound flag | F11, C7, A11, G18, G24 |
+| B18 | Medium | `OpenSearchException` is unchecked and bypasses every `catch(IOException)`, so search callers get a raw RuntimeException | `QuickSearchOpenSearchClient.java` all client methods; `QuickSearchAction.java:91` | Catch `OpenSearchException` and wrap in `QException` | F10 |
+| B19 | Medium | `quickSearchIndex.tableName` has no unique key, so concurrent first runs on two replicas create duplicate rows and the table is processed twice per cycle | `AbstractIndexingStep.java:143-195`; `QuickSearchQBitProducer.java:457-475` | `withUniqueKey(new UniqueKey("tableName"))` plus DDL note | F14 |
+| B20 | Medium | Per-table basepull failures are logged at WARN and the process reports success forever | `BasepullIndexStep.java:124-132` | Aggregate failures and fail the step or set a FAILED status | F15 |
+| B21 | Medium | `quickSearchIndex.enabled` is honoured only by basepull; customizers, full reindex, reconcile and search ignore it | `BasepullIndexStep.java:89-92` only | Consult the row (cached) in all paths; exclude disabled tables from search | F12, G07 |
+| B22 | Medium | Integration tests pin OpenSearch 2.11.0 (2023) and search one whole word; nothing covers 2.19.x or 3.x, typeahead, multi-word, boosts, basepull, scheduling or permissions; JaCoCo check is skipped (no `prepare-agent`) | `*IntegrationTest.java:76-85`; `pom.xml:188-212` | Parameterize the image, run 2.19.6 and 3.9.0 in CI, bind the JaCoCo agent, add the tests listed in reports 04 and 05 | C10, G32 |
+| B23 | Medium | No orphan sweep for tables removed from config or renamed; no detection that config or mapping drifted from the live index (`ensureIndexExists` returns early if the index exists) | `ReconcileIndexStep.java:98`; `QuickSearchOpenSearchClient.java:167-171`; `AbstractIndexingStep.java:167-175` | Reconcile deletes `sourceTable` not in the discovered set; compare config to `searchableFieldsJson` and set `NEEDS_REINDEX`; verify analyzer settings on an existing index | G14, G15 |
+| B24 | Medium | Validation never checks that searchable fields or the basepull timestamp field exist on the table; tables without `modifyDate` fail at run time, caught and logged | `QuickSearchQBitConfig.java:73-157` | Fail fast at produce; allow `basepullTimestampField = null` meaning reconcile-only | G13 |
+| B25 | Medium | Processes have no input screen, so `tableName` cannot be supplied from the admin UI; `enableScheduledProcesses=false` also removes the manual full reindex and reconcile | `QuickSearchQBitProducer.java:228-232, 510-549` | Frontend step with an optional table PVS; split the flag | G17, G20 |
+
+Also from report 01: drop the local JUnit 5.10.0 pins in favour of the parent's `junit-bom` 6.0.1 (01 F-13); add `getDefaultBackendNameForTables()` and a settable `tableMetaDataCustomizer` on the config so hosts can set RDBMS backend details the way qqq-all does for the other QBits (01 answer 1); use the `QBitMetaData.namespace` as the default `tableNamePrefix` (01 F-12).
+
+Non-blocking but recommended before 1.0: external document versioning from `modifyDate` so stale writes are rejected (F05); failed-event replay plus bounded retry on bulk 429/5xx (G06); byte-size cap on bulk batches for the AWS 10 MiB payload limit on small instances (C8); close the client and publisher on shutdown and on re-produce (A8, F17); `IndexEvent` needs `occurredAt` and a version if the publisher SPI is to be called stable (G26); run-history purge (G21); admin table fields `lastRunStatus`, `lastErrorMessage`, `documentCount`, `lastReconcileTime` (G22); health method (G23); per-field `maxLength` (G25); mark `indexEventPublisher` transient on the Serializable config (A10).
+
+## 3. OpenSearch and AWS compatibility matrix
+
+| Target | Works today | What breaks | Needed |
+|---|---|---|---|
+| OpenSearch 2.x self-managed, security disabled | Yes | Nothing; the only tested configuration (image 2.11.0) | Test 2.19.x too |
+| OpenSearch 2.x self-managed, security plugin, basic auth over TLS | Partly | Demo or private-CA certificates fail the JVM trust check; basic auth only | B11; documented least-privilege role |
+| OpenSearch 3.x self-managed | No (client line) | opensearch-java 2.x is documented for 1.x to 2.x; server negotiates HTTP/2 | B13; add 3.x to the IT matrix |
+| AWS managed domain 2.x or 3.x, FGAC, internal-database master user, basic auth | Yes for 2.x, 3.x after B13 | Needs port 443, `useSsl`, and an access policy that does not require signed requests; 10 MiB payload limit on t3 and `*.large` | B13, byte-capped bulks, README guidance |
+| AWS managed domain with IAM master user or IAM principals in the access policy (SigV4) | No | Unsigned requests rejected; no `AwsSdk2Transport`, region or service name | `authMode=AWS_SIGV4` with `AwsSdk2Transport`, service `es`, default credential chain, optional assume-role; declare `software.amazon.awssdk:aws-crt-client` and `auth` as optional dependencies (decision D6) |
+| Amazon OpenSearch Serverless | No | SigV4 service `aoss` plus `x-amz-content-sha256`; `_delete_by_query` and `_refresh` are not in the AOSS supported-operations table (used by full reindex and reconcile); custom `_id` only on Search collections; refresh lag about 10 s; client must be 3.x | Declare unsupported for 1.0; later: SigV4, replace delete-by-query with search plus bulk delete, replace refresh with a bounded wait |
+| Legacy Elasticsearch 7.10 domains, OpenSearch 1.x | Untested | Client matrix lists only OpenSearch 1.x to 3.x; renamed APIs | Out of scope |
+
+Recommended 1.0 supported-platforms statement: self-managed OpenSearch 2.19.x and 3.x latest minor over HTTP or HTTPS with the security plugin disabled or HTTP basic auth; Amazon OpenSearch Service managed domains 2.x and 3.x with fine-grained access control, an internal-database master user, basic auth, port 443 and TLS. Not supported in 1.0: IAM SigV4 (unless D6 says otherwise), Serverless, Elasticsearch endpoints, OpenSearch 1.x. Least-privilege role on the one index: `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*` and an `indices:admin/*` read for the exists check, and the cluster-level `cluster_composite_ops` group for `_bulk`. Sources are listed in report 02.
+
+## 4. Authentication options that make sense for 1.0
+
+Today: none, or HTTP basic. Recommended minimum: an `authMode` enum with `NONE`, `BASIC`, `AWS_SIGV4` (`awsRegion`, `awsServiceName` default `es`, optional `awsAssumeRoleArn`), plus one escape hatch, a `transportCustomizer` `QCodeReference` to an interface with `customizeTransport(ApacheHttpClient5TransportBuilder)` and `customizeHttpClient(HttpAsyncClientBuilder)`, which lets hosts add bearer or JWT tokens, API-key headers and mTLS without a fork. Do not add `BEARER`, `API_KEY` or `MTLS` enum values in 1.0; each needs a token-refresh or key-store story the SPI defers. All secret-bearing strings are resolved with `QMetaDataVariableInterpreter` inside the client factory so hosts write `"${env.OPENSEARCH_PASSWORD}"` and metadata never holds the value. Validation: BASIC needs both username and password; AWS_SIGV4 needs a resolvable region and forbids username/password; credentials over http rejected unless `allowPlaintextCredentials`; `insecureSkipVerify` refused for non-loopback hosts unless a second explicit flag is set, with a WARN at startup. Full field sketch with defaults is in report 03, section "Recommended 1.0 auth configuration". The deprecation path keeps `withOpensearchHost/Port/UseSsl/Username/Password` working and infers `BASIC` when a username is set.
+
+Supply chain: no concrete CVE in the resolved set (httpclient5 5.6.3, jackson 2.21.5, log4j 2.25.5). qqq-backend-core still pulls AWS SDK for Java v1 1.12.797, which reached end of support 2025-12-31; a qqq concern, and this QBit should use SDK v2 only.
+
+## 5. Feature completeness versus QQQ 4.1 core search
+
+QQQ 4.1 added `RecordSearchAction`, `QTableMetaData.searchFields` and `POST /qqq/v1/search`, consumed by the Next UI. Core has: table READ permission and capability checks, field personalisation (hidden and masked fields excluded), record locks via `QueryAction`, `limitPerTable` (5 default, 25 max), a `tableNames` list, a 100-character term cap, a 30 s timeout, `tableLabel`, typed `recordId`, label fallback and display values. Quick Search has: relevance scoring with weights, typeahead (defective, B7), highlights, offset pagination with totals, cross-table ranking, OLTP isolation, run history and the publisher SPI. To back the Next UI (PRD E4) Quick Search must first accept `tableNames` and `limitPerTable` and return `tableLabel` and a typed `recordId` (G16), and it should default its field list from `searchFields` so hosts do not maintain two lists. Core has no provider SPI; see decision D1.
+
+## 6. Consumers (verified)
+
+- qqq-all `DemoQBits.java:151-163` (origin/develop): config-driven `customer` with `name` and `email`, hand-registered `QBitMetaData` at `0.2.1-SNAPSHOT`, no `schedulerName`, `OPENSEARCH_SSL` defaults false. Its `FullQBitsTest` casts the config back out of `QInstance.getQBits()`.
+- Voyage `pom.xml:249-253` (origin/develop): pins `qbit-quick-search:0.2.0` from GitHub Packages and produces the QBit in `OrderAppMetaDataProvider.produceQuickSearchQBit` for `crmContact` and `crmCompany` when `OPENSEARCH_HOST` is set. Report 04's statement that Voyage has no reference was based on a stale local checkout and is wrong.
+
+## 7. Missing tests (consolidated)
+
+Reports 04 and 05 list them by class and scenario. The themes: a transaction test against H2 through `qqq-backend-module-rdbms` (insert, roll back, assert no publish); inverted customizer tests (publisher throws, records carry a warning or a failed-event row exists); keyset paging assertions on every source query; watermark captured before querying with an overlap; stale-write rejection once versioning lands; "stella" does not match "stone"; integer and date searchable fields do not break search; annotation tables have a record label; full-reindex FAILED run has an error message; `OpenSearchException` wrapped; limit and window clamps; disabled rows skipped everywhere; unique key on `tableName`; one failing table fails the basepull step; produce twice closes the previous client; recordLabelFormat with a null numeric; PVS label indexing; permission denial; string primary keys through reconcile; OpenSearch down during a host insert; pre-existing index with a different mapping; two tables sharing a field name with different types; image matrix 2.19.6 and 3.9.0.

docs/initiatives/quick-search-1.0.md

@@ -0,0 +1,174 @@
+---
+tracker_key: "QRun-IO/qbit-quick-search#12"
+project: qbit-quick-search
+status: released-rc
+updated: 2026-10-09
+---
+
+# Initiative: QBit Quick Search 1.0 on QQQ 4.1
+
+> Stakeholder-readable PRD. This markdown is canonical; the tracker mirrors it. Scope claims are grounded in the code at `develop` (`000e5bb`) and in QQQ `v4.1.0-RC.1`, and in the deep-dive audit of 2026-10-04 (`docs/audits/2026-10-04/README.md`, blockers B1 to B25).
+
+## Status
+
+`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). `1.0.0-RC.2` followed on 2026-10-06 with the review follow-ups #13 to #17. Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
+
+## Executive Summary
+
+QBit Quick Search gives QQQ applications OpenSearch-backed full-text search across tables. It is at 0.2.1-SNAPSHOT, published publicly as 0.2.0, pinned by Voyage and bundled by `qqq-all`. The July 2026 audit found the index could not be trusted to converge with the source database. PR #7 (merged 2026-09-26) fixed update clobbering, silent bulk failures and added a reconcile process. The 2026-10-04 deep-dive confirmed the QBit builds and passes all tests on QQQ 4.1.0-RC.1 unchanged, and found 25 remaining blockers across trust, security, search quality, platform compatibility and operability.
+
+1.0 means three things. First, a standard QQQ 4.1 QBit: built on the 4.1 build parent, registered through `QBitMetaDataProducer`, scheduled through QQQ's scheduler, composable with other QBits on the same tables, fail-fast at startup. Second, a trustworthy index: no uncommitted or lost writes, no silent failures, deterministic batch jobs, a reindex that keeps serving results, and search that honours QQQ permissions. Third, a credible platform story: current OpenSearch client, self-managed 2.x and 3.x plus Amazon OpenSearch Service managed domains, TLS trust configuration, secrets by reference, and an authentication model that covers basic auth and, if decision D6 says so, AWS IAM SigV4.
+
+The 4.1 release train does not wait for 1.0. A compatibility-only 0.3.0 rides the proposed `qbit-build-parent` 2.1.0 chain (qqq#921). 1.0 follows on the same parent once the epics below land.
+
+## Business Objective
+
+Ship a search QBit that QQQ adopters (qqq-all demo users, Voyage, future Business Platform tenants) can run in production without a reconciliation babysitter, against the OpenSearch deployments they actually have, and that the Next UI can expose directly.
+
+## Problem Statement
+
+Current state at `develop` as of 2026-10-04:
+
+| Area | State | Evidence |
+|---|---|---|
+| Build | Parent `qbit-build-parent:2.0.0` (qqq 4.0.0). Opt-in `-Pqqq-snapshot` profile. | `pom.xml` |
+| 4.1 compatibility | 233 unit tests pass on 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1; `mvn verify` on RC.1 with Docker passes all 12 integration tests. No source change needed. | surefire and failsafe reports |
+| Coverage gate | JaCoCo `check` is skipped (no `prepare-agent`), so the 0.70 / 0.90 thresholds never run. | `mvn verify` output |
+| Open PRs | #8 (orb 0.6.8, tag-only GA publish, Apache checkstyle, AssertJ 3.27.7): CI green, awaiting review. #1 superseded by #8. | GitHub |
+| Release chain | QQQ 4.1.0-RC.1 published 2026-10-02. Parent 2.1.0-RC.1 and quick-search 0.3.0-RC.1 proposed in qqq#921; qqq-all BOM pins 0.2.1-SNAPSHOT. | qqq#921, qbit-bom#4, qqq-all#10 |
+| Index trust | Real-time indexing runs inside the host transaction before commit (B1); failures are swallowed with no record (B2); basepull pages without order-by (B4) and loses rows modified during a run (B5); basepull and reconcile are never scheduled (B6); full reindex wipes first (B10); no unique key on `quickSearchIndex.tableName` (B19). | audit B1, B2, B4, B5, B6, B10, B19 |
+| Search quality | Query terms are edge-ngrammed because no search analyzer is set, and multi-match defaults to OR (B7); shared dynamic `fieldValues` object breaks on mixed field types and non-lenient boosts (B8); record labels null for annotation tables and PVS ids indexed instead of labels (B9); unbounded paging and term length (B17). | audit B7, B8, B9, B17 |
+| Security | Search has no permission or session checks (B3); no permission rules on processes, tables or app (B16); credentials allowed over plain HTTP and no TLS trust configuration (B11); password stored on serializable metadata (B12). | audit B3, B11, B12, B16 |
+| Platform | opensearch-java 2.10.0 is the end-of-life 2.x line (B13); IAM SigV4 and Serverless unsupported; no timeouts (B15); integration tests pin OpenSearch 2.11.0 only (B22). | audit section 3 |
+| QBit contract | Plain producer, no `QBitMetaData` (qqq-all hand-registers with a hard-coded version), no `sourceQBitName`, static context, `withCustomizer` throws on occupied slots, best-effort startup (B14). | `QuickSearchQBitProducer`, qqq-all `DemoQBits.java:161` |
+| Core overlap | QQQ 4.1 `RecordSearchAction`, `QTableMetaData.searchFields` and `POST /qqq/v1/search` back the Next UI with permissions and no provider SPI; Quick Search has no UI surface and cannot yet satisfy `tableNames` or `limitPerTable`. | qqq commit `7dd36c8ad`; audit section 5 |
+| Docs | README shows a `0.2.0-SNAPSHOT` dependency, promises a scheduled basepull, admin-UI per-table reindex, "field label" prefixes, enforced coverage and qbit-crm/wms integrations, none of which hold. | `README.md` |
+
+Doing nothing leaves qqq-all and Voyage shipping a search feature whose basepull never runs, whose index can contain rolled-back and lost writes, whose results leak across tables regardless of permissions, and whose reindex blanks search.
+
+## Vision / Desired Future State
+
+A host adds the dependency, configures the QBit like any other 4.1 QBit with secrets by reference, points it at a self-managed or AWS-managed OpenSearch cluster over TLS, and gets: real-time indexing that reflects committed data and survives slot collisions, a scheduled catch-up and reconcile with a replayed failed-event log, a reindex that keeps serving results, a search API that returns only what the session may read, and a Next UI search box backed by OpenSearch when the QBit is present.
+
+## Scope
+
+### In Scope
+
+- Released 4.1 build parent; CI matrix on the 4.1 GA line plus the next snapshot; Docker-backed integration tests on OpenSearch 2.19.x and 3.x; JaCoCo gate actually enforced.
+- Standard QBit contract: `QBitMetaDataProducer`, self-registered `QBitMetaData` with the real artifact version, namespace support, `sourceQBitName`, fail-fast validation including field existence, explicit permission rules, instance-scoped runtime state, composable customizers.
+- Index trust: post-commit indexing or a documented contract, failed-event capture and replay, keyset paging everywhere, start-anchored basepull watermark with overlap, scheduling through `QScheduleMetaData`, blackout-free full reindex, unique key on the index table, orphan and drift detection, external document versioning.
+- Search quality and safety: search analyzer, AND semantics, typed-safe `fieldValues` mapping, display values and labels, permission-aware results, bounded paging and term length, wrapped OpenSearch exceptions, `tableNames` and `limitPerTable` inputs, `tableLabel` and typed `recordId` outputs.
+- Connection security: opensearch-java 3.x, `opensearchUrl`, TLS trust block, `authMode` (`NONE`, `BASIC`, and `AWS_SIGV4` per D6), secrets by `${env.}` reference, redaction, timeouts and pool sizing, transport customizer SPI, client lifecycle.
+- Operability and docs: process input screens, admin table status fields, run-history purge, health method, truthful README with a supported-platforms statement and least-privilege roles, configuration reference, migration notes from 0.x, CHANGELOG.
+
+### Out of Scope
+
+- Asynchronous publisher implementations (RabbitMQ, Kafka, ESB). The `IndexEventPublisher` SPI stays and gets `occurredAt` and a version so an async implementation can be correct, but it is marked experimental in 1.0.
+- Amazon OpenSearch Serverless (needs SigV4 `aoss`, and lacks `_delete_by_query` and `_refresh`). Legacy Elasticsearch endpoints and OpenSearch 1.x.
+- Bearer, JWT, API-key and mTLS as first-class `authMode` values (available through the transport customizer SPI).
+- Record-level security beyond the `QueryAction` post-filter (D3). Index-time security keys.
+- Multi-index layouts, composite primary keys, synonyms, fuzzy matching, phrase queries, language analyzers, sort options.
+- `com.kingsrook` to `io.qrun` groupId migration (ADR-0007). qbit-crm / qbit-wms companion integrations.
+- Model-assisted table routing or re-ranking (TypeSafe Jev); candidates in `docs/reference/typesafe-jev-candidates.md` for 1.x.
+
+## Success Criteria
+
+- `com.kingsrook.qbits:qbit-quick-search:1.0.0` on Maven Central, parent importing the released qqq 4.1 BOM, Apache-2.0 metadata, opensearch-java 3.x.
+- CI runs the Testcontainers suite on OpenSearch 2.19.x and 3.x latest minor with no silent skips, on 4.1 GA and the next snapshot; JaCoCo thresholds enforced.
+- qqq-all `full` profile and Voyage pin 1.0.0; qqq-all's smoke finds a seeded customer through the Next UI search box (or through the API if D1's SPI slips).
+- A session without read permission on a table gets zero results from that table; a session with a record security lock sees only its rows. Both covered by tests.
+- A rolled-back insert produces no document (or, under the documented alternative, is removed by the next scheduled reconcile). Covered by an H2 transaction test.
+- Full reindex of a seeded table returns non-empty results at every point during the run. Covered by an integration test.
+- Basepull and reconcile fire on schedule in a host with a configured scheduler; a row modified during a basepull run is indexed by the next run. Covered by tests.
+- "stella" does not match "stone"; a table with an integer and a date searchable field searches without error. Covered by integration tests.
+- Basic auth over TLS against a cluster with a private CA works with the `tls` block; credentials over plain HTTP are rejected unless explicitly allowed. Covered by a secured Testcontainers test.
+- Every blocker B1 to B25 in the 2026-10-04 audit is closed or explicitly deferred with a reason in this doc.
+
+## Key Stakeholders
+
+| Role | Person | Responsibility |
+|---|---|---|
+| Sponsor / Product Owner | James Maes | Decisions D1 to D11, release approval |
+| Engineering Lead | James Maes (with Claude Code) | Build, review, release |
+| Downstream consumers | qqq-all (QRun-IO/qqq-all#10); Voyage (pins 0.2.0 from GitHub Packages, `OrderAppMetaDataProvider.produceQuickSearchQBit`) | Pin and verify 1.0.0 |
+
+## Initiative-level Constraints
+
+- Parent `qbit-build-parent` is the only source of the qqq version (ADR-0007). No child BOM imports outside the opt-in snapshot profile. httpclient5 and Jackson stay BOM-managed.
+- Apache-2.0 everywhere; Central artifacts are immutable, so metadata fixes ship as new versions.
+- 3-space indentation, flower-box Javadoc, fluent `withX()` setters, wrapper types (CODE_STYLE.md).
+- JaCoCo gates stay at 0.70 instruction / 0.90 class, and must actually run.
+- Source compatibility for the fluent config setters and `QuickSearchAction` input/output used by qqq-all and Voyage (`withOpensearchHost/Port/UseSsl/Username/Password`, `withSearchableTable`, `withSearchableEntityClasses`, `QuickSearchQBitProducer().withConfig().produce(qInstance)`). New fields are additive; deprecated fields keep working through 1.x.
+- No secrets in metadata objects, logs, or the frontend `environmentValues` (never name an OpenSearch secret `QQQ_ENV_*`).
+- Any mapping change (search analyzer, `fieldValues` templates) lands before the first 1.0 index is created, with an index-version marker so an existing 0.x index is detected and rebuilt rather than reused.
+
+## Assumptions
+
+- QQQ 4.1.0 GA and `qbit-build-parent` 2.1.0 publish before the 1.0 release; work proceeds on `4.1.0-RC.1` via the snapshot profile.
+- OpenSearch 2.19.x and 3.x are the supported server lines; opensearch-java 3.10.0 is the client (its only compile break here is `.refresh(true)` to `.refresh(Refresh.True)`).
+- Hosts that want scheduling already configure a QQQ scheduler; the QBit does not provide one.
+- QQQ 4.1 provides `RecordChangeListenerInterface`, `QBackendTransaction.addAfterCommitCallback` and `QRuntimeServiceInterface`, none of which exist in 4.0.0 (verified by tag). 1.0 uses them and therefore requires qqq 4.1 (D10).
+
+## Risks
+
+| Risk | Mitigation |
+|---|---|
+| Core `RecordSearchAction` has no SPI, so Next UI integration needs a qqq core change | Raise the SPI as a qqq issue early (D1); 1.0 can ship API-only, UI integration in 1.1 |
+| Adopting 4.1-only APIs drops 4.0 hosts | Voyage and qqq-all both move to 4.1; 0.3.0 remains the last 4.0-compatible line (D10) |
+| After-commit publishing depends on the backend transaction honouring the callback contract | RDBMS does at the 4.1 tag; verify Mongo and other backends in the integration matrix; flush immediately when no transaction is passed; scheduled reconcile is the backstop |
+| Removing the static context changes how custom publishers and process steps obtain the client | Keep static accessors as deprecated delegates to the instance-scoped holder for one minor version |
+| Permission post-filtering reduces page sizes unpredictably | Over-fetch (limit times two, capped) and filter down; document that `totalHits` is an upper bound |
+| Alias-swap reindex needs index-create and alias privileges on the cluster | Fall back to in-place reconcile when the configured user lacks the privilege; document required roles |
+| opensearch-java 3.x bump changes transport behaviour against 2.x servers (HTTP/2 negotiation, URL encoding) | CI matrix on 2.19.6 and 3.9.0; keep the content-compression workaround until proven unnecessary |
+| SigV4 pulls AWS SDK v2 artifacts into a QBit | Declare them `optional`; load the AWS transport reflectively or in a separate class only when `authMode=AWS_SIGV4` |
+
+## Dependencies
+
+- qqq 4.1.0 GA (qqq#798, #921), `qbit-build-parent` 2.1.0 (qbit-bom#4), qqq-orb 0.6.8 (PR #8).
+- CircleCI executor with Docker for the integration suite (orb capability to confirm).
+- qqq core change for a record-search provider SPI (new issue, D1).
+- Optional: `software.amazon.awssdk:aws-crt-client` and `auth` for SigV4 (D6).
+
+## Proposed Epics
+
+Estimates are engineer-days with AI-augmented throughput. E1 is foundational and gets broken into stories first. Blocker IDs reference `docs/audits/2026-10-04/README.md`.
+
+### Epic 1: 4.1 platform and QBit contract (foundational, ~7 days)
+Purpose: make the QBit a standard 4.1 citizen, following the ordered migration plan in audit report 01. `QBitMetaDataProducer<QuickSearchQBitConfig>` with `getQBitMetaData()` (groupId `com.kingsrook.qbits`, artifactId `quick-search` which qqq-all asserts, version from a Maven-filtered resource, namespace, config); tables, processes and app as package-scanned component producers or `@QMetaDataProducingEntity`; registration and bootstrap in `postProduceActions`; `withConfig()` and a `produce(QInstance)` override that adds to the instance kept as deprecated shims so qqq-all and Voyage compile unchanged; `isEnabled()` false when the config is null; `getDefaultBackendNameForTables()` and a settable `tableMetaDataCustomizer`; namespace as default `tableNamePrefix`. Explicit `QPermissionRules` with config override (B16). Null-safe list-based validation covering backend, field existence, timestamp field and OpenSearch naming rules, plus a `QInstanceValidatorPluginInterface` for late checks (B24). `QuickSearchRuntimeService implements QRuntimeServiceInterface` owning connect, `ensureIndexExists` and close, with `startupMode` per D11 (B14); config resolved from `SourceQBitAware.getSourceQBitConfig()` in steps and the listener, `QuickSearchQBitContext` kept as a deprecated facade for one release. Unique key on `quickSearchIndex.tableName` (B19); split `enableScheduledProcesses` into basepull and maintenance flags with the old name as a deprecated alias (B25). Build: JaCoCo `prepare-agent` with an explicit version, drop the JUnit 5.10.0 pins for the parent's `junit-bom` 6.0.1, parent 2.1.x re-pin when released; CI matrix (GA plus next snapshot), Docker-backed integration tests on 2.19.x and 3.x (B22).
+
+### Epic 2: Index trustworthiness (~7 days)
+Purpose: finish the convergence story. Replace the three customizers with one `QuickSearchRecordChangeListener implements RecordChangeListenerInterface` (`appliesTo` only discovered tables; UPDATE keeps the full-record re-read through `event.getTransaction()`), registered with `qInstance.withRecordChangeListener`, publishing through `QBackendTransaction.addAfterCommitCallback` when a transaction is present and immediately otherwise (B1, D7); the listener cannot fail the write, so record failures in a `quickSearchFailedEvent` table that the scheduled run replays, with bounded retry on bulk 429/5xx and failure counts on `quickSearchIndex` (B2, D8); keyset paging in basepull and full reindex reusing the reconcile loop (B4); start-anchored watermark with configurable overlap and greater-than-or-equal (B5); `QScheduleMetaData` on basepull and reconcile with `schedulerName` and per-table interval (B6); full reindex into a fresh index with alias swap and in-place fallback, `conflicts(Proceed)`, error message on FAILED runs (B10); aggregate per-table failures into the step result (B20); honour `enabled` in every path (B21); orphan sweep for removed or renamed tables, config and mapping drift detection with a `NEEDS_REINDEX` status and an index-version marker (B23); external document versioning from `modifyDate`; run-history purge; `IndexEvent.occurredAt` and version.
+
+### Epic 3: Search quality and permission-aware search (~5 days)
+Purpose: make results correct and safe. Standard+lowercase `search_analyzer`, `operator=AND` or `minimum_should_match`, boosts that apply to prefix matches (B7); `dynamic_templates` for `fieldValues.*`, `date_detection: false`, stringified values, `lenient(true)` (B8); display values and PVS labels at index time, `QValueFormatter.formatRecordLabel` fallback, per-field `maxLength` (B9); table-level READ permission filter mirroring core, `terms` filter on `sourceTable`, `QueryAction` post-filter by primary key for record locks per D3, hidden and masked fields excluded at index time (B3); bounded `limit`, window and term length, `track_total_hits` with a lower-bound flag (B17); `OpenSearchException` wrapped in `QException` (B18); disabled tables excluded from search (B21).
+
+### Epic 4: Search surface and UI integration (~4 days, partly in qqq core)
+Purpose: make search usable from the UI. Input parity with core: `tableNames`, `limitPerTable`, `inputSource`; output parity: `tableLabel`, typed `recordId`; default the field list from `QTableMetaData.searchFields` when present; propose and implement a provider SPI in qqq core so `RecordSearchAction` delegates to a registered provider (D1); ship `QuickSearchRecordSearchProvider`; keep `QuickSearchAction` as the richer backend API (scores, highlights, pagination).
+
+### Epic 5: Connection security and authentication (~5 days)
+Purpose: run against real clusters safely. opensearch-java 3.10.0 bump with the `Refresh.True` fix and content-compression workaround kept (B13); `opensearchUrl` and optional host list, https default; `tls` block (CA path, trust store, key store for mTLS, hostname verification, guarded `insecureSkipVerify`) (B11); `authMode` enum `NONE`, `BASIC`, `AWS_SIGV4` with `awsRegion`, `awsServiceName`, `awsAssumeRoleArn` and optional AWS SDK v2 dependencies per D6; `transportCustomizer` `QCodeReference` SPI for bearer, API key and mTLS; `${env.}` interpretation of secrets via `QMetaDataVariableInterpreter` at client construction, `@JsonIgnore` secret getters, redacting `toString()`, `transient` publisher (B12); `allowPlaintextCredentials` gate; `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `User-Agent`, byte-capped bulk batches (B15); validation rules and the deprecation path for the old setters; a secured Testcontainers test (`withSecurityEnabled`).
+
+### Epic 6: Operations, docs and release (~4 days)
+Purpose: ship 1.0 cleanly. Process input screens with a table PVS and summary screen (B25); admin table fields `lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`, read-only tracking fields, PVS from run to index, sections and labels; health method surfaced in the admin table; README rewrite (truthful feature list, dependency snippet, supported-platforms statement, least-privilege roles for the security plugin and AWS FGAC, required cluster privileges for alias swap, `QQQ_ENV_*` warning, SPI stability statement); configuration reference; 0.x to 1.0 migration notes including the forced reindex; CHANGELOG and release notes; 0.3.0 compatibility release on parent 2.1.0 first, then 1.0.0; qqq-all and Voyage pin bumps.
+
+## Decisions for the sponsor
+
+Each has a recommendation. Resolve one at a time; answers get folded into tickets as facts.
+
+| ID | Question | Recommendation |
+|---|---|---|
+| D1 | Integrate with core `RecordSearchAction` via a new qqq SPI, or keep Quick Search API-only? | Propose the SPI in qqq now; ship 1.0 API-only if it slips, UI in 1.1 |
+| D2 | Version plan: does the qqq#921 chain ship 0.3.0 (compat only) and 1.0 follows, or does 4.1 wait for 1.0? | 0.3.0 on the RC chain, 1.0 after E1 to E6; do not gate 4.1 on 1.0 |
+| D3 | Permission depth for 1.0: table-level read permission only, or record-level via `QueryAction` post-filter? | Table-level plus `QueryAction` post-filter by primary key (reuses locks); no indexed security keys in 1.0 |
+| D4 | Scheduling default: schedule automatically when a scheduler exists, or require `schedulerName`? | Require `schedulerName` when basepull is enabled; validation error otherwise so the README promise is enforced |
+| D5 | Full reindex strategy: alias swap (needs cluster privileges) or in-place reconcile semantics only? | Alias swap with documented privileges, in-place fallback when `createIndex` is denied |
+| D6 | AWS IAM SigV4 in 1.0 (optional AWS SDK v2 dependencies, `authMode=AWS_SIGV4`), or declare unsupported and ship basic-auth FGAC only? | Include it: the transport ships inside opensearch-java, the SDK artifacts are optional, and IAM is the default posture for AWS domains; Serverless stays out |
+| D7 | Pre-commit indexing (B1): use 4.1's `RecordChangeListenerInterface` plus `QBackendTransaction.addAfterCommitCallback`, or keep customizers and document pre-commit semantics with scheduled reconcile as the corrector? | Use the 4.1 listener and after-commit callback (reference: `qqq-esb` `EsbRecordChangeListener`); flush immediately when no transaction is passed; keep reconcile scheduled as the backstop; this makes 1.0 4.1-only (D10) |
+| D8 | Real-time failure contract (B2): surface as QQQ record warnings plus a failed-event table (host write succeeds), or fail the host write when OpenSearch is down? | Warnings plus failed-event table; never fail the host write for a search index |
+| D9 | Supported platforms statement for 1.0: self-managed 2.19.x and 3.x plus AWS managed domains (FGAC basic auth, and IAM if D6); Serverless, Elasticsearch and OpenSearch 1.x unsupported? | Yes, as stated; revisit Serverless in 1.x once `_delete_by_query` is removed from the design |
+| D10 | Does 1.0 require qqq 4.1 (adopting `RecordChangeListenerInterface`, after-commit callbacks and `QRuntimeServiceInterface`), or keep a 4.0 fallback path with customizers and `MultiCustomizer`? | 4.1-only; 0.3.0 is the last 4.0 line; delete the customizers in 1.0 rather than carrying two real-time paths |
+| D11 | `startupMode` default: `FAIL_FAST` (host refuses to boot when OpenSearch is unreachable) or `DEGRADED` (current behaviour, which qqq-all's `FullQBitsTest` relies on with OpenSearch at `127.0.0.1:1`)? | `FAIL_FAST` by default for production safety; qqq-all's full profile sets `DEGRADED` explicitly in its follow-up PR, and `DEGRADED` surfaces a health notice |
+
+## Definition of Done
+
+All six epics complete; success criteria met; 1.0.0 published on Central and pinned by qqq-all and Voyage; README, configuration reference, migration notes and CHANGELOG accurate; every blocker in `docs/audits/2026-10-04/README.md` closed or deferred with a reason recorded here and in the second-brain project note.

docs/reference/typesafe-jev-candidates.md

@@ -0,0 +1,36 @@
+# TypeSafe (Jev) candidates for Quick Search
+
+Captured 2026-10-04. Not in 1.0 scope (see `docs/initiatives/quick-search-1.0.md`); a possible 1.x epic or companion extension. Nothing here is implemented.
+
+Jev is TypeSafe's System One model: it returns typed answers (Choice, Score, Noul probability) over a `state` payload via `POST https://api.typesafe.ai/v1/systemone`. There is no Java SDK; the HTTP API is a single JSON POST, and the project already has httpclient5 and Jackson. Docs: https://docs.typesafe.ai/llms.txt
+
+## How the project judges things today
+
+| Concern | Current mechanism | Code |
+|---|---|---|
+| Ranking | BM25 over edge-ngram `searchableText` plus per-field `should` boosts with hand-set weights | `QuickSearchOpenSearchClient.search`, `buildFieldBoostQueries` |
+| Routing | Optional `tableName` filter only; no logic decides which table a query is about | `QuickSearchInput`, `QuickSearchAction` |
+| Query classification | Lowercase and whitespace collapse | `IndexingUtils.normalizeSearchText` |
+| Index event routing | Deterministic INDEX / DELETE; no semantic step, no Jev fit | `IndexEventAction`, customizers, `ReconcileIndexStep` |
+
+## Candidates, strongest first
+
+1. **Table routing (one Choice).** State: the query plus a catalog of indexed tables (label, searchable field names). Question: which kind of record is this query looking for, one option per table plus "no specific table". Use the probability distribution as per-table boosts on `sourceTable`, or feed it to core `RecordSearchAction.tableNames` / `limitPerTable` once the Epic 4 provider exists. Confidence-gate: narrow only when confidence is high, otherwise search everything. Tiny state, cheap, fast. New capability, not a simplification.
+2. **Re-ranking the OpenSearch shortlist (one Noul per candidate).** Over-fetch about three pages, ask "is this record what the query is asking for" with the query and each candidate's `recordLabel` and `fieldValues` in state (all candidates in one request as parallel questions), sort by probability, cut to `limit`. Pattern: https://docs.typesafe.ai/cookbooks/rerank_typesafe.md. Costs: a network hop on the search hot path (not for per-keystroke typeahead), and record data leaving the cluster (needs enterprise zero-data-retention terms for CRM or health-adjacent data). Only as an opt-in reranker behind a flag, explicit searches, first page.
+3. **Highlight selection.** Choice over highlight fragment ids ("which snippet best shows why this matched") instead of joining all fragments with "...". Small win, same egress concern as item 2.
+4. **Offline weight suggestion.** Dev-time scoring of sample records per field for how identifying each value is, to suggest `weight`. Not runtime.
+
+## Rejected
+
+- Query-shape classification (email, phone, order number, name, free text) to switch exact keyword vs ngram matching. Regexes do this deterministically; Jev's own jaggedness notes say to keep literal pattern work in code.
+
+## Integration notes
+
+- Put any client behind an interface (`SearchRouter`, `ResultReranker`) so the core QBit stays OpenSearch-only and the integration is optional. Dependency or build changes need owner approval.
+- Jev 1.13 limits: 80 requests per second, 100K tokens per second, 32k tokens of state per question. Fine for routing; tight for reranking under load.
+- Pin a versioned model id (`jev-1.13.0`) once confidence thresholds are tuned; aliases move.
+- API key stays server-side in host configuration, never in the QBit's metadata.
+
+## Recommendation
+
+Prototype item 1 only, as a confidence-gated boost, and measure against the seeded qqq-all customer data before considering item 2.

mvnw

@@ -0,0 +1,295 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.3.4
+#
+# Optional ENV vars
+# -----------------
+#   JAVA_HOME - location of a JDK home dir, required when download maven via java source
+#   MVNW_REPOURL - repo url base for downloading maven distribution
+#   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+#   MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
+# ----------------------------------------------------------------------------
+
+set -euf
+[ "${MVNW_VERBOSE-}" != debug ] || set -x
+
+# OS specific support.
+native_path() { printf %s\\n "$1"; }
+case "$(uname)" in
+CYGWIN* | MINGW*)
+  [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
+  native_path() { cygpath --path --windows "$1"; }
+  ;;
+esac
+
+# set JAVACMD and JAVACCMD
+set_java_home() {
+  # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
+  if [ -n "${JAVA_HOME-}" ]; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+      JAVACCMD="$JAVA_HOME/jre/sh/javac"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+      JAVACCMD="$JAVA_HOME/bin/javac"
+
+      if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
+        echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
+        echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
+        return 1
+      fi
+    fi
+  else
+    JAVACMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v java
+    )" || :
+    JAVACCMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v javac
+    )" || :
+
+    if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
+      echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
+      return 1
+    fi
+  fi
+}
+
+# hash string like Java String::hashCode
+hash_string() {
+  str="${1:-}" h=0
+  while [ -n "$str" ]; do
+    char="${str%"${str#?}"}"
+    h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
+    str="${str#?}"
+  done
+  printf %x\\n $h
+}
+
+verbose() { :; }
+[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
+
+die() {
+  printf %s\\n "$1" >&2
+  exit 1
+}
+
+trim() {
+  # MWRAPPER-139:
+  #   Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
+  #   Needed for removing poorly interpreted newline sequences when running in more
+  #   exotic environments such as mingw bash on Windows.
+  printf "%s" "${1}" | tr -d '[:space:]'
+}
+
+scriptDir="$(dirname "$0")"
+scriptName="$(basename "$0")"
+
+# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
+while IFS="=" read -r key value; do
+  case "${key-}" in
+  distributionUrl) distributionUrl=$(trim "${value-}") ;;
+  distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
+  esac
+done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
+[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+
+case "${distributionUrl##*/}" in
+maven-mvnd-*bin.*)
+  MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
+  case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
+  *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
+  :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
+  :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
+  :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
+  *)
+    echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
+    distributionPlatform=linux-amd64
+    ;;
+  esac
+  distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
+  ;;
+maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
+*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
+esac
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
+distributionUrlName="${distributionUrl##*/}"
+distributionUrlNameMain="${distributionUrlName%.*}"
+distributionUrlNameMain="${distributionUrlNameMain%-bin}"
+MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
+MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
+
+exec_maven() {
+  unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
+  exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
+}
+
+if [ -d "$MAVEN_HOME" ]; then
+  verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  exec_maven "$@"
+fi
+
+case "${distributionUrl-}" in
+*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
+*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
+esac
+
+# prepare tmp dir
+if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
+  clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
+  trap clean HUP INT TERM EXIT
+else
+  die "cannot create temp dir"
+fi
+
+mkdir -p -- "${MAVEN_HOME%/*}"
+
+# Download and Install Apache Maven
+verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+verbose "Downloading from: $distributionUrl"
+verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+# select .zip or .tar.gz
+if ! command -v unzip >/dev/null; then
+  distributionUrl="${distributionUrl%.zip}.tar.gz"
+  distributionUrlName="${distributionUrl##*/}"
+fi
+
+# verbose opt
+__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
+[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
+
+# normalize http auth
+case "${MVNW_PASSWORD:+has-password}" in
+'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+esac
+
+if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
+  verbose "Found wget ... using wget"
+  wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
+elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
+  verbose "Found curl ... using curl"
+  curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
+elif set_java_home; then
+  verbose "Falling back to use Java to download"
+  javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
+  targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
+  cat >"$javaSource" <<-END
+	public class Downloader extends java.net.Authenticator
+	{
+	  protected java.net.PasswordAuthentication getPasswordAuthentication()
+	  {
+	    return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
+	  }
+	  public static void main( String[] args ) throws Exception
+	  {
+	    setDefault( new Downloader() );
+	    java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
+	  }
+	}
+	END
+  # For Cygwin/MinGW, switch paths to Windows format before running javac and java
+  verbose " - Compiling Downloader.java ..."
+  "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
+  verbose " - Running Downloader.java ..."
+  "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
+fi
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+if [ -n "${distributionSha256Sum-}" ]; then
+  distributionSha256Result=false
+  if [ "$MVN_CMD" = mvnd.sh ]; then
+    echo "Checksum validation is not supported for maven-mvnd." >&2
+    echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  elif command -v sha256sum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  elif command -v shasum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  else
+    echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
+    echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  fi
+  if [ $distributionSha256Result = false ]; then
+    echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
+    echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
+    exit 1
+  fi
+fi
+
+# unzip and move
+if command -v unzip >/dev/null; then
+  unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
+else
+  tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
+fi
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+actualDistributionDir=""
+
+# First try the expected directory name (for regular distributions)
+if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
+  if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
+    actualDistributionDir="$distributionUrlNameMain"
+  fi
+fi
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if [ -z "$actualDistributionDir" ]; then
+  # enable globbing to iterate over items
+  set +f
+  for dir in "$TMP_DOWNLOAD_DIR"/*; do
+    if [ -d "$dir" ]; then
+      if [ -f "$dir/bin/$MVN_CMD" ]; then
+        actualDistributionDir="$(basename "$dir")"
+        break
+      fi
+    fi
+  done
+  set -f
+fi
+
+if [ -z "$actualDistributionDir" ]; then
+  verbose "Contents of $TMP_DOWNLOAD_DIR:"
+  verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
+  die "Could not find Maven distribution directory in extracted archive"
+fi
+
+verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
+mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
+
+clean || :
+exec_maven "$@"

mvnw.cmd

@@ -0,0 +1,189 @@
+<# : batch portion
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Apache Maven Wrapper startup batch script, version 3.3.4
+@REM
+@REM Optional ENV vars
+@REM   MVNW_REPOURL - repo url base for downloading maven distribution
+@REM   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+@REM   MVNW_VERBOSE - true: enable verbose log; others: silence the output
+@REM ----------------------------------------------------------------------------
+
+@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
+@SET __MVNW_CMD__=
+@SET __MVNW_ERROR__=
+@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
+@SET PSModulePath=
+@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
+  IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
+)
+@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
+@SET __MVNW_PSMODULEP_SAVE=
+@SET __MVNW_ARG0_NAME__=
+@SET MVNW_USERNAME=
+@SET MVNW_PASSWORD=
+@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
+@echo Cannot start maven from wrapper >&2 && exit /b 1
+@GOTO :EOF
+: end batch / begin powershell #>
+
+$ErrorActionPreference = "Stop"
+if ($env:MVNW_VERBOSE -eq "true") {
+  $VerbosePreference = "Continue"
+}
+
+# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
+$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
+if (!$distributionUrl) {
+  Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+}
+
+switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
+  "maven-mvnd-*" {
+    $USE_MVND = $true
+    $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
+    $MVN_CMD = "mvnd.cmd"
+    break
+  }
+  default {
+    $USE_MVND = $false
+    $MVN_CMD = $script -replace '^mvnw','mvn'
+    break
+  }
+}
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+if ($env:MVNW_REPOURL) {
+  $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
+  $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
+}
+$distributionUrlName = $distributionUrl -replace '^.*/',''
+$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
+
+$MAVEN_M2_PATH = "$HOME/.m2"
+if ($env:MAVEN_USER_HOME) {
+  $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
+}
+
+if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
+    New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
+}
+
+$MAVEN_WRAPPER_DISTS = $null
+if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
+  $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
+} else {
+  $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
+}
+
+$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
+$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
+$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
+
+if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
+  Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+  exit $?
+}
+
+if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
+  Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
+}
+
+# prepare tmp dir
+$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
+$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
+$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
+trap {
+  if ($TMP_DOWNLOAD_DIR.Exists) {
+    try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+    catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+  }
+}
+
+New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
+
+# Download and Install Apache Maven
+Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+Write-Verbose "Downloading from: $distributionUrl"
+Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+$webclient = New-Object System.Net.WebClient
+if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
+  $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
+}
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
+if ($distributionSha256Sum) {
+  if ($USE_MVND) {
+    Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
+  }
+  Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
+  if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
+    Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
+  }
+}
+
+# unzip and move
+Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+$actualDistributionDir = ""
+
+# First try the expected directory name (for regular distributions)
+$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
+$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
+if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
+  $actualDistributionDir = $distributionUrlNameMain
+}
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if (!$actualDistributionDir) {
+  Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
+    $testPath = Join-Path $_.FullName "bin/$MVN_CMD"
+    if (Test-Path -Path $testPath -PathType Leaf) {
+      $actualDistributionDir = $_.Name
+    }
+  }
+}
+
+if (!$actualDistributionDir) {
+  Write-Error "Could not find Maven distribution directory in extracted archive"
+}
+
+Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
+try {
+  Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
+} catch {
+  if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
+    Write-Error "fail to move MAVEN_HOME"
+  }
+} finally {
+  try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+  catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+}
+
+Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"

pom.xml

@@ -18,16 +18,22 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
-      <revision>0.2.0</revision>
+      <qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
+      <qbit.guardrails.enforce>true</qbit.guardrails.enforce>
+      <qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
+      <revision>1.1.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
-      <opensearch.version>2.10.0</opensearch.version>
+      <opensearch.version>3.10.0</opensearch.version>
+      <jacoco.version>0.8.15</jacoco.version>
+      <opensearch.test.image>opensearchproject/opensearch:2.19.6</opensearch.test.image>
+      <aws.sdk.version>2.41.10</aws.sdk.version>
    </properties>
 
    <parent>
       <groupId>com.kingsrook</groupId>
       <artifactId>qbit-build-parent</artifactId>
-      <version>1.6.0</version>
+      <version>2.2.1</version>
    </parent>
 
    <groupId>com.kingsrook.qbits</groupId>
@@ -66,26 +72,12 @@
       <tag>HEAD</tag>
    </scm>
 
-   <dependencyManagement>
-      <dependencies>
-         <dependency>
-            <groupId>org.junit</groupId>
-            <artifactId>junit-bom</artifactId>
-            <version>5.10.0</version>
-            <type>pom</type>
-            <scope>import</scope>
-         </dependency>
-         <dependency>
-            <groupId>com.kingsrook.qqq</groupId>
-            <artifactId>qqq-bom-pom</artifactId>
-            <version>0.40.0</version>
-            <type>pom</type>
-            <scope>import</scope>
-         </dependency>
-      </dependencies>
-   </dependencyManagement>
-
    <dependencies>
+      <dependency>
+         <groupId>com.kingsrook.qqq</groupId>
+         <artifactId>qqq-middleware-javalin</artifactId>
+      </dependency>
+
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
@@ -97,16 +89,32 @@
          <version>${opensearch.version}</version>
       </dependency>
 
+      <!-- httpclient5 and jackson versions are managed by the qqq BOM, so they -->
+      <!-- stay binary-compatible with the httpcore5 and jackson it brings.       -->
       <dependency>
          <groupId>org.apache.httpcomponents.client5</groupId>
          <artifactId>httpclient5</artifactId>
-         <version>5.3</version>
       </dependency>
 
       <dependency>
          <groupId>com.fasterxml.jackson.datatype</groupId>
          <artifactId>jackson-datatype-jsr310</artifactId>
-         <version>2.21.0</version>
+      </dependency>
+
+      <!-- Only needed for authMode AWS_SIGV4 (Amazon OpenSearch Service with IAM). -->
+      <!-- Optional: hosts that use it declare them; the core modules already come  -->
+      <!-- through qqq-backend-core at the same version.                            -->
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>apache-client</artifactId>
+         <version>${aws.sdk.version}</version>
+         <optional>true</optional>
+      </dependency>
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>sts</artifactId>
+         <version>${aws.sdk.version}</version>
+         <optional>true</optional>
       </dependency>
 
       <dependency>
@@ -125,14 +133,13 @@
       <dependency>
          <groupId>org.junit.jupiter</groupId>
          <artifactId>junit-jupiter</artifactId>
-         <version>5.10.0</version>
          <scope>test</scope>
       </dependency>
 
       <dependency>
          <groupId>org.assertj</groupId>
          <artifactId>assertj-core</artifactId>
-         <version>3.27.3</version>
+         <version>3.27.7</version>
          <scope>test</scope>
       </dependency>
 
@@ -147,18 +154,36 @@
       <dependency>
          <groupId>org.testcontainers</groupId>
          <artifactId>testcontainers</artifactId>
-         <version>1.21.0</version>
+         <version>1.21.4</version>
          <scope>test</scope>
       </dependency>
       <dependency>
          <groupId>org.testcontainers</groupId>
          <artifactId>junit-jupiter</artifactId>
-         <version>1.21.0</version>
+         <version>1.21.4</version>
          <scope>test</scope>
       </dependency>
    </dependencies>
 
    <build>
+      <resources>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>true</filtering>
+            <includes>
+               <include>help/*.yaml</include>
+               <include>quick-search.properties</include>
+            </includes>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+            <excludes>
+               <exclude>help/*.yaml</exclude>
+               <exclude>quick-search.properties</exclude>
+            </excludes>
+         </resource>
+      </resources>
       <plugins>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
@@ -177,6 +202,9 @@
             <artifactId>maven-failsafe-plugin</artifactId>
             <version>3.1.2</version>
             <configuration>
+               <systemPropertyVariables>
+                  <opensearch.test.image>${opensearch.test.image}</opensearch.test.image>
+               </systemPropertyVariables>
                <includes>
                   <include>**/*IntegrationTest.java</include>
                   <include>**/*IT.java</include>
@@ -194,7 +222,21 @@
          <plugin>
             <groupId>org.jacoco</groupId>
             <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.version}</version>
             <executions>
+               <execution>
+                  <id>prepare-agent</id>
+                  <goals>
+                     <goal>prepare-agent</goal>
+                  </goals>
+               </execution>
+               <execution>
+                  <id>report</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>report</goal>
+                  </goals>
+               </execution>
                <execution>
                   <id>check</id>
                   <goals>
@@ -224,4 +266,37 @@
          </plugin>
       </plugins>
    </build>
+
+   <profiles>
+      <!-- 1.0 requires qqq 4.1 APIs (record-change listeners, after-commit        -->
+      <!-- callbacks, runtime services). Until qbit-build-parent 2.1.0 imports the -->
+      <!-- released 4.1 BOM, this profile is active by default and imports          -->
+      <!-- qqq-bom-pom 4.1.0-RC.1 ahead of the parent's 4.0.0 BOM. Override with    -->
+      <!-- -Dqqq.snapshot.version=... ; deactivate with -P!qqq-snapshot. Remove the -->
+      <!-- activation when the parent moves to 4.1 (ADR-0007).                      -->
+      <!-- 1.1: the record-search provider needs the core SPI                       -->
+      <!-- (RecordSearchProviderInterface), first released in qqq 4.1.0-RC.2.       -->
+      <!-- The record developer panel provider needs the developer panel         -->
+      <!-- contract (DeveloperPanelProviderInterface), first in qqq 4.1.0-RC.4.   -->
+      <profile>
+         <id>qqq-snapshot</id>
+         <activation>
+            <activeByDefault>true</activeByDefault>
+         </activation>
+         <properties>
+            <qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
+         </properties>
+         <dependencyManagement>
+            <dependencies>
+               <dependency>
+                  <groupId>com.kingsrook.qqq</groupId>
+                  <artifactId>qqq-bom-pom</artifactId>
+                  <version>${qqq.snapshot.version}</version>
+                  <type>pom</type>
+                  <scope>import</scope>
+               </dependency>
+            </dependencies>
+         </dependencyManagement>
+      </profile>
+   </profiles>
 </project>

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchAuthMode.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+/*******************************************************************************
+ ** How the QBit authenticates to OpenSearch.
+ **
+ ** NONE      - no credentials (local development, security plugin disabled).
+ ** BASIC     - HTTP basic authentication (security plugin internal users, or
+ **             Amazon OpenSearch Service fine-grained access control with an
+ **             internal-database master user).
+ ** AWS_SIGV4 - AWS Signature Version 4 request signing with the default AWS
+ **             credential chain (Amazon OpenSearch Service domains that use
+ **             IAM). Requires the optional AWS SDK v2 dependencies.
+ **
+ ** Bearer tokens, API keys and client certificates are added through an
+ ** {@link com.kingsrook.qbits.quicksearch.opensearch.OpenSearchTransportCustomizer}.
+ *******************************************************************************/
+public enum QuickSearchAuthMode
+{
+   NONE,
+   BASIC,
+   AWS_SIGV4
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContext.java

@@ -13,44 +13,58 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.kingsrook.qbits.quicksearch;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
 
 
 /*******************************************************************************
- ** Static holder for Quick Search QBit runtime state.
+ ** Deprecated static facade over {@link QuickSearchRuntime}.
  **
- ** Provides access to the active configuration, the OpenSearch client, the
- ** index-event publisher, and the list of tables discovered at startup.
+ ** Until 1.0 the QBit kept its live state in these static fields. The state now
+ ** lives on the produced QBit's runtime, resolved from QContext. The getters
+ ** here return an explicitly set static value when one exists (tests and
+ ** legacy hosts), otherwise they delegate to {@link QuickSearchRuntime#get()}.
  **
- ** Call clear() in test @AfterEach / @BeforeEach to reset all state.
+ ** @deprecated resolve state with {@link QuickSearchRuntime#get()}; this
+ ** facade will be removed in 1.1.
  *******************************************************************************/
+@Deprecated
 public class QuickSearchQBitContext
 {
-   private static volatile QuickSearchQBitConfig             config;
-   private static volatile List<QuickSearchableTableConfig>  discoveredTables;
-   private static volatile QuickSearchOpenSearchClient       client;
-   private static volatile IndexEventPublisher               publisher;
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchQBitContext.class);
+
+   private static volatile QuickSearchQBitConfig            config;
+   private static volatile List<QuickSearchableTableConfig> discoveredTables;
+   private static volatile QuickSearchOpenSearchClient      client;
+   private static volatile IndexEventPublisher              publisher;
 
 
 
-   /***************************************************************************
-    ** Getter for config
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the config: the static override, else the resolved runtime's config.
+    *******************************************************************************/
    public static QuickSearchQBitConfig getConfig()
    {
-      return config;
+      if(config != null)
+      {
+         return (config);
+      }
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime == null ? null : runtime.getConfig());
    }
 
 
 
-   /***************************************************************************
-    ** Setter for config
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static config override.
+    *******************************************************************************/
    public static void setConfig(QuickSearchQBitConfig config)
    {
       QuickSearchQBitContext.config = config;
@@ -58,19 +72,24 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Getter for discoveredTables
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the discovered tables: the static override, else the runtime's.
+    *******************************************************************************/
    public static List<QuickSearchableTableConfig> getDiscoveredTables()
    {
-      return discoveredTables;
+      if(discoveredTables != null)
+      {
+         return (discoveredTables);
+      }
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime == null ? null : runtime.getDiscoveredTables());
    }
 
 
 
-   /***************************************************************************
-    ** Setter for discoveredTables
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static discovered-tables override.
+    *******************************************************************************/
    public static void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
    {
       QuickSearchQBitContext.discoveredTables = discoveredTables;
@@ -78,44 +97,62 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Find a table config by table name from the discovered tables list.
-    **
-    ** Returns null if discoveredTables is null or no match is found.
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Find a table config by name, or null.
+    *******************************************************************************/
    public static QuickSearchableTableConfig getTableConfig(String tableName)
    {
-      if(discoveredTables == null)
+      List<QuickSearchableTableConfig> tables = getDiscoveredTables();
+      if(tables == null || tableName == null)
       {
-         return null;
+         return (null);
       }
 
-      for(QuickSearchableTableConfig tableConfig : discoveredTables)
+      for(QuickSearchableTableConfig tableConfig : tables)
       {
          if(tableName.equals(tableConfig.getTableName()))
          {
-            return tableConfig;
+            return (tableConfig);
          }
       }
-
-      return null;
+      return (null);
    }
 
 
 
-   /***************************************************************************
-    ** Getter for client
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the client: the static override, else the runtime's (built lazily).
+    ** Returns null when no runtime is registered or the client cannot be built.
+    *******************************************************************************/
    public static QuickSearchOpenSearchClient getClient()
    {
-      return client;
+      if(client != null)
+      {
+         return (client);
+      }
+
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (null);
+      }
+
+      try
+      {
+         return (runtime.getClient());
+      }
+      catch(QException e)
+      {
+         LOG.warn("Could not build OpenSearch client", e);
+         return (null);
+      }
    }
 
 
 
-   /***************************************************************************
-    ** Setter for client
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static client override.
+    *******************************************************************************/
    public static void setClient(QuickSearchOpenSearchClient client)
    {
       QuickSearchQBitContext.client = client;
@@ -123,19 +160,38 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Getter for publisher
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the publisher: the static override, else the runtime's.
+    *******************************************************************************/
    public static IndexEventPublisher getPublisher()
    {
-      return publisher;
+      if(publisher != null)
+      {
+         return (publisher);
+      }
+
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (null);
+      }
+
+      try
+      {
+         return (runtime.getPublisher());
+      }
+      catch(QException e)
+      {
+         LOG.warn("Could not build index event publisher", e);
+         return (null);
+      }
    }
 
 
 
-   /***************************************************************************
-    ** Setter for publisher
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static publisher override.
+    *******************************************************************************/
    public static void setPublisher(IndexEventPublisher publisher)
    {
       QuickSearchQBitContext.publisher = publisher;
@@ -143,18 +199,15 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Reset all static state to null.
-    **
-    ** Call in test @BeforeEach or @AfterEach to prevent state leakage
-    ** between tests.
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Clear the static overrides (tests).
+    *******************************************************************************/
    public static void clear()
    {
-      config = null;
+      config           = null;
       discoveredTables = null;
-      client = null;
-      publisher = null;
+      client           = null;
+      publisher        = null;
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntime.java

@@ -0,0 +1,828 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigStore;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticEventRecorder;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsRateLimiter;
+import com.kingsrook.qbits.quicksearch.diagnostics.StatusCache;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
+import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Live state of one produced Quick Search QBit: its discovered tables, the
+ ** lazily built OpenSearch client and publisher, and a short-lived cache of
+ ** which tables are enabled.
+ **
+ ** One runtime exists per produced QBit. It hangs off the QBit's config
+ ** (QuickSearchQBitConfig.getRuntime()), which the host's QInstance holds in
+ ** QBitMetaData, so steps, listeners and actions resolve it from QContext with
+ ** {@link #get()} rather than from JVM-global statics. Two QInstances in one
+ ** JVM therefore no longer share state.
+ **
+ ** The tables declared in code (annotations, searchableTables) are kept as
+ ** the code defaults; getDiscoveredTables() and getTableConfig() return the
+ ** effective configs, which come from the config records once they are
+ ** primed (see QuickSearchConfigStore for the precedence).
+ *******************************************************************************/
+public class QuickSearchRuntime
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRuntime.class);
+
+   private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
+
+   /*******************************************************************************
+    ** Cluster cache name for every runtime's config cache (the config store's
+    ** snapshot and settings, and the enabled flags). A config row edited on
+    ** one node clears it on every node; the TTLs remain the safety net.
+    *******************************************************************************/
+   public static final String CONFIG_CACHE_NAME = "qbit-quick-search.config";
+
+   private static final Set<QuickSearchRuntime> RUNTIMES = ClusterCaches.shared(CONFIG_CACHE_NAME,
+      Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())),
+      QuickSearchRuntime::invalidateAllLocal, null);
+
+   public static final long RECORD_SEARCH_BACKOFF_MILLIS = 30_000L;
+
+   private final StatusCache statusCache = new StatusCache();
+
+   private final QuickSearchQBitConfig config;
+
+   private List<QuickSearchableTableConfig>        codeTables       = new ArrayList<>();
+   private Map<String, QuickSearchableTableConfig> codeTablesByName = new LinkedHashMap<>();
+
+   private final QuickSearchConfigStore configStore;
+
+   private volatile QuickSearchOpenSearchClient client;
+   private volatile IndexEventPublisher         publisher;
+   private volatile boolean                     indexReady                     = false;
+   private volatile boolean                     closed                         = false;
+   private volatile long                        recordSearchBackoffUntilMillis = 0L;
+
+   private final Map<String, CachedFlag> enabledCache  = new ConcurrentHashMap<>();
+   private final Map<String, Long>       buildFailures = new ConcurrentHashMap<>();
+
+   private volatile DiagnosticEventRecorder diagnosticEventRecorder;
+   private final DiagnosticsRateLimiter     diagnosticsRateLimiter = new DiagnosticsRateLimiter();
+   private final AtomicBoolean              validationRunning      = new AtomicBoolean(false);
+   private final AtomicLong                 recordSearchFailures   = new AtomicLong();
+   private final AtomicLong                 backoffDeclinedClaims  = new AtomicLong();
+   private final Instant                    startedAt              = Instant.now();
+   private volatile Instant                 lastRecordSearchFailureTime;
+   private volatile QInstance               startInstance;
+   private volatile String                  lastRecordSearchFailureMessage;
+
+
+
+   /*******************************************************************************
+    ** Instance-owned bounded administrative status cache.
+    *******************************************************************************/
+   public StatusCache getStatusCache()
+   {
+      return (statusCache);
+   }
+
+
+
+   /*******************************************************************************
+    ** A cached enabled flag with the time it was read.
+    *******************************************************************************/
+   private record CachedFlag(boolean enabled, long readAtMillis)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchRuntime(QuickSearchQBitConfig config)
+   {
+      this.config      = config;
+      this.configStore = new QuickSearchConfigStore(this);
+      RUNTIMES.add(this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve the runtime of the Quick Search QBit registered in the current
+    ** QContext's QInstance. Returns null when there is no QContext, no QInstance,
+    ** no Quick Search QBit, or the QBit has not been produced yet.
+    *******************************************************************************/
+   public static QuickSearchRuntime get()
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(qInstance == null || qInstance.getQBits() == null)
+      {
+         return (null);
+      }
+
+      for(QBitMetaData qBitMetaData : qInstance.getQBits().values())
+      {
+         if(qBitMetaData.getConfig() instanceof QuickSearchQBitConfig quickSearchConfig && quickSearchConfig.getRuntime() != null)
+         {
+            return (quickSearchConfig.getRuntime());
+         }
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Like {@link #get()}, but throws when no runtime can be resolved.
+    *******************************************************************************/
+   public static QuickSearchRuntime getOrThrow() throws QException
+   {
+      QuickSearchRuntime runtime = get();
+      if(runtime == null)
+      {
+         throw (new QException("Quick Search QBit is not registered in the current QInstance (or has not been produced)"));
+      }
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for config
+    *******************************************************************************/
+   public QuickSearchQBitConfig getConfig()
+   {
+      return (this.config);
+   }
+
+
+
+   /*******************************************************************************
+    ** The effective searchable tables (never null): from the config records
+    ** once primed, else the code-declared ones.
+    *******************************************************************************/
+   public List<QuickSearchableTableConfig> getDiscoveredTables()
+   {
+      return (Collections.unmodifiableList(configStore.getTables()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace the code-declared tables (the defaults priming seeds).
+    *******************************************************************************/
+   public void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
+   {
+      this.codeTables       = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
+      this.codeTablesByName = new LinkedHashMap<>();
+      for(QuickSearchableTableConfig tableConfig : this.codeTables)
+      {
+         this.codeTablesByName.put(tableConfig.getTableName(), tableConfig);
+      }
+      configStore.invalidate();
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables declared in code (annotations and searchableTables), never null.
+    *******************************************************************************/
+   public List<QuickSearchableTableConfig> getCodeTables()
+   {
+      return (Collections.unmodifiableList(this.codeTables));
+   }
+
+
+
+   /*******************************************************************************
+    ** One code-declared table's config, or null.
+    *******************************************************************************/
+   public QuickSearchableTableConfig getCodeTableConfig(String tableName)
+   {
+      return (tableName == null ? null : codeTablesByName.get(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Find the effective config for one table, or null when it is not indexed.
+    *******************************************************************************/
+   public QuickSearchableTableConfig getTableConfig(String tableName)
+   {
+      return (configStore.getTable(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** The store of the effective config records.
+    *******************************************************************************/
+   public QuickSearchConfigStore getConfigStore()
+   {
+      return (configStore);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an admin has paused all indexing (search keeps serving).
+    *******************************************************************************/
+   public boolean isIndexingPaused()
+   {
+      return (configStore.isIndexingPaused());
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop every cached config (a config row changed): on this node now, and
+    ** on the other nodes through the cluster bus.
+    *******************************************************************************/
+   public void invalidateConfig()
+   {
+      invalidateConfigLocal();
+      ClusterCaches.invalidate(CONFIG_CACHE_NAME);
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop this runtime's cached config on this node only.
+    *******************************************************************************/
+   void invalidateConfigLocal()
+   {
+      configStore.invalidate();
+      enabledCache.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the cached config of every live runtime on this node (a cluster
+    ** invalidation arrived).
+    *******************************************************************************/
+   private static void invalidateAllLocal()
+   {
+      List<QuickSearchRuntime> runtimes;
+      synchronized(RUNTIMES)
+      {
+         runtimes = new ArrayList<>(RUNTIMES);
+      }
+      for(QuickSearchRuntime runtime : runtimes)
+      {
+         runtime.invalidateConfigLocal();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Return the OpenSearch client, building it on first use. Building the
+    ** client does not contact the cluster; {@link #ensureIndexReady()} does.
+    *******************************************************************************/
+   public QuickSearchOpenSearchClient getClient() throws QException
+   {
+      QuickSearchOpenSearchClient existing = client;
+      if(existing != null)
+      {
+         return (existing);
+      }
+
+      synchronized(this)
+      {
+         if(client == null)
+         {
+            client = new QuickSearchOpenSearchClient(config);
+         }
+         return (client);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Make sure the index (or alias) exists with the current mapping. Runs the
+    ** check once per runtime; a failure leaves it unready so the next call
+    ** retries.
+    *******************************************************************************/
+   public void ensureIndexReady() throws QException
+   {
+      if(indexReady)
+      {
+         return;
+      }
+
+      synchronized(this)
+      {
+         if(!indexReady)
+         {
+            getClient().ensureIndexExists();
+            indexReady = true;
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether {@link #ensureIndexReady()} has succeeded.
+    *******************************************************************************/
+   public boolean isIndexReady()
+   {
+      return (indexReady);
+   }
+
+
+
+   /*******************************************************************************
+    ** Return the publisher: the host-supplied one from config, else a
+    ** SynchronousIndexEventPublisher over this runtime's client.
+    *******************************************************************************/
+   public IndexEventPublisher getPublisher() throws QException
+   {
+      IndexEventPublisher existing = publisher;
+      if(existing != null)
+      {
+         return (existing);
+      }
+
+      synchronized(this)
+      {
+         if(publisher == null)
+         {
+            if(config.getIndexEventPublisher() != null)
+            {
+               publisher = config.getIndexEventPublisher();
+            }
+            else
+            {
+               publisher = new SynchronousIndexEventPublisher(getClient(), this::getTableConfig, config.getBulkBatchSize());
+            }
+         }
+         return (publisher);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table is enabled for indexing and search, per its
+    ** quick_search_index row. A table with no row yet is enabled. Results are
+    ** cached for one minute; a failed lookup is treated as enabled so indexing
+    ** does not stop because the operational table is unreachable.
+    *******************************************************************************/
+   public boolean isTableEnabled(String tableName)
+   {
+      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         return (false);
+      }
+      if(tableConfig.getEnabled() != null)
+      {
+         return (tableConfig.getEnabled());
+      }
+
+      long       now    = System.currentTimeMillis();
+      CachedFlag cached = enabledCache.get(tableName);
+      if(cached != null && now - cached.readAtMillis() < ENABLED_CACHE_TTL_MILLIS)
+      {
+         return (cached.enabled());
+      }
+
+      boolean enabled = readEnabledFlag(tableName);
+      enabledCache.put(tableName, new CachedFlag(enabled, now));
+      return (enabled);
+   }
+
+
+
+   /*******************************************************************************
+    ** When a queued build of the table last failed on this server (0: never).
+    *******************************************************************************/
+   public long getLastBuildFailureMillis(String tableName)
+   {
+      return (buildFailures.getOrDefault(tableName, 0L));
+   }
+
+
+
+   /*******************************************************************************
+    ** Remember that a queued build of the table failed now.
+    *******************************************************************************/
+   public void recordBuildFailure(String tableName)
+   {
+      buildFailures.put(tableName, System.currentTimeMillis());
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget a table's build failure (it built).
+    *******************************************************************************/
+   public void clearBuildFailure(String tableName)
+   {
+      buildFailures.remove(tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** How many enabled flags are cached (for tests).
+    *******************************************************************************/
+   int getCachedEnabledFlagCount()
+   {
+      return (enabledCache.size());
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget cached enabled flags (after an operator toggles a table).
+    *******************************************************************************/
+   public void invalidateEnabledCache()
+   {
+      enabledCache.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Record that a core record search through OpenSearch failed. For the next
+    ** RECORD_SEARCH_BACKOFF_MILLIS the record-search provider claims no tables,
+    ** so core searches them itself at once instead of waiting for another
+    ** timeout on every search while OpenSearch is down.
+    *******************************************************************************/
+   public void markRecordSearchFailed()
+   {
+      markRecordSearchFailed(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** As markRecordSearchFailed(), also counting the failure and keeping its
+    ** message and time for the diagnostics.
+    *******************************************************************************/
+   public void markRecordSearchFailed(String message)
+   {
+      recordSearchBackoffUntilMillis = System.currentTimeMillis() + RECORD_SEARCH_BACKOFF_MILLIS;
+      recordSearchFailures.incrementAndGet();
+      lastRecordSearchFailureTime    = Instant.now();
+      lastRecordSearchFailureMessage = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Milliseconds left in the current record-search backoff (0 when none).
+    *******************************************************************************/
+   public long getRecordSearchBackoffRemainingMillis()
+   {
+      return (Math.max(0L, recordSearchBackoffUntilMillis - System.currentTimeMillis()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Count a table the provider declined because of a backoff (core searched
+    ** it with its own search instead).
+    *******************************************************************************/
+   public void countBackoffDeclinedClaim()
+   {
+      backoffDeclinedClaims.incrementAndGet();
+   }
+
+
+
+   /*******************************************************************************
+    ** Record searches through OpenSearch that failed since this runtime started.
+    *******************************************************************************/
+   public long getRecordSearchFailureCount()
+   {
+      return (recordSearchFailures.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Table searches core ran itself because the provider was backing off,
+    ** since this runtime started.
+    *******************************************************************************/
+   public long getBackoffDeclinedClaimCount()
+   {
+      return (backoffDeclinedClaims.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** When the last record search through OpenSearch failed (null when none).
+    *******************************************************************************/
+   public Instant getLastRecordSearchFailureTime()
+   {
+      return (lastRecordSearchFailureTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** The last record-search failure's message (null when none).
+    *******************************************************************************/
+   public String getLastRecordSearchFailureMessage()
+   {
+      return (lastRecordSearchFailureMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** When this runtime was created.
+    *******************************************************************************/
+   public Instant getStartedAt()
+   {
+      return (startedAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** The diagnostic event recorder, built on first use.
+    *******************************************************************************/
+   public DiagnosticEventRecorder getDiagnosticEventRecorder()
+   {
+      DiagnosticEventRecorder existing = diagnosticEventRecorder;
+      if(existing != null)
+      {
+         return (existing);
+      }
+      synchronized(this)
+      {
+         if(diagnosticEventRecorder == null)
+         {
+            diagnosticEventRecorder = new DiagnosticEventRecorder(config);
+         }
+         return (diagnosticEventRecorder);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The per-user rate limiter of the diagnostic tools.
+    *******************************************************************************/
+   public DiagnosticsRateLimiter getDiagnosticsRateLimiter()
+   {
+      return (diagnosticsRateLimiter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Claim the single validation slot of this JVM; false when a validation is
+    ** already running. Release with endValidation().
+    *******************************************************************************/
+   public boolean beginValidation()
+   {
+      return (validationRunning.compareAndSet(false, true));
+   }
+
+
+
+   /*******************************************************************************
+    ** Release the validation slot.
+    *******************************************************************************/
+   public void endValidation()
+   {
+      validationRunning.set(false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether core record search is backing off after a recent failure.
+    *******************************************************************************/
+   public boolean isRecordSearchBackingOff()
+   {
+      return (System.currentTimeMillis() < recordSearchBackoffUntilMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** End a record-search backoff early (an operator or test knows OpenSearch is
+    ** back).
+    *******************************************************************************/
+   public void clearRecordSearchBackoff()
+   {
+      recordSearchBackoffUntilMillis = 0L;
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the enabled flag from the quick_search_index row.
+    *******************************************************************************/
+   private boolean readEnabledFlag(String tableName)
+   {
+      try
+      {
+         if(QContext.getQInstance() == null || QContext.getQInstance().getTable(config.getQuickSearchIndexTableName()) == null)
+         {
+            return (true);
+         }
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(config.getQuickSearchIndexTableName());
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+         List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+
+         if(rows == null || rows.isEmpty())
+         {
+            return (true);
+         }
+
+         Boolean enabled = rows.get(0).getValueBoolean("enabled");
+         return (enabled == null || enabled);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not read enabled flag for table; treating as enabled", e, logPair("tableName", tableName));
+         return (true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Start the runtime per the configured startup mode: FAIL_FAST throws when
+    ** the index cannot be prepared; DEGRADED logs and continues. A LinkageError
+    ** (an optional library such as the AWS SDK missing while the client is
+    ** built) follows the same modes, with a message naming the missing class,
+    ** so a DEGRADED host still boots.
+    *******************************************************************************/
+   public void start() throws QException
+   {
+      primeConfig();
+
+      try
+      {
+         ensureIndexReady();
+      }
+      catch(LinkageError e)
+      {
+         String message = "Quick Search could not load a class to build its OpenSearch client [" + e.getMessage()
+            + "]; authMode AWS_SIGV4 needs the optional AWS SDK v2 dependencies (apache-client, sts) on the classpath";
+         if(config.getStartupMode() == QuickSearchStartupMode.DEGRADED)
+         {
+            LOG.warn(message + "; Quick Search is running degraded", e, logPair("indexName", config.getOpensearchIndexName()));
+            return;
+         }
+
+         throw (new QException(message, e));
+      }
+      catch(Exception e)
+      {
+         if(config.getStartupMode() == QuickSearchStartupMode.DEGRADED)
+         {
+            LOG.warn("OpenSearch is not reachable; Quick Search is running degraded and will retry on first use", e,
+               logPair("indexName", config.getOpensearchIndexName()));
+            return;
+         }
+
+         throw (new QException("Quick Search could not prepare its OpenSearch index [" + config.getOpensearchIndexName()
+            + "]; set startupMode=DEGRADED to boot without it: " + e.getMessage(), e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Prime the config records from the code defaults (never throws; a failure
+    ** is retried on later use). Runs in a system session when no QContext is
+    ** set, as when a launcher starts the runtime service.
+    *******************************************************************************/
+   void primeConfig()
+   {
+      try
+      {
+         if(QContext.getQInstance() != null)
+         {
+            configStore.tryPrime();
+            return;
+         }
+
+         QInstance qInstance = startInstance;
+         if(qInstance == null)
+         {
+            return;
+         }
+         QContext.init(qInstance, new QSystemUserSession(qInstance));
+         try
+         {
+            configStore.tryPrime();
+         }
+         finally
+         {
+            QContext.clear();
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Quick Search could not prime its config records at startup; it retries on first use", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The instance to prime against when the runtime is started without a
+    ** QContext (QuickSearchRuntimeService.start sets it).
+    *******************************************************************************/
+   public void setStartInstance(QInstance startInstance)
+   {
+      this.startInstance = startInstance;
+   }
+
+
+
+   /*******************************************************************************
+    ** Close the publisher and client. Safe to call more than once.
+    *******************************************************************************/
+   public synchronized void close()
+   {
+      if(closed)
+      {
+         return;
+      }
+      closed = true;
+      RUNTIMES.remove(this);
+      statusCache.close();
+
+      if(publisher != null)
+      {
+         try
+         {
+            publisher.close();
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error closing Quick Search publisher", e);
+         }
+      }
+
+      if(diagnosticEventRecorder != null)
+      {
+         diagnosticEventRecorder.close();
+      }
+
+      if(client != null)
+      {
+         client.close();
+      }
+
+      indexReady = false;
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntimeService.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+
+
+/*******************************************************************************
+ ** Lifecycle hook for hosts that start through QApplicationLauncher: prepares
+ ** the OpenSearch index on start (per startupMode) and closes the transport on
+ ** stop. Hosts without a launcher get the same start behaviour from produce()
+ ** in FAIL_FAST mode and lazily on first use in DEGRADED mode.
+ *******************************************************************************/
+public class QuickSearchRuntimeService implements QRuntimeServiceInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRuntimeService.class);
+
+   private QuickSearchRuntime runtime;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getName()
+   {
+      return ("quickSearch");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void start(QInstance qInstance) throws QException
+   {
+      runtime = findRuntime(qInstance);
+      if(runtime == null)
+      {
+         LOG.warn("Quick Search runtime service started, but no produced Quick Search QBit was found in the QInstance");
+         return;
+      }
+      runtime.setStartInstance(qInstance);
+      runtime.start();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void stop()
+   {
+      if(runtime != null)
+      {
+         runtime.close();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Find the runtime on the instance's Quick Search QBit (not through
+    ** QContext, which may not be set when the launcher starts services).
+    *******************************************************************************/
+   static QuickSearchRuntime findRuntime(QInstance qInstance)
+   {
+      if(qInstance == null || qInstance.getQBits() == null)
+      {
+         return (null);
+      }
+      for(QBitMetaData qBitMetaData : qInstance.getQBits().values())
+      {
+         if(qBitMetaData.getConfig() instanceof QuickSearchQBitConfig config && config.getRuntime() != null)
+         {
+            return (config.getRuntime());
+         }
+      }
+      return (null);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchStartupMode.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+/*******************************************************************************
+ ** What happens when OpenSearch cannot be reached while the QBit starts.
+ **
+ ** FAIL_FAST - produce() and the runtime service throw, so the host does not
+ **             boot with search silently broken. The default.
+ ** DEGRADED  - the failure is logged; the host boots; indexing and search
+ **             retry the connection on first use and report through the
+ **             health check. Intended for demo profiles that may run without
+ **             an OpenSearch cluster.
+ *******************************************************************************/
+public enum QuickSearchStartupMode
+{
+   FAIL_FAST,
+   DEGRADED
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchTlsConfig.java

@@ -0,0 +1,376 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.io.Serializable;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+
+/*******************************************************************************
+ ** TLS trust and client-certificate settings for the OpenSearch connection.
+ **
+ ** By default the JVM trust store and hostname verification are used. Supply
+ ** caCertificatePath (a PEM bundle) or trustStorePath to trust a private CA;
+ ** supply keyStorePath for client-certificate (mTLS) authentication.
+ ** insecureSkipVerify disables certificate and hostname checks and is refused
+ ** for non-loopback hosts unless allowInsecureInProduction is also set.
+ ** hostnameVerification=false keeps certificate checks but skips the host name
+ ** check; it is allowed for any host and logs a warning.
+ **
+ ** Path and password values may be ${env.X} / ${prop.X} references; they are
+ ** interpreted when the transport is built, never stored resolved. A path that
+ ** resolves to an empty value is treated as missing.
+ *******************************************************************************/
+public class QuickSearchTlsConfig implements Serializable
+{
+   private String  caCertificatePath;
+   private String  trustStorePath;
+   private String  trustStorePassword;
+   private String  trustStoreType;
+   private String  keyStorePath;
+   private String  keyStorePassword;
+   private String  keyStoreType;
+   private Boolean hostnameVerification      = true;
+   private Boolean insecureSkipVerify        = false;
+   private Boolean allowInsecureInProduction = false;
+
+
+
+   /*******************************************************************************
+    ** Whether any custom trust or key material is configured.
+    *******************************************************************************/
+   @JsonIgnore
+   public boolean hasCustomTrustOrKeyMaterial()
+   {
+      return (caCertificatePath != null || trustStorePath != null || keyStorePath != null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for caCertificatePath
+    *******************************************************************************/
+   public String getCaCertificatePath()
+   {
+      return (this.caCertificatePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for caCertificatePath
+    *******************************************************************************/
+   public void setCaCertificatePath(String caCertificatePath)
+   {
+      this.caCertificatePath = caCertificatePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for caCertificatePath
+    *******************************************************************************/
+   public QuickSearchTlsConfig withCaCertificatePath(String caCertificatePath)
+   {
+      this.caCertificatePath = caCertificatePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trustStorePath
+    *******************************************************************************/
+   public String getTrustStorePath()
+   {
+      return (this.trustStorePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for trustStorePath
+    *******************************************************************************/
+   public void setTrustStorePath(String trustStorePath)
+   {
+      this.trustStorePath = trustStorePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trustStorePath
+    *******************************************************************************/
+   public QuickSearchTlsConfig withTrustStorePath(String trustStorePath)
+   {
+      this.trustStorePath = trustStorePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trustStorePassword. Hidden from JSON so the secret (or its
+    ** reference) never leaves the JVM.
+    *******************************************************************************/
+   @JsonIgnore
+   public String getTrustStorePassword()
+   {
+      return (this.trustStorePassword);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for trustStorePassword
+    *******************************************************************************/
+   public void setTrustStorePassword(String trustStorePassword)
+   {
+      this.trustStorePassword = trustStorePassword;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trustStorePassword
+    *******************************************************************************/
+   public QuickSearchTlsConfig withTrustStorePassword(String trustStorePassword)
+   {
+      this.trustStorePassword = trustStorePassword;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trustStoreType (JKS or PKCS12; defaults to the JVM default)
+    *******************************************************************************/
+   public String getTrustStoreType()
+   {
+      return (this.trustStoreType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for trustStoreType
+    *******************************************************************************/
+   public void setTrustStoreType(String trustStoreType)
+   {
+      this.trustStoreType = trustStoreType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trustStoreType
+    *******************************************************************************/
+   public QuickSearchTlsConfig withTrustStoreType(String trustStoreType)
+   {
+      this.trustStoreType = trustStoreType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyStorePath
+    *******************************************************************************/
+   public String getKeyStorePath()
+   {
+      return (this.keyStorePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyStorePath
+    *******************************************************************************/
+   public void setKeyStorePath(String keyStorePath)
+   {
+      this.keyStorePath = keyStorePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyStorePath
+    *******************************************************************************/
+   public QuickSearchTlsConfig withKeyStorePath(String keyStorePath)
+   {
+      this.keyStorePath = keyStorePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyStorePassword. Hidden from JSON.
+    *******************************************************************************/
+   @JsonIgnore
+   public String getKeyStorePassword()
+   {
+      return (this.keyStorePassword);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyStorePassword
+    *******************************************************************************/
+   public void setKeyStorePassword(String keyStorePassword)
+   {
+      this.keyStorePassword = keyStorePassword;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyStorePassword
+    *******************************************************************************/
+   public QuickSearchTlsConfig withKeyStorePassword(String keyStorePassword)
+   {
+      this.keyStorePassword = keyStorePassword;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyStoreType
+    *******************************************************************************/
+   public String getKeyStoreType()
+   {
+      return (this.keyStoreType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyStoreType
+    *******************************************************************************/
+   public void setKeyStoreType(String keyStoreType)
+   {
+      this.keyStoreType = keyStoreType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyStoreType
+    *******************************************************************************/
+   public QuickSearchTlsConfig withKeyStoreType(String keyStoreType)
+   {
+      this.keyStoreType = keyStoreType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hostnameVerification
+    *******************************************************************************/
+   public Boolean getHostnameVerification()
+   {
+      return (this.hostnameVerification);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hostnameVerification
+    *******************************************************************************/
+   public void setHostnameVerification(Boolean hostnameVerification)
+   {
+      this.hostnameVerification = hostnameVerification;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hostnameVerification
+    *******************************************************************************/
+   public QuickSearchTlsConfig withHostnameVerification(Boolean hostnameVerification)
+   {
+      this.hostnameVerification = hostnameVerification;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for insecureSkipVerify
+    *******************************************************************************/
+   public Boolean getInsecureSkipVerify()
+   {
+      return (this.insecureSkipVerify);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for insecureSkipVerify
+    *******************************************************************************/
+   public void setInsecureSkipVerify(Boolean insecureSkipVerify)
+   {
+      this.insecureSkipVerify = insecureSkipVerify;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for insecureSkipVerify
+    *******************************************************************************/
+   public QuickSearchTlsConfig withInsecureSkipVerify(Boolean insecureSkipVerify)
+   {
+      this.insecureSkipVerify = insecureSkipVerify;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for allowInsecureInProduction
+    *******************************************************************************/
+   public Boolean getAllowInsecureInProduction()
+   {
+      return (this.allowInsecureInProduction);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowInsecureInProduction
+    *******************************************************************************/
+   public void setAllowInsecureInProduction(Boolean allowInsecureInProduction)
+   {
+      this.allowInsecureInProduction = allowInsecureInProduction;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowInsecureInProduction
+    *******************************************************************************/
+   public QuickSearchTlsConfig withAllowInsecureInProduction(Boolean allowInsecureInProduction)
+   {
+      this.allowInsecureInProduction = allowInsecureInProduction;
+      return (this);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java

@@ -18,6 +18,8 @@ package com.kingsrook.qbits.quicksearch;
 
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
 
 
 /*******************************************************************************
@@ -26,6 +28,12 @@ import java.util.Map;
  ** Describes which fields to index, optional per-field boost weights,
  ** optional per-field label-inclusion flags, basepull scheduling, and
  ** whether the table is enabled by default.
+ **
+ ** Code (annotations, searchableTables) builds these as defaults; at runtime
+ ** the effective configs are built from the quick_search_index and
+ ** quick_search_index_field records (QuickSearchConfigStore), which add each
+ ** field's role, analyzer and snippet flag, the table's enabled flag and its
+ ** index row id.
  *******************************************************************************/
 public class QuickSearchableTableConfig
 {
@@ -39,6 +47,12 @@ public class QuickSearchableTableConfig
    private Boolean             enabledByDefault;
    private String              recordLabelFormat;
    private List<String>        recordLabelFields;
+   private Integer             maxFieldLength;
+   private Map<String, String>  fieldRoles;
+   private Map<String, String>  fieldAnalyzers;
+   private Map<String, Boolean> fieldIncludeInSnippet;
+   private Boolean             enabled;
+   private Integer             quickSearchIndexId;
 
 
 
@@ -350,4 +364,253 @@ public class QuickSearchableTableConfig
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxFieldLength
+    *******************************************************************************/
+   public Integer getMaxFieldLength()
+   {
+      return (this.maxFieldLength);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxFieldLength
+    *******************************************************************************/
+   public void setMaxFieldLength(Integer maxFieldLength)
+   {
+      this.maxFieldLength = maxFieldLength;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxFieldLength
+    *******************************************************************************/
+   public QuickSearchableTableConfig withMaxFieldLength(Integer maxFieldLength)
+   {
+      this.maxFieldLength = maxFieldLength;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fieldRoles
+    *******************************************************************************/
+   public Map<String, String> getFieldRoles()
+   {
+      return (this.fieldRoles);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldRoles
+    *******************************************************************************/
+   public void setFieldRoles(Map<String, String> fieldRoles)
+   {
+      this.fieldRoles = fieldRoles;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldRoles
+    *******************************************************************************/
+   public QuickSearchableTableConfig withFieldRoles(Map<String, String> fieldRoles)
+   {
+      this.fieldRoles = fieldRoles;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldAnalyzers
+    *******************************************************************************/
+   public Map<String, String> getFieldAnalyzers()
+   {
+      return (this.fieldAnalyzers);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldAnalyzers
+    *******************************************************************************/
+   public void setFieldAnalyzers(Map<String, String> fieldAnalyzers)
+   {
+      this.fieldAnalyzers = fieldAnalyzers;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldAnalyzers
+    *******************************************************************************/
+   public QuickSearchableTableConfig withFieldAnalyzers(Map<String, String> fieldAnalyzers)
+   {
+      this.fieldAnalyzers = fieldAnalyzers;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for fieldIncludeInSnippet
+    *******************************************************************************/
+   public Map<String, Boolean> getFieldIncludeInSnippet()
+   {
+      return (this.fieldIncludeInSnippet);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldIncludeInSnippet
+    *******************************************************************************/
+   public void setFieldIncludeInSnippet(Map<String, Boolean> fieldIncludeInSnippet)
+   {
+      this.fieldIncludeInSnippet = fieldIncludeInSnippet;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldIncludeInSnippet
+    *******************************************************************************/
+   public QuickSearchableTableConfig withFieldIncludeInSnippet(Map<String, Boolean> fieldIncludeInSnippet)
+   {
+      this.fieldIncludeInSnippet = fieldIncludeInSnippet;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for enabled
+    *******************************************************************************/
+   public Boolean getEnabled()
+   {
+      return (this.enabled);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for enabled
+    *******************************************************************************/
+   public void setEnabled(Boolean enabled)
+   {
+      this.enabled = enabled;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for enabled
+    *******************************************************************************/
+   public QuickSearchableTableConfig withEnabled(Boolean enabled)
+   {
+      this.enabled = enabled;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for quickSearchIndexId
+    *******************************************************************************/
+   public Integer getQuickSearchIndexId()
+   {
+      return (this.quickSearchIndexId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for quickSearchIndexId
+    *******************************************************************************/
+   public void setQuickSearchIndexId(Integer quickSearchIndexId)
+   {
+      this.quickSearchIndexId = quickSearchIndexId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for quickSearchIndexId
+    *******************************************************************************/
+   public QuickSearchableTableConfig withQuickSearchIndexId(Integer quickSearchIndexId)
+   {
+      this.quickSearchIndexId = quickSearchIndexId;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** A field's role; BODY when none is set.
+    *******************************************************************************/
+   public QuickSearchFieldRole getRole(String fieldName)
+   {
+      return (QuickSearchFieldRole.fromName(fieldRoles == null ? null : fieldRoles.get(fieldName), QuickSearchFieldRole.BODY));
+   }
+
+
+
+   /*******************************************************************************
+    ** A field's analyzer: the one set, else EXACT for an EXACT field, WORD for a
+    ** KEYWORD field and PREFIX for any other.
+    *******************************************************************************/
+   public QuickSearchFieldAnalyzer getAnalyzer(String fieldName)
+   {
+      QuickSearchFieldRole     role     = getRole(fieldName);
+      QuickSearchFieldAnalyzer fallback = role == QuickSearchFieldRole.EXACT ? QuickSearchFieldAnalyzer.EXACT
+         : (role == QuickSearchFieldRole.KEYWORD ? QuickSearchFieldAnalyzer.WORD : QuickSearchFieldAnalyzer.PREFIX);
+      if(role == QuickSearchFieldRole.EXACT)
+      {
+         return (QuickSearchFieldAnalyzer.EXACT);
+      }
+      return (QuickSearchFieldAnalyzer.fromName(fieldAnalyzers == null ? null : fieldAnalyzers.get(fieldName), fallback));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field's text goes into the highlighted snippet (searchableText).
+    ** Defaults to true, except for KEYWORD and EXACT fields and EXACT-analyzed
+    ** fields, which are matched on their own instead.
+    *******************************************************************************/
+   public boolean isIncludedInSnippet(String fieldName)
+   {
+      if(getAnalyzer(fieldName) == QuickSearchFieldAnalyzer.EXACT)
+      {
+         return (false);
+      }
+      Boolean flag = fieldIncludeInSnippet == null ? null : fieldIncludeInSnippet.get(fieldName);
+      if(flag != null)
+      {
+         return (flag);
+      }
+      QuickSearchFieldRole role = getRole(fieldName);
+      return (role != QuickSearchFieldRole.KEYWORD && role != QuickSearchFieldRole.EXACT);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether every field is plain: in the snippet with the PREFIX analyzer, so
+    ** documents and queries look exactly as they did before roles existed.
+    *******************************************************************************/
+   public boolean isAllPlainFields()
+   {
+      for(String fieldName : searchableFields == null ? List.<String>of() : searchableFields)
+      {
+         if(!isIncludedInSnippet(fieldName) || getAnalyzer(fieldName) != QuickSearchFieldAnalyzer.PREFIX)
+         {
+            return (false);
+         }
+      }
+      return (true);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java

@@ -17,116 +17,559 @@
 package com.kingsrook.qbits.quicksearch.actions;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import org.opensearch.client.opensearch._types.OpenSearchException;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.opensearch.client.opensearch.core.search.Hit;
+import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** Action that executes a quick-search query against OpenSearch and returns
- ** normalized, paginated results.
+ ** Permission-aware, bounded search over the Quick Search index.
  **
- ** Validates and normalizes the input, delegates to the
- ** QuickSearchOpenSearchClient, maps each hit to a QuickSearchResult, and
- ** computes pagination metadata (totalHits, hasMore).
+ ** Only tables the session may read (table READ permission), that are
+ ** configured and enabled, are searched. With applyRecordSecurityLocks (the
+ ** default) hits are re-read through QueryAction by primary key, so record
+ ** security locks apply and no label or snippet is returned for a record the
+ ** session cannot query.
+ **
+ ** limit is capped at maxSearchLimit and offset + limit at OpenSearch's result
+ ** window. tableNames restricts the search; limitPerTable searches each table
+ ** separately and returns up to that many hits per table.
+ **
+ ** offset counts results the caller can see. Without locks it is passed
+ ** straight to OpenSearch. With locks, hits are read from position 0 in
+ ** batches that double in size, filtered, the first offset accessible results
+ ** are skipped and limit are kept, so consecutive pages neither overlap nor
+ ** skip an accessible hit. No request reads past MAX_RESULT_WINDOW (10,000
+ ** raw hits), so accessible results beyond it are not reachable.
+ **
+ ** With locks, each scan has a hard budget of raw hits, read in batches of
+ ** at most MAX_LOCKED_BATCH_SIZE, and of OpenSearch requests, each followed
+ ** by one QueryAction per table in the batch. A single-list search gets
+ ** MAX_RESULT_WINDOW hits and MAX_LOCKED_ROUND_TRIPS requests. In per-table
+ ** mode each of the T tables gets its own equal share,
+ ** max(MAX_RESULT_WINDOW / T, min(MAX_RESULT_WINDOW, 2 x limitPerTable)) hits
+ ** and max(2, MAX_LOCKED_ROUND_TRIPS / T) requests, so a fully locked table
+ ** cannot starve another and one call reads at most
+ ** max(MAX_RESULT_WINDOW, T x 2 x limitPerTable) raw hits over at most
+ ** max(MAX_LOCKED_ROUND_TRIPS, 2 x T) requests; offset never raises it.
+ ** A budget ends a scan like the result window does: hasMore is false and
+ ** the total is a lower bound, so in per-table mode paging within a table
+ ** stops past max(MAX_RESULT_WINDOW / T, 2 x limitPerTable) raw hits. Hits
+ ** are deduplicated by table and record id within a call; batches are
+ ** separate from/size requests without a tiebreak sort, so a hit that moves
+ ** between batches during a refresh can still be missed.
  *******************************************************************************/
 public class QuickSearchAction
 {
-   private static final QLogger LOG             = QLogger.getLogger(QuickSearchAction.class);
-   private static final int     DEFAULT_LIMIT   = 25;
-   private static final int     DEFAULT_OFFSET  = 0;
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchAction.class);
+
+   public static final int DEFAULT_LIMIT       = 25;
+   public static final int MIN_TERM_LENGTH     = 2;
+   public static final int MAX_TERM_LENGTH     = 100;
+   public static final int MAX_LIMIT_PER_TABLE = 100;
+
+   public static final int MAX_LOCKED_ROUND_TRIPS = 32;
+   public static final int MAX_LOCKED_BATCH_SIZE  = 1_000;
 
 
 
    /*******************************************************************************
-    ** Execute a quick-search for the given input.
     **
-    ** Returns an empty output (totalHits=0, hasMore=false, empty results list)
-    ** when the search term is null or blank.  Otherwise normalizes the term,
-    ** applies default pagination values, calls the OpenSearch client, and maps
-    ** hits to QuickSearchResult objects.
-    **
-    ** @param input the search parameters; must not be null
-    ** @return populated QuickSearchOutput
-    ** @throws QException if the OpenSearch call fails
     *******************************************************************************/
    public QuickSearchOutput execute(QuickSearchInput input) throws QException
    {
       String searchTerm = input.getSearchTerm();
-
       if(searchTerm == null || searchTerm.isBlank())
       {
-         LOG.debug("QuickSearch called with blank/null search term - returning empty output");
-         return (new QuickSearchOutput()
-            .withResults(Collections.emptyList())
-            .withTotalHits(0L)
-            .withHasMore(false));
+         return (emptyOutput());
       }
 
       String normalizedTerm = IndexingUtils.normalizeSearchText(searchTerm);
+      if(normalizedTerm.length() < MIN_TERM_LENGTH)
+      {
+         return (emptyOutput());
+      }
+      if(normalizedTerm.length() > MAX_TERM_LENGTH)
+      {
+         throw (new QException("Search term is too long (maximum " + MAX_TERM_LENGTH + " characters)"));
+      }
 
-      int limit  = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : input.getLimit();
-      int offset = (input.getOffset() == null || input.getOffset() < 0) ? DEFAULT_OFFSET : input.getOffset();
-
-      QuickSearchOpenSearchClient       client       = QuickSearchQBitContext.getClient();
-      List<QuickSearchableTableConfig>  tableConfigs = QuickSearchQBitContext.getDiscoveredTables();
+      QuickSearchRuntime               runtime      = QuickSearchRuntime.get();
+      QuickSearchQBitConfig            config       = runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig();
+      QuickSearchOpenSearchClient      client       = runtime != null ? runtime.getClient() : QuickSearchQBitContext.getClient();
+      List<QuickSearchableTableConfig> tableConfigs = runtime != null ? runtime.getDiscoveredTables() : QuickSearchQBitContext.getDiscoveredTables();
 
       if(client == null)
       {
          throw new QException("QuickSearch client is not initialized. The QBit may not have connected to OpenSearch during startup.");
       }
+      if(tableConfigs == null)
+      {
+         tableConfigs = Collections.emptyList();
+      }
 
-      LOG.debug("Executing quick search", "term", normalizedTerm, "tableName", input.getTableName(), "limit", limit, "offset", offset);
+      int maxLimit = (config == null || config.getMaxSearchLimit() == null) ? MAX_LIMIT_PER_TABLE : config.getMaxSearchLimit();
+      int limit    = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : Math.min(input.getLimit(), maxLimit);
+      int offset   = (input.getOffset() == null || input.getOffset() < 0) ? 0 : input.getOffset();
 
-      SearchResponse<OpenSearchDocument> response = client.search(normalizedTerm, input.getTableName(), limit, offset, tableConfigs);
+      if(offset >= QuickSearchOpenSearchClient.MAX_RESULT_WINDOW)
+      {
+         return (emptyOutput());
+      }
 
-      List<QuickSearchResult> results = new ArrayList<>();
+      List<String> allowedTables = resolveAllowedTables(input, runtime, tableConfigs);
+      if(allowedTables.isEmpty())
+      {
+         return (emptyOutput());
+      }
+
+      boolean applyLocks = config == null || Boolean.TRUE.equals(config.getApplyRecordSecurityLocks());
+
+      LOG.debug("Executing quick search", logPair("termLength", normalizedTerm.length()), logPair("tables", allowedTables.size()), logPair("limit", limit), logPair("offset", offset));
+
+      if(input.getLimitPerTable() != null && input.getLimitPerTable() > 0)
+      {
+         return (searchPerTable(client, normalizedTerm, allowedTables, offset, Math.min(input.getLimitPerTable(), maxLimit), tableConfigs, applyLocks, Boolean.TRUE.equals(input.getSinglePass())));
+      }
+
+      Page page = searchPage(client, normalizedTerm, allowedTables, offset, limit, tableConfigs, applyLocks, new ScanBudget(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW, MAX_LOCKED_ROUND_TRIPS));
+      return (page.toOutput());
+   }
+
+
+
+   /*******************************************************************************
+    ** Search each allowed table separately, skipping offset results and
+    ** returning up to limitPerTable hits in each table.
+    ** singlePass bounds locked searches to one batch of twice the requested
+    ** page size; it never scans again to fill gaps left by inaccessible rows.
+    *******************************************************************************/
+   private QuickSearchOutput searchPerTable(QuickSearchOpenSearchClient client, String term, List<String> allowedTables, int offset, int limitPerTable, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks, boolean singlePass) throws QException
+   {
+      List<QuickSearchResult> results    = new ArrayList<>();
+      long                    totalHits  = 0;
+      boolean                 hasMore    = false;
+      boolean                 lowerBound = false;
+
+      int window     = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
+      int tableHits  = Math.max(window / allowedTables.size(), Math.min(window, 2 * limitPerTable));
+      int tableTrips = Math.max(2, MAX_LOCKED_ROUND_TRIPS / allowedTables.size());
+      if(singlePass)
+      {
+         tableHits = Math.min(window, 2 * limitPerTable);
+         tableTrips = 1;
+      }
+
+      for(String tableName : allowedTables)
+      {
+         Page page = searchPage(client, term, List.of(tableName), offset, limitPerTable, tableConfigs, applyLocks, new ScanBudget(tableHits, tableTrips));
+         results.addAll(page.results());
+         totalHits += page.totalHits();
+         hasMore |= page.hasMore();
+         lowerBound |= page.lowerBound();
+      }
+
+      return (new QuickSearchOutput().withResults(results).withTotalHits(totalHits).withTotalHitsIsLowerBound(lowerBound).withHasMore(hasMore));
+   }
+
+
+
+   /*******************************************************************************
+    ** One page over the given tables: skip offset results, keep up to limit.
+    ** Without locks offset goes straight to OpenSearch; with locks offset and
+    ** limit count accessible results within one scan budget (see the class
+    ** comment). Every request keeps from + size within MAX_RESULT_WINDOW.
+    *******************************************************************************/
+   private Page searchPage(QuickSearchOpenSearchClient client, String term, List<String> tables, int offset, int limit, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks, ScanBudget budget) throws QException
+   {
+      int window   = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
+      int pageSize = Math.min(limit, window - offset);
+      if(pageSize <= 0)
+      {
+         return (new Page(Collections.emptyList(), 0L, false, false));
+      }
+
+      if(!applyLocks)
+      {
+         SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, pageSize, offset, tableConfigs, null);
+         List<QuickSearchResult>            results  = toResults(response, tableConfigs);
+
+         long    totalHits  = response.hits().total() != null ? response.hits().total().value() : results.size();
+         boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
+         return (new Page(results, totalHits, lowerBound, (offset + results.size()) < totalHits));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // read candidates from position 0 until offset + pageSize accessible     //
+      // results are known, the hits run out, the result window is reached, or  //
+      // the scan budget is spent. each batch over-reads 2x what is still       //
+      // needed and at least doubles the last one, up to MAX_LOCKED_BATCH_SIZE, //
+      // and the last round trip takes up to MAX_LOCKED_BATCH_SIZE of the rest. //
+      ////////////////////////////////////////////////////////////////////////////
+      int                     target     = offset + pageSize;
+      List<QuickSearchResult> accessible = new ArrayList<>();
+      int                     position   = 0;
+      int                     batchSize  = 0;
+      boolean                 exhausted  = false;
+      boolean                 rawFloor   = false;
+      Set<String>             seen       = new HashSet<>();
+      String                  preference = "quick-search-" + UUID.randomUUID();
+
+      while(accessible.size() < target && !exhausted && position < window && budget.hasRemaining())
+      {
+         int ceiling = Math.min(MAX_LOCKED_BATCH_SIZE, Math.min(window - position, budget.remainingHits));
+         batchSize = budget.remainingRoundTrips == 1 ? ceiling : Math.min(ceiling, Math.max(2 * (target - accessible.size()), 2 * batchSize));
+
+         SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, batchSize, position, tableConfigs, preference);
+         int                                rawHits  = response.hits().hits().size();
+         long                               rawTotal = response.hits().total() != null ? response.hits().total().value() : position + rawHits;
+         rawFloor = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
+
+         List<QuickSearchResult> candidates = new ArrayList<>();
+         for(QuickSearchResult result : toResults(response, tableConfigs))
+         {
+            if(seen.add(result.getTableName() + ":" + result.getRecordId()))
+            {
+               candidates.add(result);
+            }
+         }
+         accessible.addAll(filterByRecordAccess(candidates, tableConfigs));
+         position += rawHits;
+         budget.spend(rawHits);
+         exhausted = rawHits < batchSize || position >= rawTotal;
+      }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // the raw total would reveal how many matching rows the session may not  //
+      // see; report only what was verified accessible. hasMore says only that  //
+      // more candidates exist, so a caller can page on to accessible rows.     //
+      ////////////////////////////////////////////////////////////////////////////
+      boolean truncated = accessible.size() > target;
+      boolean hasMore   = truncated || (!exhausted && position < window && budget.hasRemaining());
+
+      List<QuickSearchResult> results = accessible.size() > offset
+         ? new ArrayList<>(accessible.subList(offset, Math.min(target, accessible.size())))
+         : new ArrayList<>();
+
+      return (new Page(results, (long) accessible.size(), !exhausted || rawFloor, hasMore));
+   }
+
+
+
+   /*******************************************************************************
+    ** Requested tables (tableNames, else tableName, else all configured) that
+    ** are configured, enabled, and readable by the session.
+    *******************************************************************************/
+   private List<String> resolveAllowedTables(QuickSearchInput input, QuickSearchRuntime runtime, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      Set<String> configured = new HashSet<>();
+      for(QuickSearchableTableConfig tableConfig : tableConfigs)
+      {
+         configured.add(tableConfig.getTableName());
+      }
+
+      List<String> requested = new ArrayList<>();
+      if(!CollectionUtils.nullSafeIsEmpty(input.getTableNames()))
+      {
+         requested.addAll(input.getTableNames());
+      }
+      else if(StringUtils.hasContent(input.getTableName()))
+      {
+         requested.add(input.getTableName());
+      }
+      else
+      {
+         requested.addAll(configured);
+      }
+
+      boolean      checkPermissions = QContext.getQInstance() != null && QContext.getQSession() != null;
+      List<String> allowed          = new ArrayList<>();
+      for(String tableName : new LinkedHashSet<>(requested))
+      {
+         if(!configured.contains(tableName) || QuickSearchFieldRules.hasProtectedPrimaryKey(tableName))
+         {
+            continue;
+         }
+         if(runtime != null && !runtime.isTableEnabled(tableName))
+         {
+            continue;
+         }
+         if(checkPermissions && !PermissionsHelper.hasTablePermission(input, tableName, TablePermissionSubType.READ))
+         {
+            continue;
+         }
+         allowed.add(tableName);
+      }
+
+      return (allowed);
+   }
+
+
+
+   /*******************************************************************************
+    ** Keep only hits whose record the session can query (record security
+    ** locks). Tables not in the QInstance are left as they are.
+    *******************************************************************************/
+   private List<QuickSearchResult> filterByRecordAccess(List<QuickSearchResult> results, List<QuickSearchableTableConfig> tableConfigs) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(qInstance == null || QContext.getQSession() == null || results.isEmpty())
+      {
+         return (results);
+      }
+
+      Map<String, List<QuickSearchResult>> byTable = new LinkedHashMap<>();
+      for(QuickSearchResult result : results)
+      {
+         byTable.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
+      }
+
+      Set<QuickSearchResult> accessible = new HashSet<>();
+      for(Map.Entry<String, List<QuickSearchResult>> entry : byTable.entrySet())
+      {
+         QTableMetaData table = qInstance.getTable(entry.getKey());
+         if(table == null)
+         {
+            accessible.addAll(entry.getValue());
+            continue;
+         }
+
+         List<Serializable> ids = new ArrayList<>();
+         for(QuickSearchResult result : entry.getValue())
+         {
+            ids.add(result.getRecordId());
+         }
+
+         Set<String> found = new HashSet<>();
+         for(int start = 0; start < ids.size(); start += MAX_LOCKED_BATCH_SIZE)
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table.getName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_LOCKED_BATCH_SIZE)))));
+
+            for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+            {
+               found.add(record.getValueString(table.getPrimaryKeyField()));
+            }
+         }
+
+         for(QuickSearchResult result : entry.getValue())
+         {
+            if(found.contains(result.getRecordId()))
+            {
+               accessible.add(result);
+            }
+         }
+      }
+
+      List<QuickSearchResult> filtered = new ArrayList<>();
+      for(QuickSearchResult result : results)
+      {
+         if(accessible.contains(result))
+         {
+            filtered.add(result);
+         }
+      }
+      return (filtered);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QuickSearchResult> toResults(SearchResponse<OpenSearchDocument> response, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      QInstance               qInstance = QContext.getQInstance();
+      List<QuickSearchResult> results   = new ArrayList<>();
 
       for(Hit<OpenSearchDocument> hit : response.hits().hits())
       {
          OpenSearchDocument source = hit.source();
+         if(source == null)
+         {
+            continue;
+         }
 
-         String highlightSnippet = null;
-         Map<String, List<String>> highlight = hit.highlight();
-
-         if(highlight != null)
+         String sourceTable = source.getSourceTable();
+         boolean filterFields = QuickSearchFieldRules.requiresFieldFiltering(sourceTable);
+         source = IndexingUtils.filterResult(source, tableConfigs.stream().filter(c -> sourceTable.equals(c.getTableName())).findFirst().orElse(null));
+         if(source == null)
+         {
+            continue;
+         }
+         String                    highlightSnippet = filterFields ? source.getSearchableText() : null;
+         Map<String, List<String>> highlight        = hit.highlight();
+         if(highlight != null && !filterFields)
          {
             List<String> fragments = highlight.get("searchableText");
-
             if(fragments != null && !fragments.isEmpty())
             {
                highlightSnippet = String.join("...", fragments);
             }
          }
 
-         Double rawScore = hit.score();
-         Float  score    = rawScore != null ? rawScore.floatValue() : null;
+         QTableMetaData table      = qInstance == null ? null : qInstance.getTable(source.getSourceTable());
+         String         tableLabel = table != null && StringUtils.hasContent(table.getLabel()) ? table.getLabel() : source.getSourceTable();
+         Double         rawScore   = hit.score();
 
          results.add(new QuickSearchResult()
             .withTableName(source.getSourceTable())
+            .withTableLabel(tableLabel)
             .withRecordId(source.getRecordId())
             .withRecordLabel(source.getRecordLabel())
-            .withScore(score)
-            .withHighlightSnippet(highlightSnippet));
+            .withScore(rawScore != null ? rawScore.floatValue() : null)
+            .withHighlightSnippet(highlightSnippet)
+            .withSubtitle(source.getSubtitle())
+            .withDisplayValues(source.getDisplayValues()));
       }
 
-      long    totalHits = response.hits().total() != null ? response.hits().total().value() : results.size();
-      boolean hasMore   = (offset + results.size()) < totalHits;
+      return (results);
+   }
 
-      return (new QuickSearchOutput()
-         .withResults(results)
-         .withTotalHits(totalHits)
-         .withHasMore(hasMore));
+
+
+   /*******************************************************************************
+    ** Run one search, surfacing an OpenSearch client error as a QException so
+    ** callers never see an unchecked OpenSearchException.
+    *******************************************************************************/
+   private static SearchResponse<OpenSearchDocument> runSearch(QuickSearchOpenSearchClient client, String term, List<String> tables, int fetchSize, int offset, List<QuickSearchableTableConfig> tableConfigs, String preference) throws QException
+   {
+      long start = System.currentTimeMillis();
+      try
+      {
+         return (client.search(term, tables, fetchSize, offset, tableConfigs, preference));
+      }
+      catch(OpenSearchException e)
+      {
+         QException wrapped = new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e);
+         recordFailure(tables, term, start, wrapped);
+         throw (wrapped);
+      }
+      catch(QException e)
+      {
+         recordFailure(tables, term, start, e);
+         throw (e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a failed search as a diagnostic event (the term's length only,
+    ** never the term).
+    *******************************************************************************/
+   private static void recordFailure(List<String> tables, String term, long start, Exception e)
+   {
+      DiagnosticsSupport.recordSearchFailure(QuickSearchRuntime.get(), QuickSearchDiagnosticEvent.SOURCE_QUICK_SEARCH_ACTION, tables, term,
+         System.currentTimeMillis() - start, e, "term length " + (term == null ? 0 : term.length()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Work budget for one locked scan: raw hits fetched and OpenSearch round
+    ** trips.
+    *******************************************************************************/
+   private static final class ScanBudget
+   {
+      private Integer remainingHits;
+      private Integer remainingRoundTrips;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       *******************************************************************************/
+      ScanBudget(Integer hits, Integer roundTrips)
+      {
+         this.remainingHits       = hits;
+         this.remainingRoundTrips = roundTrips;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      boolean hasRemaining()
+      {
+         return (remainingHits > 0 && remainingRoundTrips > 0);
+      }
+
+
+
+      /*******************************************************************************
+       ** Record one round trip that fetched the given number of raw hits.
+       *******************************************************************************/
+      void spend(int rawHits)
+      {
+         remainingHits -= rawHits;
+         remainingRoundTrips--;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** One page of results with its total and paging flags.
+    *******************************************************************************/
+   private record Page(List<QuickSearchResult> results, Long totalHits, Boolean lowerBound, Boolean hasMore)
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      QuickSearchOutput toOutput()
+      {
+         return (new QuickSearchOutput().withResults(results).withTotalHits(totalHits).withTotalHitsIsLowerBound(lowerBound).withHasMore(hasMore));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QuickSearchOutput emptyOutput()
+   {
+      return (new QuickSearchOutput().withResults(Collections.emptyList()).withTotalHits(0L).withTotalHitsIsLowerBound(false).withHasMore(false));
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchInput.java

@@ -17,18 +17,46 @@
 package com.kingsrook.qbits.quicksearch.actions;
 
 
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+
+
 /*******************************************************************************
  ** Input DTO for the QuickSearch action.
  **
  ** Carries the search term plus optional filtering and pagination parameters.
  *******************************************************************************/
-public class QuickSearchInput
+public class QuickSearchInput extends AbstractActionInput
 {
 
    private String  searchTerm;
    private String  tableName;
    private Integer limit;
    private Integer offset;
+   private List<String> tableNames;
+   private Integer limitPerTable;
+   private Boolean singlePass = false;
+
+
+
+   /*******************************************************************************
+    ** Whether per-table searches stop after one bounded candidate batch.
+    *******************************************************************************/
+   public Boolean getSinglePass()
+   {
+      return (singlePass);
+   }
+
+
+
+   /*******************************************************************************
+    ** Opt in to bounded typeahead latency, accepting partially filled pages.
+    *******************************************************************************/
+   public QuickSearchInput withSinglePass(Boolean singlePass)
+   {
+      this.singlePass = singlePass;
+      return (this);
+   }
 
 
    /*******************************************************************************
@@ -116,7 +144,14 @@ public class QuickSearchInput
 
 
    /*******************************************************************************
-    ** Getter for offset.
+    ** Getter for offset: the number of results to skip, counting only results
+    ** the session can see (with record security locks on, hits filtered out
+    ** by a lock are not counted). With limitPerTable it applies within each
+    ** table. Results are reachable only within the first 10,000 raw hits; in
+    ** per-table mode with locks each table gets an equal share of that scan
+    ** budget, so paging within a table stops past
+    ** max(10,000 / T, 2 x limitPerTable) raw hits (hasMore turns false and
+    ** totalHitsIsLowerBound stays true).
     *******************************************************************************/
    public Integer getOffset()
    {
@@ -142,4 +177,65 @@ public class QuickSearchInput
       return this;
    }
 
+
+   /*******************************************************************************
+    ** Getter for tableNames (restrict the search to these tables)
+    *******************************************************************************/
+   public List<String> getTableNames()
+   {
+      return tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableNames
+    *******************************************************************************/
+   public void setTableNames(List<String> tableNames)
+   {
+      this.tableNames = tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableNames
+    *******************************************************************************/
+   public QuickSearchInput withTableNames(List<String> tableNames)
+   {
+      this.tableNames = tableNames;
+      return this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for limitPerTable (search each table separately, up to this many)
+    *******************************************************************************/
+   public Integer getLimitPerTable()
+   {
+      return limitPerTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for limitPerTable
+    *******************************************************************************/
+   public void setLimitPerTable(Integer limitPerTable)
+   {
+      this.limitPerTable = limitPerTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for limitPerTable
+    *******************************************************************************/
+   public QuickSearchInput withLimitPerTable(Integer limitPerTable)
+   {
+      this.limitPerTable = limitPerTable;
+      return this;
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchOutput.java

@@ -31,6 +31,7 @@ public class QuickSearchOutput
 
    private List<QuickSearchResult> results;
    private Long                    totalHits;
+   private Boolean totalHitsIsLowerBound = false;
    private Boolean                 hasMore;
 
 
@@ -117,4 +118,36 @@ public class QuickSearchOutput
       return this;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for totalHitsIsLowerBound: true when totalHits is a floor (hits
+    ** beyond the result window, or hits removed by permission filtering)
+    *******************************************************************************/
+   public Boolean getTotalHitsIsLowerBound()
+   {
+      return totalHitsIsLowerBound;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for totalHitsIsLowerBound
+    *******************************************************************************/
+   public void setTotalHitsIsLowerBound(Boolean totalHitsIsLowerBound)
+   {
+      this.totalHitsIsLowerBound = totalHitsIsLowerBound;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for totalHitsIsLowerBound
+    *******************************************************************************/
+   public QuickSearchOutput withTotalHitsIsLowerBound(Boolean totalHitsIsLowerBound)
+   {
+      this.totalHitsIsLowerBound = totalHitsIsLowerBound;
+      return this;
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchRecordSearchProvider.java

@@ -0,0 +1,285 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.actions;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlight;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlightMatch;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchTableHits;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Serves QQQ core record search (RecordSearchAction, POST /qqq/v1/search,
+ ** the Next UI global search) from the Quick Search index, for the tables
+ ** this QBit indexes. Registered on the QInstance by QuickSearchQBitProducer
+ ** when serveCoreRecordSearch is on (the default).
+ **
+ ** Core creates a new instance for every search, so this class holds no
+ ** state: it resolves the QBit's QuickSearchRuntime (client, tables, config)
+ ** from QContext.
+ **
+ ** A table is claimed when it is indexed by this QBit and enabled, and core
+ ** record search is not backing off after a recent failure. search() sends
+ ** one _msearch request with a top-N search per table (after skipping the
+ ** input's offset) and returns record ids, best first, each with its score
+ ** and highlights (plain text plus match ranges, from the indexed field
+ ** values), and each table's total hits; core re-reads the ids by primary key
+ ** through QueryAction, so table permissions, record security locks and
+ ** record labels are core's job, and ids of deleted records drop out (core
+ ** also keeps only highlights of fields the session can see). The index has
+ ** no record modify dates, so a RECENT sort gets relevance order. Each table gets
+ ** IDS_PER_LIMIT_MULTIPLIER x limitPerTable ids (at most
+ ** RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE) so dropouts do not leave
+ ** the table short.
+ **
+ ** The request is bounded by recordSearchTimeoutMillis. Any failure throws,
+ ** and core then searches the claimed tables with its own LIKE search; the
+ ** failure also starts a short backoff (QuickSearchRuntime
+ ** RECORD_SEARCH_BACKOFF_MILLIS) during which no tables are claimed.
+ *******************************************************************************/
+public class QuickSearchRecordSearchProvider implements RecordSearchProviderInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordSearchProvider.class);
+
+   public static final int IDS_PER_LIMIT_MULTIPLIER = 4;
+
+   private static final String FIELD_VALUES_PREFIX = "fieldValues.";
+
+
+
+   /*******************************************************************************
+    ** Claim tables this QBit indexes and that are enabled, unless core record
+    ** search is off or backing off. Cheap: an in-memory lookup plus the
+    ** runtime's cached enabled flag.
+    *******************************************************************************/
+   @Override
+   public boolean claimsTable(QTableMetaData table)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || table == null || !Boolean.TRUE.equals(runtime.getConfig().getServeCoreRecordSearch()))
+      {
+         return (false);
+      }
+
+      if(runtime.getTableConfig(table.getName()) == null || QuickSearchFieldRules.hasProtectedPrimaryKey(table.getName()))
+      {
+         return (false);
+      }
+
+      if(runtime.isRecordSearchBackingOff())
+      {
+         runtime.countBackoffDeclinedClaim();
+         return (false);
+      }
+
+      return (runtime.isTableEnabled(table.getName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Ids of the best matches in each requested table, in relevance order.
+    *******************************************************************************/
+   @Override
+   public RecordSearchOutput search(RecordSearchInput input) throws QException
+   {
+      QuickSearchRuntime    runtime = QuickSearchRuntime.getOrThrow();
+      QuickSearchQBitConfig config  = runtime.getConfig();
+
+      List<RecordSearchResult> results = new ArrayList<>();
+      String                   term    = IndexingUtils.normalizeSearchText(input.getSearchTerm());
+      if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         return (new RecordSearchOutput().withResults(results));
+      }
+
+      List<String> tableNames = new ArrayList<>();
+      for(String tableName : new LinkedHashSet<>(CollectionUtils.nonNullList(input.getTableNames())))
+      {
+         if(runtime.getTableConfig(tableName) != null
+            && !QuickSearchFieldRules.hasProtectedPrimaryKey(tableName))
+         {
+            tableNames.add(tableName);
+         }
+      }
+      if(tableNames.isEmpty())
+      {
+         return (new RecordSearchOutput().withResults(results));
+      }
+
+      int limitPerTable = input.getLimitPerTable() == null || input.getLimitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : input.getLimitPerTable();
+      int idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * IDS_PER_LIMIT_MULTIPLIER);
+      int timeoutMillis = config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis();
+
+      int offset = input.getOffset() == null ? 0 : Math.max(0, input.getOffset());
+
+      Map<String, RecordSearchTableHits> hitsByTable;
+      long                               start = System.currentTimeMillis();
+      try
+      {
+         hitsByTable = runtime.getClient().searchRecordsPerTable(term, tableNames, idsPerTable, offset, true, runtime.getDiscoveredTables(), timeoutMillis);
+      }
+      catch(QException | RuntimeException e)
+      {
+         runtime.markRecordSearchFailed(e.getMessage());
+
+         //////////////////////////////////////////////////////////////////////
+         // the user's term is not recorded (it may hold personal data); its //
+         // length is, so a length-related failure is still visible          //
+         //////////////////////////////////////////////////////////////////////
+         DiagnosticsSupport.recordSearchFailure(runtime, QuickSearchDiagnosticEvent.SOURCE_CORE_RECORD_SEARCH, tableNames, term, System.currentTimeMillis() - start, e,
+            "term length " + term.length() + ", " + idsPerTable + " ids per table, timeout " + timeoutMillis + " ms; core searched these tables itself");
+         LOG.info("Quick Search record search failed; core searches these tables itself, and Quick Search claims none for a while", logPair("tables", tableNames),
+            logPair("backoffMillis", QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS), logPair("error", e.getMessage()));
+         throw (e instanceof QException qe ? qe : new QException("Quick Search record search failed: " + e.getMessage(), e));
+      }
+
+      Map<String, Integer> totalCountByTable = new HashMap<>();
+      for(String tableName : tableNames)
+      {
+         RecordSearchTableHits tableHits = hitsByTable.get(tableName);
+         if(tableHits == null)
+         {
+            continue;
+         }
+
+         if(tableHits.totalHits() != null)
+         {
+            totalCountByTable.put(tableName, (int) Math.min(Integer.MAX_VALUE, tableHits.totalHits()));
+         }
+
+         for(RecordSearchHit hit : CollectionUtils.nonNullList(tableHits.hits()))
+         {
+            List<RecordSearchHighlight> highlights = highlights(hit.highlights());
+            results.add(new RecordSearchResult()
+               .withTableName(tableName)
+               .withRecordId(hit.recordId())
+               .withScore(hit.score())
+               .withHighlights(highlights.isEmpty() ? null : highlights));
+         }
+      }
+
+      return (new RecordSearchOutput().withResults(results).withTotalCountByTable(totalCountByTable));
+   }
+
+
+
+   /*******************************************************************************
+    ** A hit's highlight fragments (by index field) as core record search
+    ** highlights: one per fieldValues.{fieldName} field, as plain text with
+    ** match ranges.  Other index fields, and fragments without a match, are
+    ** skipped.
+    *******************************************************************************/
+   public static List<RecordSearchHighlight> highlights(Map<String, List<String>> fragmentsByIndexField)
+   {
+      List<RecordSearchHighlight> highlights = new ArrayList<>();
+      for(Map.Entry<String, List<String>> entry : CollectionUtils.nonNullMap(fragmentsByIndexField).entrySet())
+      {
+         String indexField = entry.getKey();
+         if(indexField == null || !indexField.startsWith(FIELD_VALUES_PREFIX) || indexField.length() == FIELD_VALUES_PREFIX.length() || CollectionUtils.nullSafeIsEmpty(entry.getValue()))
+         {
+            continue;
+         }
+
+         RecordSearchHighlight highlight = highlight(indexField.substring(FIELD_VALUES_PREFIX.length()), entry.getValue().get(0));
+         if(highlight != null)
+         {
+            highlights.add(highlight);
+         }
+      }
+      return (highlights);
+   }
+
+
+
+   /*******************************************************************************
+    ** A highlight from a fragment whose matches are wrapped in the client's
+    ** pre/post tags: the fragment's text without the tags, and the ranges they
+    ** wrapped (adjacent ranges merged).  Null if the fragment is blank or has
+    ** no match.
+    *******************************************************************************/
+   public static RecordSearchHighlight highlight(String fieldName, String fragment)
+   {
+      if(fragment == null)
+      {
+         return (null);
+      }
+
+      StringBuilder                    text    = new StringBuilder();
+      List<RecordSearchHighlightMatch> matches = new ArrayList<>();
+      int                              open    = -1;
+      for(int i = 0; i < fragment.length(); i++)
+      {
+         char c = fragment.charAt(i);
+         if(c == QuickSearchOpenSearchClient.HIGHLIGHT_PRE_TAG.charAt(0))
+         {
+            open = text.length();
+         }
+         else if(c == QuickSearchOpenSearchClient.HIGHLIGHT_POST_TAG.charAt(0))
+         {
+            if(open >= 0 && text.length() > open)
+            {
+               RecordSearchHighlightMatch last = matches.isEmpty() ? null : matches.get(matches.size() - 1);
+               if(last != null && last.getEnd() == open)
+               {
+                  last.setEnd(text.length());
+               }
+               else
+               {
+                  matches.add(new RecordSearchHighlightMatch().withStart(open).withEnd(text.length()));
+               }
+            }
+            open = -1;
+         }
+         else
+         {
+            text.append(c);
+         }
+      }
+
+      if(matches.isEmpty() || text.toString().isBlank())
+      {
+         return (null);
+      }
+
+      return (new RecordSearchHighlight().withFieldName(fieldName).withText(text.toString()).withMatches(matches));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java

@@ -17,6 +17,9 @@
 package com.kingsrook.qbits.quicksearch.actions;
 
 
+import java.util.Map;
+
+
 /*******************************************************************************
  ** A single search hit returned by a QuickSearch query, representing one
  ** matching record with its relevance score and optional highlight snippet.
@@ -25,10 +28,13 @@ public class QuickSearchResult
 {
 
    private String tableName;
+   private String tableLabel;
    private String recordId;
    private String recordLabel;
    private Float  score;
    private String highlightSnippet;
+   private String subtitle;
+   private Map<String, String> displayValues;
 
 
    /*******************************************************************************
@@ -170,4 +176,97 @@ public class QuickSearchResult
       return this;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for tableLabel
+    *******************************************************************************/
+   public String getTableLabel()
+   {
+      return tableLabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableLabel
+    *******************************************************************************/
+   public void setTableLabel(String tableLabel)
+   {
+      this.tableLabel = tableLabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableLabel
+    *******************************************************************************/
+   public QuickSearchResult withTableLabel(String tableLabel)
+   {
+      this.tableLabel = tableLabel;
+      return this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for subtitle
+    *******************************************************************************/
+   public String getSubtitle()
+   {
+      return (this.subtitle);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for subtitle
+    *******************************************************************************/
+   public void setSubtitle(String subtitle)
+   {
+      this.subtitle = subtitle;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for subtitle
+    *******************************************************************************/
+   public QuickSearchResult withSubtitle(String subtitle)
+   {
+      this.subtitle = subtitle;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for displayValues
+    *******************************************************************************/
+   public Map<String, String> getDisplayValues()
+   {
+      return (this.displayValues);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for displayValues
+    *******************************************************************************/
+   public void setDisplayValues(Map<String, String> displayValues)
+   {
+      this.displayValues = displayValues;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for displayValues
+    *******************************************************************************/
+   public QuickSearchResult withDisplayValues(Map<String, String> displayValues)
+   {
+      this.displayValues = displayValues;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchApi.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension;
+
+
+/*******************************************************************************
+ ** Supported host navigation and UI API extension identity.
+ *******************************************************************************/
+public final class QuickSearchApi
+{
+   public static final String APP_NAME = QuickSearchAdminAppMetaDataProducer.APP_NAME;
+   public static final String HOME_HREF = "/app/" + APP_NAME;
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private QuickSearchApi()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The spec registration shared by production and contract generation.
+    *******************************************************************************/
+   public static MiddlewareSpecExtension extension()
+   {
+      return (new MiddlewareSpecExtension().withName("quick-search").withSemanticVersion("1.1.0")
+         .withDescription("Quick Search administration and suggestions").withSpec(QuickSearchStatusSpec.class).withSpec(QuickSearchSuggestSpec.class));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatus.java

@@ -0,0 +1,116 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema;
+import com.kingsrook.qqq.openapi.model.Schema;
+
+
+/*******************************************************************************
+ ** Immutable administrative snapshot. Null measurements are unknown, never zero.
+ ** Counts come from persisted diagnostics; countsCheckedAt identifies their age.
+ **
+ ** unassignedReplicaShards is the unassigned shard count when every primary is
+ ** active (so all unassigned shards are replicas), else null; targetReplicas and
+ ** replicaSettingsFailed report the index replica sizing (null when unknown).
+ *******************************************************************************/
+public record QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
+                                Integer unassignedReplicaShards, Integer targetReplicas, Boolean replicaSettingsFailed,
+                                Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
+                                Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref) implements ToSchema
+{
+   /*******************************************************************************
+    ** The 1.1.0-RC.14 shape, without the replica measurements (all unknown).
+    *******************************************************************************/
+   public QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
+                            Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
+                            Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref)
+   {
+      this(state, clusterReachable, clusterHealth, nodes, activeShards, unassignedShards, null, null, null, indexingPaused, indexes, pending, exhausted,
+         backlogTruncated, tablesTruncated, lastSuccessfulSync, lastError, checkedAt, homeHref);
+   }
+
+
+
+   /*******************************************************************************
+    ** Yellow only because replicas cannot be placed on a single-node cluster:
+    ** one node, every primary active and every unassigned shard a replica.
+    ** Such a cluster cannot do better, so it is healthy rather than degraded.
+    *******************************************************************************/
+   public static boolean isSingleNodeReplicaYellow(String clusterHealth, Integer nodes, Integer unassignedReplicaShards)
+   {
+      return ("yellow".equals(clusterHealth) && Integer.valueOf(1).equals(nodes) && unassignedReplicaShards != null && unassignedReplicaShards > 0);
+   }
+
+
+
+   /*******************************************************************************
+    ** Published wire schema for the same immutable value used by Java hosts.
+    *******************************************************************************/
+   @Override
+   public Schema toSchema()
+   {
+      return (new QuickSearchStatusSpec().defineComponentSchemas().get("QuickSearchStatusV1"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Protect the snapshot from mutations by callers.
+    *******************************************************************************/
+   public QuickSearchStatus
+   {
+      indexes = List.copyOf(indexes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Overall availability; DEGRADED also covers incomplete measurements.
+    *******************************************************************************/
+   public enum State
+   {
+      OK, DEGRADED, DOWN, DISABLED
+   }
+
+
+
+   /*******************************************************************************
+    ** A persisted index row with counts from the latest saved validation, when
+    ** available. Otherwise documentCount is the last indexing run's count and
+    ** sourceCount/countsCheckedAt are unknown. Sync time is a successful watermark.
+    *******************************************************************************/
+   public record IndexRow(String tableName, Boolean enabled, String status, Long documentCount, Long sourceCount, Instant countsCheckedAt,
+                          Instant lastSuccessfulSync, String lastError, Integer realTimeErrorCount) implements ToSchema
+   {
+      /*******************************************************************************
+       ** Published per-table schema.
+       *******************************************************************************/
+      @Override
+      public Schema toSchema()
+      {
+         return (new QuickSearchStatusSpec().defineComponentSchemas().get("QuickSearchIndexStatusV1"));
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatusService.java

@@ -0,0 +1,70 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.StatusDiagnostics;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+
+
+/*******************************************************************************
+ ** Supported entry point for hosts. Uses the current QContext and checks the
+ ** Quick Search admin app permission on EVERY call, including cache hits.
+ *******************************************************************************/
+public final class QuickSearchStatusService
+{
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private QuickSearchStatusService()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Return a briefly cached snapshot, waiting at most two seconds for a refresh.
+    ** Call with an authenticated QSession (or an explicitly trusted system session).
+    *******************************************************************************/
+   public static QuickSearchStatus get() throws QException
+   {
+      if(QContext.getQSession() == null)
+      {
+         throw (new QPermissionDeniedException("A session is required for Quick Search status."));
+      }
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (StatusDiagnostics.unavailable(QuickSearchStatus.State.DISABLED, null, QuickSearchApi.HOME_HREF));
+      }
+      String appName = runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
+      if(QContext.getQInstance().getApp(appName) == null || !PermissionsHelper.hasAppPermission(new QueryInput(), appName))
+      {
+         throw (new QPermissionDeniedException("Quick Search admin permission is required."));
+      }
+      return (runtime.getStatusCache().get(runtime));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchStatusSpec.java

@@ -0,0 +1,187 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
+import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
+import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
+import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
+import com.kingsrook.qqq.openapi.model.HttpMethod;
+import com.kingsrook.qqq.openapi.model.Method;
+import com.kingsrook.qqq.openapi.model.Parameter;
+import com.kingsrook.qqq.openapi.model.Schema;
+import com.kingsrook.qqq.openapi.model.Type;
+import io.javalin.http.HttpStatus;
+
+
+/*******************************************************************************
+ ** Administrative status, available through the spec-managed UI API only.
+ *******************************************************************************/
+public class QuickSearchStatusSpec extends AbstractJsonEndpointSpec
+{
+   /*******************************************************************************
+    ** Describe the supported operation.
+    *******************************************************************************/
+   @Override
+   public BasicOperation defineBasicOperation()
+   {
+      return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/quick-search/status")
+         .withShortSummary("Get Quick Search status")
+         .withLongDescription("Requires Quick Search admin permission. Five-second instance cache; two-second refresh wait. Counts are saved diagnostics, not live source scans. Unknown values are null. Backlog counts are lower bounds when backlogTruncated is true."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Stable generated-client operation name.
+    *******************************************************************************/
+   @Override
+   protected void customizeMethod(Method method)
+   {
+      method.setOperationId("getQuickSearchStatus");
+   }
+
+
+
+   /*******************************************************************************
+    ** No request parameters.
+    *******************************************************************************/
+   @Override
+   public List<Parameter> defineRequestParameters()
+   {
+      return (List.of());
+   }
+
+
+
+   /*******************************************************************************
+    ** Versioned response component.
+    *******************************************************************************/
+   @Override
+   protected String getResponseSchemaName()
+   {
+      return ("QuickSearchStatusV1");
+   }
+
+
+
+   /*******************************************************************************
+    ** Use exactly the same authorized service as in-process hosts.
+    *******************************************************************************/
+   @Override
+   protected Object handle(JsonInput input) throws Exception
+   {
+      return (QuickSearchStatusService.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Standard framework errors, including the diagnostics rate limit.
+    *******************************************************************************/
+   @Override
+   public List<BasicResponse> defineAdditionalBasicResponses()
+   {
+      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Diagnostic rate limit exceeded", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.FORBIDDEN, "Quick Search admin permission required", BasicErrorResponseV1.class.getSimpleName())));
+   }
+
+
+
+   /*******************************************************************************
+    ** Explicit schemas make nullability, bounds and measurement age discoverable.
+    *******************************************************************************/
+   @Override
+   public Map<String, Schema> defineComponentSchemas()
+   {
+      Map<String, Schema> status = new TreeMap<>();
+      status.put("state", field(Type.STRING, "Overall availability; incomplete diagnostics are DEGRADED.", false)
+         .withEnumValues(List.of("OK", "DEGRADED", "DOWN", "DISABLED")));
+      status.put("clusterReachable", field(Type.BOOLEAN, "Whether OpenSearch answered; null when not probed.", true));
+      status.put("clusterHealth", field(Type.STRING, "OpenSearch health color; null when unknown.", true));
+      for(String name : List.of("nodes", "activeShards", "unassignedShards"))
+      {
+         status.put(name, field(Type.INTEGER, "Cluster health counter; null when unavailable.", true));
+      }
+      status.put("unassignedReplicaShards", field(Type.INTEGER, "Unassigned shards when every primary is active (all are replicas); null when unknown or a primary is not active.", true));
+      status.put("targetReplicas", field(Type.INTEGER, "number_of_replicas Quick Search applies to its index; null when unknown or not managed (Serverless).", true));
+      status.put("replicaSettingsFailed", field(Type.BOOLEAN, "Sizing or applying the index replicas failed at startup; null when not attempted.", true));
+      status.put("indexingPaused", field(Type.BOOLEAN, "Indexing is paused; search can still serve.", true));
+      status.put("indexes", new Schema().withType(Type.ARRAY).withItems(new Schema().withRef("#/components/schemas/QuickSearchIndexStatusV1"))
+         .withDescription("At most 100 persisted index rows, ordered by id."));
+      status.put("pending", field(Type.INTEGER, "Pending and awaiting-reindex failed events, from at most 1000 backlog rows.", true));
+      status.put("exhausted", field(Type.INTEGER, "Exhausted failed events, from the same bounded backlog read.", true));
+      status.put("backlogTruncated", field(Type.BOOLEAN, "Counts are lower bounds when more than 1000 backlog rows exist.", false));
+      status.put("tablesTruncated", field(Type.BOOLEAN, "More than 100 index rows exist.", false));
+      status.put("lastSuccessfulSync", timestamp("Newest successful indexing watermark; not a guarantee all tables are current."));
+      status.put("lastError", field(Type.STRING, "Last known indexing/search error or snapshot failure; null when absent.", true));
+      status.put("checkedAt", timestamp("Snapshot completion time; remains unchanged on cache hits.").withNullable(false));
+      status.put("homeHref", field(Type.STRING, "Quick Search home link, including the configured metadata prefix.", false));
+      Map<String, Schema> row = new TreeMap<>();
+      row.put("tableName", field(Type.STRING, "Indexed source table metadata name.", false));
+      row.put("enabled", field(Type.BOOLEAN, "Whether this index is enabled.", false));
+      row.put("status", field(Type.STRING, "Persisted indexing state, such as ACTIVE or NEEDS_REINDEX.", true));
+      row.put("documentCount", field(Type.INTEGER, "Saved validation document count, else last indexing run count; not live.", true).withFormat("int64"));
+      row.put("sourceCount", field(Type.INTEGER, "Source count from saved IndexValidator report; null without validation.", true).withFormat("int64"));
+      row.put("countsCheckedAt", timestamp("Saved validation completion; null when validation counts are unavailable."));
+      row.put("lastSuccessfulSync", timestamp("Newest successful basepull/full-reindex/reconcile watermark."));
+      row.put("lastError", field(Type.STRING, "Persisted indexing error; null when absent.", true));
+      row.put("realTimeErrorCount", field(Type.INTEGER, "Persisted real-time indexing error count.", true));
+      return (Map.of("QuickSearchStatusV1", object(status), "QuickSearchIndexStatusV1", object(row),
+         "QuickSearchErrorV1", object(Map.of("error", field(Type.STRING, "Framework error message.", false)))
+            .withDescription("Deprecated since 1.1.0: errors use BasicErrorResponseV1.")));
+   }
+
+
+
+   /*******************************************************************************
+    ** A response object always includes its properties, including nulls.
+    *******************************************************************************/
+   private static Schema object(Map<String, Schema> properties)
+   {
+      return (new Schema().withType(Type.OBJECT).withProperties(new TreeMap<>(properties)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A scalar with explicit unknown semantics.
+    *******************************************************************************/
+   private static Schema field(Type type, String description, Boolean nullable)
+   {
+      return (new Schema().withType(type).withDescription(description).withNullable(nullable));
+   }
+
+
+
+   /*******************************************************************************
+    ** ISO-8601 instant.
+    *******************************************************************************/
+   private static Schema timestamp(String description)
+   {
+      return (field(Type.STRING, description, true).withFormat("date-time"));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchSuggest.java

@@ -0,0 +1,64 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.util.List;
+
+
+/*******************************************************************************
+ ** Minimal immutable suggestion response. Search internals never cross this API.
+ *******************************************************************************/
+public record QuickSearchSuggest(String query, List<Group> groups, Boolean truncated)
+{
+   /*******************************************************************************
+    ** Freeze the group order for callers.
+    *******************************************************************************/
+   public QuickSearchSuggest
+   {
+      groups = List.copyOf(groups);
+   }
+
+
+
+   /*******************************************************************************
+    ** One readable table, ordered by its highest scoring accessible hit.
+    *******************************************************************************/
+   public record Group(String tableName, String tableLabel, Boolean hasMore, List<Item> items)
+   {
+      /*******************************************************************************
+       ** Freeze the per-table relevance order.
+       *******************************************************************************/
+      public Group
+      {
+         items = List.copyOf(items);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Only already-filtered labels and subtitles are exposed.
+    *******************************************************************************/
+   public record Item(String recordId, String label, String subtitle)
+   {
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchSuggestService.java

@@ -0,0 +1,161 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchOutput;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+
+
+/*******************************************************************************
+ ** Session-only typeahead over the existing permission-aware search action.
+ *******************************************************************************/
+public final class QuickSearchSuggestService
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchSuggestService.class);
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private QuickSearchSuggestService()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One candidate batch per table, with one extra visible hit for hasMore.
+    ** A short term or unavailable search returns an empty response. hasMore only
+    ** confirms an extra accessible hit in this bounded batch, never hidden rows.
+    *******************************************************************************/
+   public static QuickSearchSuggest suggest(String query, Integer limitPerTable, Integer maxTables, String tables) throws QException
+   {
+      if(QContext.getQSession() == null)
+      {
+         throw (new QPermissionDeniedException("A session is required for Quick Search suggestions."));
+      }
+      if(query == null)
+      {
+         throw (new QBadRequestException("q is required."));
+      }
+      String normalized = IndexingUtils.normalizeSearchText(query);
+      if(normalized.length() > QuickSearchAction.MAX_TERM_LENGTH)
+      {
+         throw (new QBadRequestException("Search term is too long (maximum 100 characters)."));
+      }
+      Integer itemLimit = bounded(limitPerTable, 5, 10, "limitPerTable");
+      Integer groupLimit = bounded(maxTables, 8, 20, "maxTables");
+      QuickSearchSuggest empty = new QuickSearchSuggest(normalized, List.of(), false);
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(normalized.length() < QuickSearchAction.MIN_TERM_LENGTH || runtime == null)
+      {
+         return (empty);
+      }
+
+      List<String> tableNames = tables == null ? null : Arrays.stream(tables.split(",")).map(String::trim).filter(name -> !name.isEmpty()).distinct().toList();
+      if(tableNames != null && tableNames.isEmpty())
+      {
+         return (empty);
+      }
+
+      QuickSearchOutput output;
+      try
+      {
+         if(runtime.getClient() == null)
+         {
+            return (empty);
+         }
+         output = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(normalized)
+            .withTableNames(tableNames).withLimitPerTable(itemLimit + 1).withSinglePass(true));
+      }
+      catch(QPermissionDeniedException e)
+      {
+         throw (e);
+      }
+      catch(QException | RuntimeException e)
+      {
+         LOG.debug("Quick Search suggestions unavailable", e);
+         return (empty);
+      }
+
+      Map<String, List<QuickSearchResult>> byTable = new LinkedHashMap<>();
+      for(QuickSearchResult result : output.getResults())
+      {
+         byTable.computeIfAbsent(result.getTableName(), key -> new ArrayList<>()).add(result);
+      }
+      List<List<QuickSearchResult>> ranked = new ArrayList<>(byTable.values());
+      ranked.sort(Comparator.<List<QuickSearchResult>, Float>comparing(
+         hits -> hits.stream().map(QuickSearchResult::getScore).filter(score -> score != null).max(Float::compare).orElse(Float.NEGATIVE_INFINITY))
+         .reversed().thenComparing(hits -> hits.get(0).getTableName()));
+
+      List<QuickSearchSuggest.Group> groups = new ArrayList<>();
+      for(List<QuickSearchResult> hits : ranked.subList(0, Math.min(groupLimit, ranked.size())))
+      {
+         QuickSearchResult first = hits.get(0);
+         List<QuickSearchSuggest.Item> items = hits.stream().limit(itemLimit).map(hit ->
+         {
+            String label = hit.getRecordLabel();
+            if(label == null || label.isBlank())
+            {
+               label = hit.getTableLabel() + " #" + hit.getRecordId();
+            }
+            return (new QuickSearchSuggest.Item(hit.getRecordId(), label, hit.getSubtitle()));
+         }).toList();
+         groups.add(new QuickSearchSuggest.Group(first.getTableName(), first.getTableLabel(), hits.size() > itemLimit, items));
+      }
+      return (new QuickSearchSuggest(normalized, groups, ranked.size() > groupLimit));
+   }
+
+
+
+   /*******************************************************************************
+    ** Reject invalid ranges instead of silently changing a caller's intent.
+    *******************************************************************************/
+   private static Integer bounded(Integer value, Integer defaultValue, Integer maximum, String name) throws QBadRequestException
+   {
+      if(value == null)
+      {
+         return (defaultValue);
+      }
+      if(value < 1 || value > maximum)
+      {
+         throw (new QBadRequestException(name + " must be between 1 and " + maximum + "."));
+      }
+      return (value);
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/api/QuickSearchSuggestSpec.java

@@ -0,0 +1,177 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.api;
+
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
+import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
+import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
+import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
+import com.kingsrook.qqq.openapi.model.HttpMethod;
+import com.kingsrook.qqq.openapi.model.In;
+import com.kingsrook.qqq.openapi.model.Method;
+import com.kingsrook.qqq.openapi.model.Parameter;
+import com.kingsrook.qqq.openapi.model.Schema;
+import com.kingsrook.qqq.openapi.model.Type;
+import io.javalin.http.HttpStatus;
+
+
+/*******************************************************************************
+ ** Permission-filtered suggestions through the spec-managed UI API.
+ *******************************************************************************/
+public class QuickSearchSuggestSpec extends AbstractJsonEndpointSpec
+{
+   /*******************************************************************************
+    ** Describe the session-only operation and its bounded search semantics.
+    *******************************************************************************/
+   @Override
+   public BasicOperation defineBasicOperation()
+   {
+      return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/quick-search/suggest")
+         .withShortSummary("Get Quick Search suggestions")
+         .withLongDescription("Requires a signed-in session, not Quick Search admin permission. Results inherit table READ permissions, configured record security locks and protected-field exclusions. One bounded candidate batch per table; no extra scans. Unavailable search and normalized terms shorter than two characters return empty groups."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Stable generated-client operation name.
+    *******************************************************************************/
+   @Override
+   protected void customizeMethod(Method method)
+   {
+      method.setOperationId("getQuickSearchSuggestions");
+   }
+
+
+
+   /*******************************************************************************
+    ** Bounds apply after whitespace normalization; short terms are forgiving.
+    *******************************************************************************/
+   @Override
+   public List<Parameter> defineRequestParameters()
+   {
+      return (List.of(
+         new Parameter().withName("q").withIn(In.QUERY).withRequired(true)
+            .withDescription("Search text: trim and collapse whitespace before applying the 2..100 character range. Shorter terms return empty groups; longer terms return 400.")
+            .withSchema(field(Type.STRING, "Normalized search term.", false)),
+         new Parameter().withName("limitPerTable").withIn(In.QUERY).withRequired(false)
+            .withDescription("Items per table: integer 1..10, default 5.").withSchema(field(Type.INTEGER, "Maximum items per table; default 5, maximum 10.", false)),
+         new Parameter().withName("maxTables").withIn(In.QUERY).withRequired(false)
+            .withDescription("Groups to return: integer 1..20, default 8.").withSchema(field(Type.INTEGER, "Maximum groups; default 8, maximum 20.", false)),
+         new Parameter().withName("tables").withIn(In.QUERY).withRequired(false)
+            .withDescription("Optional comma-separated table metadata names. Unknown, disabled and unreadable tables are omitted.")
+            .withSchema(field(Type.STRING, "Table-name filter; omitted means all eligible tables.", false))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Versioned response component.
+    *******************************************************************************/
+   @Override
+   protected String getResponseSchemaName()
+   {
+      return ("QuickSearchSuggestV1");
+   }
+
+
+
+   /*******************************************************************************
+    ** The HTTP and in-process entry points share validation and authorization.
+    *******************************************************************************/
+   @Override
+   protected Object handle(JsonInput input) throws Exception
+   {
+      return (QuickSearchSuggestService.suggest(input.getQueryParam("q"), input.getQueryParamInteger("limitPerTable"),
+         input.getQueryParamInteger("maxTables"), input.getQueryParam("tables")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Reuse the extension's standard framework error shape.
+    *******************************************************************************/
+   @Override
+   public List<BasicResponse> defineAdditionalBasicResponses()
+   {
+      return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Missing q, normalized term longer than 100 characters, or invalid limits", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", BasicErrorResponseV1.class.getSimpleName()),
+         new BasicResponse(HttpStatus.FORBIDDEN, "Session permission denied", BasicErrorResponseV1.class.getSimpleName())));
+   }
+
+
+
+   /*******************************************************************************
+    ** Explicit wire schemas exclude scores, highlights and display values.
+    *******************************************************************************/
+   @Override
+   public Map<String, Schema> defineComponentSchemas()
+   {
+      Schema response = object(Map.of(
+         "query", field(Type.STRING, "Search term after trimming and collapsing whitespace.", false),
+         "groups", array("QuickSearchSuggestGroupV1", "Readable tables ordered by best accessible score; at most maxTables."),
+         "truncated", field(Type.BOOLEAN, "More tables had accessible hits than maxTables.", false)));
+      Schema group = object(Map.of(
+         "tableName", field(Type.STRING, "Source table metadata name.", false),
+         "tableLabel", field(Type.STRING, "Source table label, falling back to its metadata name.", false),
+         "hasMore", field(Type.BOOLEAN, "An extra accessible item was found in the bounded batch. False does not prove the index has no further matches.", false),
+         "items", array("QuickSearchSuggestItemV1", "At most limitPerTable accessible items in per-table relevance order.")));
+      Schema item = object(Map.of(
+         "recordId", field(Type.STRING, "Source record primary key.", false),
+         "label", field(Type.STRING, "Filtered record label, falling back to tableLabel #recordId when blank.", false),
+         "subtitle", field(Type.STRING, "Already-filtered result subtitle; null when absent.", true)));
+      return (Map.of("QuickSearchSuggestV1", response, "QuickSearchSuggestGroupV1", group, "QuickSearchSuggestItemV1", item));
+   }
+
+
+
+   /*******************************************************************************
+    ** All response objects have explicit properties.
+    *******************************************************************************/
+   private static Schema object(Map<String, Schema> properties)
+   {
+      return (new Schema().withType(Type.OBJECT).withNullable(false).withProperties(new TreeMap<>(properties)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Scalar fields declare nullability and meaning.
+    *******************************************************************************/
+   private static Schema field(Type type, String description, Boolean nullable)
+   {
+      return (new Schema().withType(type).withDescription(description).withNullable(nullable));
+   }
+
+
+
+   /*******************************************************************************
+    ** Arrays always exist, including when empty.
+    *******************************************************************************/
+   private static Schema array(String itemSchema, String description)
+   {
+      return (field(Type.ARRAY, description, false).withItems(new Schema().withRef("#/components/schemas/" + itemSchema)));
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java

@@ -0,0 +1,513 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Self-priming: seeds the config records from the code defaults
+ ** (@QuickSearchable / @QuickSearchField entities and searchableTables), on
+ ** startup and until it has succeeded once per runtime.
+ **
+ ** Precedence (the records win; code only fills in what is missing):
+ ** - A table code declares that has no quick_search_index row, and no
+ **   tombstone, gets a row (source CODE, enabled per enabledByDefault) and
+ **   field rows, and is queued for its first build (status PENDING_BUILD,
+ **   which the next basepull runs). A tombstoned table is never re-added.
+ ** - On a row an admin has not edited (ownerEdited false), the basepull
+ **   interval and timestamp field follow code. enabled is set only when the
+ **   row is created, never afterwards.
+ ** - A field code declares that has no row, and no tombstone, is added; a
+ **   field row an admin has not edited follows code's weight and label flag.
+ **   Either change queues a rebuild of the table. Field rows an admin added
+ **   or edited are never changed, and nothing is ever deleted by priming:
+ **   removing an annotation leaves its rows for the admin to remove.
+ ** - Rows from before 1.1.0-RC.3 (no field rows yet) get their field rows
+ **   without a rebuild when the index was already built with those fields.
+ **
+ ** All writes are system writes (QuickSearchConfigWrites), so they are not
+ ** taken for an admin's edits. The outcome is written to quick_search_settings
+ ** (lastPrimingSummary), which the Quick Search home shows.
+ *******************************************************************************/
+public class QuickSearchConfigPrimer
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchConfigPrimer.class);
+
+   private final QuickSearchRuntime    runtime;
+   private final QuickSearchQBitConfig config;
+
+
+
+   /*******************************************************************************
+    ** What priming did.
+    **
+    ** @param newTables     labels of the tables added (and queued to build)
+    ** @param newFields     field rows added
+    ** @param updatedFields field rows updated to code's defaults
+    ** @param updatedTables index rows updated to code's defaults
+    ** @param rebuilds      labels of existing tables queued to rebuild
+    ** @param skipped       code tables or fields skipped for a tombstone
+    *******************************************************************************/
+   public record PrimingReport(List<String> newTables, int newFields, int updatedFields, int updatedTables, List<String> rebuilds, int skipped)
+   {
+      /*******************************************************************************
+       ** Whether priming changed anything.
+       *******************************************************************************/
+      public boolean hasChanges()
+      {
+         return (!newTables.isEmpty() || newFields > 0 || updatedFields > 0 || updatedTables > 0 || !rebuilds.isEmpty());
+      }
+
+
+
+      /*******************************************************************************
+       ** The one-line summary for the Quick Search home.
+       *******************************************************************************/
+      public String summary()
+      {
+         List<String> parts = new ArrayList<>();
+         if(!newTables.isEmpty())
+         {
+            parts.add("New: " + newTables.size() + (newTables.size() == 1 ? " table" : " tables") + " added and building (" + String.join(", ", newTables) + ")");
+         }
+         if(newFields > 0 || updatedFields > 0)
+         {
+            parts.add(newFields + " field" + (newFields == 1 ? "" : "s") + " added, " + updatedFields + " updated from code defaults");
+         }
+         if(!rebuilds.isEmpty())
+         {
+            parts.add("rebuilding " + String.join(", ", rebuilds));
+         }
+         if(updatedTables > 0)
+         {
+            parts.add(updatedTables + " index setting" + (updatedTables == 1 ? "" : "s") + " updated from code defaults");
+         }
+         return (parts.isEmpty() ? "No changes: the records match the code defaults." : String.join("; ", parts) + ".");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchConfigPrimer(QuickSearchRuntime runtime)
+   {
+      this.runtime = runtime;
+      this.config  = runtime.getConfig();
+   }
+
+
+
+   /*******************************************************************************
+    ** Seed the records from the code defaults. Throws when the config tables
+    ** cannot be read or written (the caller retries later).
+    *******************************************************************************/
+   public PrimingReport prime() throws QException
+   {
+      PrimingReport report = QuickSearchConfigWrites.asSystem(this::doPrime);
+      if(report.hasChanges())
+      {
+         LOG.info("Quick Search primed its config records: " + report.summary());
+      }
+      return (report);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private PrimingReport doPrime() throws QException
+   {
+      QInstance qInstance  = QContext.getQInstance();
+      String    indexTable = config.getQuickSearchIndexTableName();
+      String    fieldTable = config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME);
+
+      Set<String> tableTombstones = new HashSet<>();
+      Set<String> fieldTombstones = new HashSet<>();
+      String      tombstoneTable  = config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME);
+      if(qInstance.getTable(tombstoneTable) != null)
+      {
+         for(QRecord tombstone : queryAll(tombstoneTable))
+         {
+            String fieldName = tombstone.getValueString("fieldName");
+            if(fieldName == null)
+            {
+               tableTombstones.add(tombstone.getValueString("tableName"));
+            }
+            else
+            {
+               fieldTombstones.add(tombstone.getValueString("tableName") + "." + fieldName);
+            }
+         }
+      }
+
+      Map<String, QRecord> rowsByTable = new LinkedHashMap<>();
+      for(QRecord row : queryAll(indexTable))
+      {
+         rowsByTable.putIfAbsent(row.getValueString("tableName"), row);
+      }
+      Map<Integer, Map<String, QRecord>> fieldsByIndex = new HashMap<>();
+      for(QRecord fieldRow : queryAll(fieldTable))
+      {
+         fieldsByIndex.computeIfAbsent(fieldRow.getValueInteger("quickSearchIndexId"), k -> new LinkedHashMap<>()).putIfAbsent(fieldRow.getValueString("fieldName"), fieldRow);
+      }
+
+      List<String> newTables     = new ArrayList<>();
+      List<String> rebuilds      = new ArrayList<>();
+      int          newFields     = 0;
+      int          updatedFields = 0;
+      int          updatedTables = 0;
+      int          skipped       = 0;
+
+      for(QuickSearchableTableConfig code : runtime.getCodeTables())
+      {
+         String         tableName = code.getTableName();
+         QTableMetaData table     = qInstance.getTable(tableName);
+         if(table == null)
+         {
+            LOG.warn("A table declared searchable in code is not in the instance; it is not primed", logPair("tableName", tableName));
+            continue;
+         }
+         if(tableTombstones.contains(tableName))
+         {
+            skipped++;
+            continue;
+         }
+
+         List<QRecord> codeFields = new ArrayList<>();
+         List<String>  fieldNames = CollectionUtils.nonNullList(code.getSearchableFields());
+         for(int i = 0; i < fieldNames.size(); i++)
+         {
+            String fieldName = fieldNames.get(i);
+            if(fieldTombstones.contains(tableName + "." + fieldName))
+            {
+               skipped++;
+               continue;
+            }
+            if(!QuickSearchFieldRules.isIndexable(table, fieldName))
+            {
+               QuickSearchFieldRules.warnExcludedFields(table);
+               continue;
+            }
+            codeFields.add(codeFieldRow(code, fieldName, i));
+         }
+
+         QRecord row = rowsByTable.get(tableName);
+         if(row == null)
+         {
+            boolean enabled = !Boolean.FALSE.equals(code.getEnabledByDefault());
+            QRecord inserted = insertOne(indexTable, new QRecord()
+               .withValue("tableName", tableName)
+               .withValue("enabled", enabled)
+               .withValue("basepullIntervalMinutes", code.getBasepullIntervalMinutes())
+               .withValue("basepullTimestampField", code.getBasepullTimestampField())
+               .withValue("source", QuickSearchIndex.SOURCE_CODE)
+               .withValue("ownerEdited", false)
+               .withValue("realTimeErrorCount", 0)
+               .withValue("status", enabled ? AbstractIndexingStep.STATUS_PENDING_BUILD : AbstractIndexingStep.STATUS_ACTIVE));
+            Integer indexId = inserted.getValueInteger("id");
+            codeFields.forEach(f -> f.setValue("quickSearchIndexId", indexId));
+            insertAll(fieldTable, codeFields);
+            newFields += codeFields.size();
+            newTables.add(QuickSearchFieldRules.label(table));
+            continue;
+         }
+
+         Integer                   indexId = row.getValueInteger("id");
+         Map<String, Serializable> updates = new LinkedHashMap<>();
+         if(row.getValueString("source") == null)
+         {
+            updates.put("source", QuickSearchIndex.SOURCE_CODE);
+         }
+         if(!Boolean.TRUE.equals(row.getValueBoolean("ownerEdited")))
+         {
+            if(!Objects.equals(row.getValueInteger("basepullIntervalMinutes"), code.getBasepullIntervalMinutes()) && code.getBasepullIntervalMinutes() != null)
+            {
+               updates.put("basepullIntervalMinutes", code.getBasepullIntervalMinutes());
+            }
+            if(!Objects.equals(row.getValueString("basepullTimestampField"), code.getBasepullTimestampField()))
+            {
+               updates.put("basepullTimestampField", code.getBasepullTimestampField());
+            }
+         }
+         if(updates.keySet().stream().anyMatch(k -> !"source".equals(k)))
+         {
+            updatedTables++;
+         }
+
+         Map<String, QRecord> existingFields = fieldsByIndex.getOrDefault(indexId, Map.of());
+         boolean              migration      = existingFields.isEmpty();
+         boolean              fieldsChanged  = false;
+         List<QRecord>        toInsert       = new ArrayList<>();
+         List<QRecord>        toUpdate       = new ArrayList<>();
+         for(QRecord codeField : codeFields)
+         {
+            QRecord existing = existingFields.get(codeField.getValueString("fieldName"));
+            if(existing == null)
+            {
+               codeField.setValue("quickSearchIndexId", indexId);
+               toInsert.add(codeField);
+               fieldsChanged = true;
+            }
+            else if(!Boolean.TRUE.equals(existing.getValueBoolean("ownerEdited"))
+               && (!Objects.equals(existing.getValueInteger("weight"), codeField.getValueInteger("weight"))
+               || !Objects.equals(Boolean.TRUE.equals(existing.getValueBoolean("includeLabel")), codeField.getValueBoolean("includeLabel"))))
+            {
+               toUpdate.add(new QRecord().withValue("id", existing.getValue("id"))
+                  .withValue("weight", codeField.getValueInteger("weight"))
+                  .withValue("includeLabel", codeField.getValueBoolean("includeLabel")));
+               fieldsChanged = true;
+            }
+         }
+         insertAll(fieldTable, toInsert);
+         updateAll(fieldTable, toUpdate);
+         newFields += toInsert.size();
+         updatedFields += toUpdate.size();
+
+         if(fieldsChanged && needsRebuild(row, code, migration))
+         {
+            updates.put("status", AbstractIndexingStep.STATUS_PENDING_BUILD);
+            rebuilds.add(QuickSearchFieldRules.label(table));
+         }
+
+         if(!updates.isEmpty())
+         {
+            QRecord update = new QRecord().withValue("id", indexId);
+            updates.forEach(update::setValue);
+            updateAll(indexTable, List.of(update));
+         }
+      }
+
+      PrimingReport report = new PrimingReport(newTables, newFields, updatedFields, updatedTables, rebuilds, skipped);
+      writeSettings(report);
+      return (report);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a primed field change needs the table rebuilt: not when it is
+    ** disabled or already building, and not for rows from before field
+    ** records when the index was already built with exactly code's fields
+    ** (or never built: its first basepull reads everything anyway).
+    *******************************************************************************/
+   private static boolean needsRebuild(QRecord row, QuickSearchableTableConfig code, boolean migration)
+   {
+      String status = row.getValueString("status");
+      if(Boolean.FALSE.equals(row.getValueBoolean("enabled"))
+         || AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status)
+         || AbstractIndexingStep.STATUS_BUILDING.equals(status)
+         || AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+      {
+         return (false);
+      }
+      if(migration)
+      {
+         String built = row.getValueString("searchableFieldsJson");
+         return (built != null && !built.equals(AbstractIndexingStep.buildSearchableFieldsJson(code)));
+      }
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** A field row from code's defaults (role BODY; analyzer and snippet left
+    ** to their defaults).
+    *******************************************************************************/
+   private static QRecord codeFieldRow(QuickSearchableTableConfig code, String fieldName, int sortOrder)
+   {
+      Integer weight       = code.getFieldWeights() == null ? null : code.getFieldWeights().get(fieldName);
+      Boolean includeLabel = code.getFieldIncludeLabels() == null ? null : code.getFieldIncludeLabels().get(fieldName);
+      return (new QRecord()
+         .withValue("fieldName", fieldName)
+         .withValue("weight", weight == null ? 1 : weight)
+         .withValue("includeLabel", Boolean.TRUE.equals(includeLabel))
+         .withValue("role", QuickSearchFieldRole.BODY.name())
+         .withValue("sortOrder", (sortOrder + 1) * 10)
+         .withValue("source", QuickSearchIndexField.SOURCE_CODE)
+         .withValue("ownerEdited", false));
+   }
+
+
+
+   /*******************************************************************************
+    ** Create the settings row when missing, and record the priming outcome.
+    *******************************************************************************/
+   private void writeSettings(PrimingReport report) throws QException
+   {
+      String settingsTable = config.applyTablePrefix(QuickSearchSettings.TABLE_NAME);
+      if(QContext.getQInstance().getTable(settingsTable) == null)
+      {
+         return;
+      }
+
+      QRecord values = new QRecord().withValue("lastPrimedAt", Instant.now());
+      if(report.hasChanges())
+      {
+         values.withValue("lastPrimingSummary", report.summary()).withValue("lastPrimingNewTables", report.newTables().size());
+      }
+
+      List<QRecord> rows = queryAll(settingsTable);
+      if(rows.isEmpty())
+      {
+         values.withValue("id", QuickSearchSettings.SETTINGS_ID).withValue("indexingPaused", false);
+         if(!report.hasChanges())
+         {
+            values.withValue("lastPrimingSummary", report.summary()).withValue("lastPrimingNewTables", 0);
+         }
+         insertOne(settingsTable, values);
+      }
+      else
+      {
+         ///////////////////////////////////////////////////////////////////////
+         // Only the priming outcome (lastPrimedAt, summary) changes here, on  //
+         // every startup: bookkeeping, not a settings change, so not audited. //
+         ///////////////////////////////////////////////////////////////////////
+         values.withValue("id", rows.get(0).getValue("id"));
+         updateAll(settingsTable, List.of(values), true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Every row of a table.
+    *******************************************************************************/
+   public static List<QRecord> queryAll(String tableName) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter());
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one row, failing on any record error.
+    *******************************************************************************/
+   public static QRecord insertOne(String tableName, QRecord record) throws QException
+   {
+      List<QRecord> inserted = insertAll(tableName, List.of(record));
+      if(inserted.isEmpty())
+      {
+         throw (new QException("Insert into " + tableName + " returned no record"));
+      }
+      return (inserted.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert rows, failing on any record error.
+    *******************************************************************************/
+   public static List<QRecord> insertAll(String tableName, List<QRecord> records) throws QException
+   {
+      if(records.isEmpty())
+      {
+         return (List.of());
+      }
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(tableName);
+      insertInput.setRecords(records);
+      InsertOutput  output   = new InsertAction().execute(insertInput);
+      List<QRecord> inserted = CollectionUtils.nonNullList(output.getRecords());
+      for(QRecord record : inserted)
+      {
+         if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            throw (new QException("Could not insert into " + tableName + ": " + record.getErrors().get(0).getMessage()));
+         }
+      }
+      return (inserted);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update rows, failing on any record error.
+    *******************************************************************************/
+   public static void updateAll(String tableName, List<QRecord> records) throws QException
+   {
+      updateAll(tableName, records, false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update rows, failing on any record error; omitAudit skips the DML audit
+    ** for runtime bookkeeping writes.
+    *******************************************************************************/
+   public static void updateAll(String tableName, List<QRecord> records, boolean omitAudit) throws QException
+   {
+      if(records.isEmpty())
+      {
+         return;
+      }
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(tableName);
+      updateInput.setRecords(records);
+      updateInput.setOmitDmlAudit(omitAudit);
+      for(QRecord record : CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords()))
+      {
+         if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            throw (new QException("Could not update " + tableName + ": " + record.getErrors().get(0).getMessage()));
+         }
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigStore.java

@@ -0,0 +1,504 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The effective Quick Search configuration, read from the records: which
+ ** tables are indexed (quick_search_index), with which fields, weights,
+ ** roles and analyzers (quick_search_index_field), and the system settings
+ ** (quick_search_settings: the global pause).
+ **
+ ** Precedence: the records are the source of truth once the instance has the
+ ** config tables and priming has run. Code (annotations and searchableTables)
+ ** only seeds them (QuickSearchConfigPrimer). Until priming has succeeded,
+ ** or when the config tables are not in the instance or not readable (a host
+ ** that has not added their DDL yet), the code-declared tables are used, as
+ ** before 1.1.0-RC.3.
+ **
+ ** Reads are cached for CACHE_TTL_MILLIS and dropped at once by the config
+ ** tables' customizers when a row changes, on every server: the runtime
+ ** publishes a cluster cache invalidation (QuickSearchRuntime.CONFIG_CACHE_NAME).
+ ** The TTL remains the safety net when the cluster bus is down.
+ *******************************************************************************/
+public class QuickSearchConfigStore
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchConfigStore.class);
+
+   public static final long CACHE_TTL_MILLIS   = 30_000L;
+   public static final long PRIME_RETRY_MILLIS = 300_000L;
+
+   private static final ThreadLocal<Boolean> BUSY = ThreadLocal.withInitial(() -> false);
+
+   private final QuickSearchRuntime ownerRuntime;
+
+   private volatile Snapshot snapshot;
+   private volatile QRecord  settings;
+   private volatile long     settingsReadAtMillis = 0L;
+   private volatile boolean  primed               = false;
+   private volatile long     lastPrimeAttemptMillis = 0L;
+   private volatile long     lastFailureLogMillis   = 0L;
+
+   private final Set<String> coreSearchFieldsDefaulted = ConcurrentHashMap.newKeySet();
+
+
+
+   /*******************************************************************************
+    ** A loaded set of effective table configs.
+    *******************************************************************************/
+   private record Snapshot(List<QuickSearchableTableConfig> tables, Map<String, QuickSearchableTableConfig> byName, long loadedAtMillis)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchConfigStore(QuickSearchRuntime runtime)
+   {
+      this.ownerRuntime = runtime;
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime the config hangs off (the one the store was made for, unless
+    ** the config has since been given another, such as a wrapper of it).
+    *******************************************************************************/
+   private QuickSearchRuntime runtime()
+   {
+      QuickSearchRuntime current = ownerRuntime.getConfig() == null ? null : ownerRuntime.getConfig().getRuntime();
+      return (current != null ? current : ownerRuntime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the instance in the current context has the config tables
+    ** (quick_search_index and quick_search_index_field).
+    *******************************************************************************/
+   public boolean hasConfigTables()
+   {
+      QInstance qInstance = QContext.getQInstance();
+      QuickSearchQBitConfig config = runtime().getConfig();
+      return (qInstance != null
+         && qInstance.getTable(config.getQuickSearchIndexTableName()) != null
+         && qInstance.getTable(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME)) != null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the effective config comes from the records (else from code).
+    *******************************************************************************/
+   public boolean isRecordsMode()
+   {
+      return (primed && hasConfigTables());
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether priming has succeeded in this runtime.
+    *******************************************************************************/
+   public boolean isPrimed()
+   {
+      return (primed);
+   }
+
+
+
+   /*******************************************************************************
+    ** Record that priming succeeded (or, for tests, reset it).
+    *******************************************************************************/
+   public void setPrimed(boolean primed)
+   {
+      this.primed                 = primed;
+      this.lastPrimeAttemptMillis = 0L;
+      invalidate();
+   }
+
+
+
+   /*******************************************************************************
+    ** The effective table configs: from the records in records mode, else the
+    ** code-declared ones. Primes first when that has not happened yet (at most
+    ** once per PRIME_RETRY_MILLIS while it keeps failing). Never throws.
+    *******************************************************************************/
+   public List<QuickSearchableTableConfig> getTables()
+   {
+      Snapshot current = getSnapshot();
+      return (current == null ? runtime().getCodeTables() : current.tables());
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's effective config, or null when it is not indexed.
+    *******************************************************************************/
+   public QuickSearchableTableConfig getTable(String tableName)
+   {
+      if(tableName == null)
+      {
+         return (null);
+      }
+      Snapshot current = getSnapshot();
+      if(current == null)
+      {
+         return (runtime().getCodeTableConfig(tableName));
+      }
+      return (current.byName().get(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** The current snapshot, loading it when stale; null means "use code".
+    *******************************************************************************/
+   private Snapshot getSnapshot()
+   {
+      if(BUSY.get() || !hasConfigTables())
+      {
+         return (null);
+      }
+
+      if(!primed)
+      {
+         tryPrime();
+         if(!primed)
+         {
+            return (null);
+         }
+      }
+
+      Snapshot current = snapshot;
+      if(current != null && System.currentTimeMillis() - current.loadedAtMillis() < CACHE_TTL_MILLIS)
+      {
+         return (current);
+      }
+
+      Snapshot loaded = load();
+      if(loaded != null)
+      {
+         snapshot = loaded;
+         return (loaded);
+      }
+      return (current);
+   }
+
+
+
+   /*******************************************************************************
+    ** Prime now, unless it already succeeded or failed less than
+    ** PRIME_RETRY_MILLIS ago. Never throws.
+    *******************************************************************************/
+   public synchronized void tryPrime()
+   {
+      if(primed || BUSY.get() || !hasConfigTables())
+      {
+         return;
+      }
+      long now = System.currentTimeMillis();
+      if(lastPrimeAttemptMillis != 0 && now - lastPrimeAttemptMillis < PRIME_RETRY_MILLIS)
+      {
+         return;
+      }
+
+      BUSY.set(true);
+      try
+      {
+         new QuickSearchConfigPrimer(runtime()).prime();
+         primed                 = true;
+         lastPrimeAttemptMillis = 0L;
+      }
+      catch(Exception | LinkageError e)
+      {
+         lastPrimeAttemptMillis = now;
+         LOG.warn("Quick Search could not prime its config records; using the code-declared tables until it can (retried every "
+            + (PRIME_RETRY_MILLIS / 60_000) + " minutes)", e);
+      }
+      finally
+      {
+         BUSY.set(false);
+         invalidate();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the cached configs and settings (a config row changed).
+    *******************************************************************************/
+   public void invalidate()
+   {
+      snapshot             = null;
+      settingsReadAtMillis = 0L;
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the records into a snapshot; null (logged, at most once a minute)
+    ** when they cannot be read.
+    *******************************************************************************/
+   private Snapshot load()
+   {
+      BUSY.set(true);
+      try
+      {
+         QuickSearchQBitConfig config    = runtime().getConfig();
+         QInstance             qInstance = QContext.getQInstance();
+
+         List<QRecord> indexRows = query(config.getQuickSearchIndexTableName());
+         List<QRecord> fieldRows = query(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME));
+
+         Map<Integer, List<QRecord>> fieldsByIndex = new LinkedHashMap<>();
+         for(QRecord fieldRow : fieldRows)
+         {
+            fieldsByIndex.computeIfAbsent(fieldRow.getValueInteger("quickSearchIndexId"), k -> new ArrayList<>()).add(fieldRow);
+         }
+
+         List<QuickSearchableTableConfig>        tables = new ArrayList<>();
+         Map<String, QuickSearchableTableConfig> byName = new LinkedHashMap<>();
+         for(QRecord indexRow : indexRows)
+         {
+            QuickSearchableTableConfig tableConfig = buildTableConfig(qInstance, config, indexRow, fieldsByIndex.getOrDefault(indexRow.getValueInteger("id"), List.of()));
+            if(tableConfig != null && !byName.containsKey(tableConfig.getTableName()))
+            {
+               tables.add(tableConfig);
+               byName.put(tableConfig.getTableName(), tableConfig);
+            }
+         }
+
+         ////////////////////////////////////////////////////////////////////////
+         // tables added at runtime get core searchFields (once per table), so //
+         // the global search covers them too                                  //
+         ////////////////////////////////////////////////////////////////////////
+         if(Boolean.TRUE.equals(config.getServeCoreRecordSearch()) && qInstance != null)
+         {
+            List<QuickSearchableTableConfig> unseen = tables.stream().filter(t -> coreSearchFieldsDefaulted.add(t.getTableName())).toList();
+            if(!unseen.isEmpty())
+            {
+               QuickSearchQBitProducer.defaultCoreSearchFields(qInstance, unseen);
+            }
+         }
+
+         return (new Snapshot(Collections.unmodifiableList(tables), Collections.unmodifiableMap(byName), System.currentTimeMillis()));
+      }
+      catch(Exception | LinkageError e)
+      {
+         long now = System.currentTimeMillis();
+         if(now - lastFailureLogMillis > 60_000L)
+         {
+            lastFailureLogMillis = now;
+            LOG.warn("Could not read the Quick Search config records; keeping the last config read", e);
+         }
+         return (null);
+      }
+      finally
+      {
+         BUSY.set(false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Build one table's effective config from its index row and field rows.
+    ** A table no longer in the instance is skipped; a field that is gone or
+    ** can no longer be indexed (hidden, sensitive) is dropped with a warning.
+    *******************************************************************************/
+   static QuickSearchableTableConfig buildTableConfig(QInstance qInstance, QuickSearchQBitConfig config, QRecord indexRow, List<QRecord> fieldRows)
+   {
+      String         tableName = indexRow.getValueString("tableName");
+      QTableMetaData table     = qInstance == null || tableName == null ? null : qInstance.getTable(tableName);
+      if(table == null)
+      {
+         LOG.debug("Quick Search index row names a table that is not in the instance; skipped", logPair("tableName", tableName));
+         return (null);
+      }
+
+      List<QRecord> sorted = new ArrayList<>(fieldRows);
+      sorted.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("sortOrder") == null ? Integer.MAX_VALUE : r.getValueInteger("sortOrder"))
+         .thenComparing(r -> r.getValueInteger("id") == null ? Integer.MAX_VALUE : r.getValueInteger("id")));
+
+      List<String>         fields         = new ArrayList<>();
+      Map<String, Integer> weights        = new LinkedHashMap<>();
+      Map<String, Boolean> includeLabels  = new LinkedHashMap<>();
+      Map<String, String>  roles          = new LinkedHashMap<>();
+      Map<String, String>  analyzers      = new LinkedHashMap<>();
+      Map<String, Boolean> snippetFlags   = new LinkedHashMap<>();
+      for(QRecord fieldRow : sorted)
+      {
+         String fieldName = fieldRow.getValueString("fieldName");
+         String problem   = QuickSearchFieldRules.whyNotIndexable(table, fieldName);
+         if(problem != null)
+         {
+            QuickSearchFieldRules.warnExcludedFields(table);
+            continue;
+         }
+         if(fields.contains(fieldName))
+         {
+            continue;
+         }
+
+         fields.add(fieldName);
+         weights.put(fieldName, fieldRow.getValueInteger("weight") == null ? 1 : fieldRow.getValueInteger("weight"));
+         includeLabels.put(fieldName, Boolean.TRUE.equals(fieldRow.getValueBoolean("includeLabel")));
+
+         ///////////////////////////////////////////////////////////////////////
+         // only non-defaults go in the maps, so a table of plain fields has  //
+         // the same searchableFieldsJson as its code config (no false drift) //
+         ///////////////////////////////////////////////////////////////////////
+         QuickSearchFieldRole role = QuickSearchFieldRole.fromName(fieldRow.getValueString("role"), QuickSearchFieldRole.BODY);
+         if(role != QuickSearchFieldRole.BODY)
+         {
+            roles.put(fieldName, role.name());
+         }
+         if(StringUtils.hasContent(fieldRow.getValueString("analyzer")))
+         {
+            analyzers.put(fieldName, fieldRow.getValueString("analyzer"));
+         }
+         if(fieldRow.getValueBoolean("includeInSnippet") != null)
+         {
+            snippetFlags.put(fieldName, fieldRow.getValueBoolean("includeInSnippet"));
+         }
+      }
+
+      String timestampField = indexRow.getValueString("basepullTimestampField");
+      if(timestampField != null && !QuickSearchFieldRules.isTimestampField(table, timestampField))
+      {
+         LOG.warn("Quick Search basepull timestamp field is not a date-time field of the table; incremental basepull is off for it",
+            logPair("tableName", tableName), logPair("timestampField", timestampField));
+         timestampField = null;
+      }
+
+      Integer interval = indexRow.getValueInteger("basepullIntervalMinutes");
+      Boolean enabled  = indexRow.getValueBoolean("enabled");
+
+      QuickSearchableTableConfig codeConfig = config.getRuntime() == null ? null : config.getRuntime().getCodeTableConfig(tableName);
+
+      return (new QuickSearchableTableConfig()
+         .withTableName(tableName)
+         .withPrimaryKeyField(table.getPrimaryKeyField())
+         .withSearchableFields(fields)
+         .withFieldWeights(weights)
+         .withFieldIncludeLabels(includeLabels)
+         .withFieldRoles(roles)
+         .withFieldAnalyzers(analyzers)
+         .withFieldIncludeInSnippet(snippetFlags)
+         .withBasepullIntervalMinutes(interval == null ? config.getDefaultBasepullIntervalMinutes() : interval)
+         .withBasepullTimestampField(timestampField)
+         .withEnabledByDefault(codeConfig == null ? Boolean.TRUE : codeConfig.getEnabledByDefault())
+         .withEnabled(enabled == null || enabled)
+         .withQuickSearchIndexId(indexRow.getValueInteger("id"))
+         .withRecordLabelFormat(codeConfig == null ? null : codeConfig.getRecordLabelFormat())
+         .withRecordLabelFields(codeConfig == null ? null : codeConfig.getRecordLabelFields())
+         .withMaxFieldLength(config.getMaxFieldLength()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings row (id 1), cached like the configs; null when the table is
+    ** not in the instance, the row does not exist, or it cannot be read.
+    *******************************************************************************/
+   public QRecord getSettings()
+   {
+      long now = System.currentTimeMillis();
+      if(settingsReadAtMillis != 0 && now - settingsReadAtMillis < CACHE_TTL_MILLIS)
+      {
+         return (settings);
+      }
+
+      QRecord read = null;
+      String  name = runtime().getConfig().applyTablePrefix(QuickSearchSettings.TABLE_NAME);
+      if(QContext.getQInstance() != null && QContext.getQInstance().getTable(name) != null)
+      {
+         try
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(name);
+            queryInput.setFilter(new QQueryFilter().withLimit(1));
+            List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+            read = rows.isEmpty() ? null : rows.get(0);
+         }
+         catch(Exception | LinkageError e)
+         {
+            LOG.debug("Could not read the Quick Search settings; treating indexing as not paused", e);
+         }
+      }
+      settings             = read;
+      settingsReadAtMillis = now;
+      return (read);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an admin has paused all indexing (quick_search_settings).
+    *******************************************************************************/
+   public boolean isIndexingPaused()
+   {
+      QRecord row = getSettings();
+      return (row != null && Boolean.TRUE.equals(row.getValueBoolean("indexingPaused")));
+   }
+
+
+
+   /*******************************************************************************
+    ** All rows of a config table.
+    *******************************************************************************/
+   private static List<QRecord> query(String tableName) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter());
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigWrites.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Marks the QBit's own writes to its config tables (priming, status updates
+ ** made on the owner's behalf), so the table customizers can tell them from
+ ** an admin's edits: only an admin's edit marks a row ownerEdited, writes a
+ ** tombstone, or flags the index for a rebuild.
+ *******************************************************************************/
+public final class QuickSearchConfigWrites
+{
+   private static final ThreadLocal<Boolean> SYSTEM_WRITE = ThreadLocal.withInitial(() -> false);
+
+
+
+   /*******************************************************************************
+    ** Work that may throw.
+    *******************************************************************************/
+   @FunctionalInterface
+   public interface Work<T>
+   {
+      /*******************************************************************************
+       ** Do it.
+       *******************************************************************************/
+      T run() throws QException;
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private QuickSearchConfigWrites()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Run work as a system write (nested calls stay system writes).
+    *******************************************************************************/
+   public static <T> T asSystem(Work<T> work) throws QException
+   {
+      boolean previous = SYSTEM_WRITE.get();
+      SYSTEM_WRITE.set(true);
+      try
+      {
+         return (work.run());
+      }
+      finally
+      {
+         SYSTEM_WRITE.set(previous);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the current thread is making a system write.
+    *******************************************************************************/
+   public static boolean isSystemWrite()
+   {
+      return (SYSTEM_WRITE.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** The current user's name for audit-style columns (pausedBy, removedBy):
+    ** full name, else id, else "system".
+    *******************************************************************************/
+   public static String currentUserName()
+   {
+      QSession session = QContext.getQSession();
+      QUser    user    = session == null ? null : session.getUser();
+      if(user != null && StringUtils.hasContent(user.getFullName()))
+      {
+         return (user.getFullName());
+      }
+      if(user != null && StringUtils.hasContent(user.getIdReference()))
+      {
+         return (user.getIdReference());
+      }
+      return ("system");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchFieldRules.java

@@ -0,0 +1,443 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.WeakHashMap;
+import java.util.regex.Pattern;
+import com.kingsrook.qqq.backend.core.actions.ai.AiDataClassifications;
+import com.kingsrook.qqq.backend.core.actions.ai.DataClassification;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+
+
+/*******************************************************************************
+ ** Which tables and fields may be indexed, and the defaults offered for a new
+ ** index. Shared by the Add Search Index process, the eligible-table and
+ ** field possible-value sources, the field table's validation and priming,
+ ** so all of them agree.
+ **
+ ** A table is eligible when it is in the instance, queryable, not hidden, not
+ ** on backend variants, not produced by a QBit (qbit-internal), not a QQQ
+ ** system table (audits, saved views and reports, scheduled jobs, scripts),
+ ** not named like a secret store, and has at least one indexable field. A
+ ** field is indexable when it is not PII/PHI-classified, locked, hidden,
+ ** sensitive, masked, a blob or a boolean.
+ *******************************************************************************/
+public final class QuickSearchFieldRules
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchFieldRules.class);
+   private static final WeakHashMap<QTableMetaData, Boolean> WARNED_TABLES = ClusterCaches.nodeLocal(new WeakHashMap<>(),
+      "warn-once log de-duplication for this node's table metadata objects");
+
+   public static final int MAX_DEFAULT_FIELDS = 20;
+   public static final int TITLE_WEIGHT       = 3;
+
+   private static final Set<String> SYSTEM_TABLES = Set.of(
+      "audit", "auditdetail", "audittable", "audituser", "savedview", "savedreport", "sharedsavedview", "sharedsavedreport",
+      "scheduledjob", "scheduledjobparameter", "script", "scriptrevision", "scriptrevisionfile", "scriptlog", "scriptlogline",
+      "scripttype", "scripttypefilemode", "tabletrigger", "qqqtable", "qqqtablecache", "qqqprocess", "processtrace", "cachedrecord");
+
+   private static final Pattern SECRET_NAME = Pattern.compile("(secret|credential|password|passwd|token|api_?key|session|oauth|private_?key)", Pattern.CASE_INSENSITIVE);
+
+
+
+   /*******************************************************************************
+    ** A field picked by default for a new index.
+    *******************************************************************************/
+   public record DefaultField(String fieldName, Integer weight, QuickSearchFieldRole role)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private QuickSearchFieldRules()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Why a field cannot be indexed, or null when it can.
+    ** When the qbit moves to QQQ 4.1.0-RC.7, also honour core
+    ** ClassifiedValues.isClassified(table, field) (the host's AuditValueProtector
+    ** classification, qqq#1182) and FieldAccessHelper.getLockTier (qqq#1172).
+    *******************************************************************************/
+   public static String whyNotIndexable(QTableMetaData table, String fieldName)
+   {
+      if(table == null)
+      {
+         return ("the table is not in this application");
+      }
+      QFieldMetaData field = fieldName == null ? null : table.getFields().get(fieldName);
+      if(field == null)
+      {
+         return ("field [" + fieldName + "] does not exist on " + label(table));
+      }
+      if(Boolean.TRUE.equals(field.getIsHidden()))
+      {
+         return ("field [" + fieldName + "] is hidden");
+      }
+      DataClassification classification = AiDataClassifications.ofField(table, fieldName);
+      if(classification == DataClassification.PII || classification == DataClassification.PHI)
+      {
+         return ("field [" + fieldName + "] is classified " + classification + " and cannot be indexed");
+      }
+      if(field.getFieldSecurityLock() != null)
+      {
+         return ("field [" + fieldName + "] has a field security lock and cannot be indexed");
+      }
+      if(field.getIsSensitive() || (field.getType() != null && field.getType().needsMasked()))
+      {
+         return ("field [" + fieldName + "] is sensitive (its values must not reach the search index)");
+      }
+      if(field.getType() == null || field.getType() == QFieldType.BLOB || field.getType() == QFieldType.BOOLEAN)
+      {
+         return ("field [" + fieldName + "] is a " + (field.getType() == null ? "typeless" : field.getType().name().toLowerCase(Locale.ROOT)) + " field, which cannot be searched");
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field can be indexed.
+    *******************************************************************************/
+   public static boolean isIndexable(QTableMetaData table, String fieldName)
+   {
+      return (whyNotIndexable(table, fieldName) == null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field contains protected values, independently of searchability.
+    ** Classification falls back to the table. When the qbit moves to QQQ
+    ** 4.1.0-RC.7, also honour ClassifiedValues.isClassified(table, field)
+    ** (qqq#1182) and FieldAccessHelper.getLockTier (qqq#1172).
+    *******************************************************************************/
+   public static boolean isProtected(QTableMetaData table, String fieldName)
+   {
+      QFieldMetaData field = table == null || fieldName == null ? null : table.getFields().get(fieldName);
+      if(field == null)
+      {
+         return (false);
+      }
+      DataClassification classification = AiDataClassifications.ofField(table, fieldName);
+      return (classification == DataClassification.PII || classification == DataClassification.PHI
+         || field.getFieldSecurityLock() != null || field.getIsSensitive()
+         || (field.getType() != null && field.getType().needsMasked()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Warn once per table metadata lifetime; never include record values.
+    *******************************************************************************/
+   public static synchronized void warnExcludedFields(QTableMetaData table)
+   {
+      if(table != null && WARNED_TABLES.put(table, Boolean.TRUE) == null)
+      {
+         LOG.warn("Quick Search dropped fields that cannot be indexed (classified PII/PHI, secured, hidden, sensitive or unsupported fields) from table [" + table.getName() + "]");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Filter explicit field lists as well as cached configurations at use time.
+    *******************************************************************************/
+   public static List<String> indexableFields(QTableMetaData table, List<String> fields)
+   {
+      List<String> result = new ArrayList<>();
+      for(String field : CollectionUtils.nonNullList(fields))
+      {
+         if(isIndexable(table, field))
+         {
+            result.add(field);
+         }
+         else
+         {
+            warnExcludedFields(table);
+         }
+      }
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve a table's metadata for backend query and result filtering.
+    *******************************************************************************/
+   public static QTableMetaData lookupTable(String tableName)
+   {
+      QInstance instance = QContext.getQInstance();
+      return (instance == null || tableName == null ? null : instance.getTable(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Combined text and labels from an older index cannot be attributed to
+    ** individual fields. Use only allowed field values for these tables.
+    *******************************************************************************/
+   public static boolean requiresFieldFiltering(String tableName)
+   {
+      QTableMetaData table = lookupTable(tableName);
+      return (table != null && table.getFields().keySet().stream().anyMatch(f -> isProtected(table, f)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A protected primary key cannot be sent as an index id, even for deletes.
+    *******************************************************************************/
+   public static boolean hasProtectedPrimaryKey(String tableName)
+   {
+      QTableMetaData table = lookupTable(tableName);
+      return (table != null && isProtected(table, table.getPrimaryKeyField()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The indexable fields of a table, in the table's order.
+    *******************************************************************************/
+   public static List<QFieldMetaData> indexableFields(QTableMetaData table)
+   {
+      List<QFieldMetaData> fields = new ArrayList<>();
+      if(table != null)
+      {
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            if(isIndexable(table, field.getName()))
+            {
+               fields.add(field);
+            }
+         }
+      }
+      return (fields);
+   }
+
+
+
+   /*******************************************************************************
+    ** The fields a new index starts with: the record-label fields as TITLE
+    ** (weight 3), then the other text fields (string, text, HTML) as BODY
+    ** (weight 1), up to MAX_DEFAULT_FIELDS. The admin adjusts them afterwards on
+    ** the index's field rows.
+    *******************************************************************************/
+   public static List<DefaultField> defaultFields(QTableMetaData table)
+   {
+      List<DefaultField> fields = new ArrayList<>();
+      Set<String>        picked = new LinkedHashSet<>();
+      if(table == null)
+      {
+         return (fields);
+      }
+
+      for(String labelField : CollectionUtils.nonNullList(table.getRecordLabelFields()))
+      {
+         if(isIndexable(table, labelField) && picked.add(labelField))
+         {
+            fields.add(new DefaultField(labelField, TITLE_WEIGHT, QuickSearchFieldRole.TITLE));
+         }
+      }
+
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(fields.size() >= MAX_DEFAULT_FIELDS)
+         {
+            break;
+         }
+         if(field.getType() != null && field.getType().isStringLike() && isIndexable(table, field.getName()) && picked.add(field.getName()))
+         {
+            fields.add(new DefaultField(field.getName(), 1, QuickSearchFieldRole.BODY));
+         }
+      }
+      return (fields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a field can drive incremental basepull (a date-time).
+    *******************************************************************************/
+   public static boolean isTimestampField(QTableMetaData table, String fieldName)
+   {
+      QFieldMetaData field = table == null || fieldName == null ? null : table.getFields().get(fieldName);
+      return (field != null && field.getType() == QFieldType.DATE_TIME);
+   }
+
+
+
+   /*******************************************************************************
+    ** The timestamp field a new index starts with: modifyDate when the table
+    ** has it, else the first date-time field named like a modify or update
+    ** time, else none (incremental basepull off; reconcile covers the table).
+    *******************************************************************************/
+   public static String defaultTimestampField(QTableMetaData table)
+   {
+      if(table == null)
+      {
+         return (null);
+      }
+      if(isTimestampField(table, QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD))
+      {
+         return (QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD);
+      }
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         String name = field.getName().toLowerCase(Locale.ROOT);
+         if(field.getType() == QFieldType.DATE_TIME && (name.contains("modif") || name.contains("updat")))
+         {
+            return (field.getName());
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Why a table cannot be indexed at all (for anyone), or null when it can.
+    *******************************************************************************/
+   public static String whyNotEligible(QInstance qInstance, QuickSearchQBitConfig config, String tableName)
+   {
+      QTableMetaData table = qInstance == null || tableName == null ? null : qInstance.getTable(tableName);
+      if(table == null)
+      {
+         return ("table [" + tableName + "] is not in this application");
+      }
+      if(isQuickSearchTable(config, tableName) || table.getSourceQBitName() != null)
+      {
+         return (label(table) + " is internal to a QBit");
+      }
+      String normalized = tableName.replace("_", "").toLowerCase(Locale.ROOT);
+      if(SYSTEM_TABLES.contains(normalized))
+      {
+         return (label(table) + " is a system table");
+      }
+      if(SECRET_NAME.matcher(tableName).find())
+      {
+         return (label(table) + " looks like a store of secrets");
+      }
+      if(table.getIsHidden())
+      {
+         return (label(table) + " is hidden");
+      }
+      QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
+      if(backend != null && Boolean.TRUE.equals(backend.getUsesVariants()))
+      {
+         return (label(table) + " uses backend variants, which Quick Search does not support");
+      }
+      if(!table.isCapabilityEnabled(backend, Capability.TABLE_QUERY))
+      {
+         return (label(table) + " cannot be queried");
+      }
+      if(indexableFields(table).isEmpty())
+      {
+         return (label(table) + " has no field that can be searched");
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the current user may read a table.
+    *******************************************************************************/
+   public static boolean canRead(String tableName)
+   {
+      try
+      {
+         return (PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
+      }
+      catch(Exception e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables the current user could add: eligible, readable and not
+    ** already indexed, sorted by label.
+    *******************************************************************************/
+   public static List<QTableMetaData> eligibleTables(QInstance qInstance, QuickSearchQBitConfig config, Collection<String> alreadyIndexed)
+   {
+      List<QTableMetaData> tables = new ArrayList<>();
+      if(qInstance == null)
+      {
+         return (tables);
+      }
+      for(QTableMetaData table : qInstance.getTables().values())
+      {
+         if(alreadyIndexed.contains(table.getName()) || whyNotEligible(qInstance, config, table.getName()) != null || !canRead(table.getName()))
+         {
+            continue;
+         }
+         tables.add(table);
+      }
+      tables.sort((a, b) -> label(a).compareToIgnoreCase(label(b)));
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table is one of this QBit's own tables.
+    *******************************************************************************/
+   public static boolean isQuickSearchTable(QuickSearchQBitConfig config, String tableName)
+   {
+      return (tableName != null && config != null && tableName.startsWith(config.applyTablePrefix("quick_search_")));
+   }
+
+
+
+   /*******************************************************************************
+    ** A table's label, or its name.
+    *******************************************************************************/
+   public static String label(QTableMetaData table)
+   {
+      return (table.getLabel() == null ? table.getName() : table.getLabel());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchIndexFieldTableCustomizer.java

@@ -0,0 +1,425 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+
+
+/*******************************************************************************
+ ** Customizer of quick_search_index_field, the searchable fields.
+ **
+ ** An admin's insert or edit is validated: the index row exists, the field
+ ** exists on its table and can be indexed (not hidden, not isSensitive, not a
+ ** password, blob or boolean), it is not already a field of that index, the
+ ** weight is 1 to 1000, and the role and analyzer are known. The row is marked
+ ** ownerEdited (an insert also source OWNER), so priming never overwrites it.
+ **
+ ** Changes take effect for searches and real-time indexing at once (weights
+ ** are applied at query time). After an admin's change other than the sort
+ ** order, the index row is marked NEEDS_REINDEX, so the documents indexed
+ ** before can be brought in line: Apply Search Index Changes previews the
+ ** change and rebuilds the table. Removing a field leaves a tombstone, and
+ ** adding it back clears it. Every change drops the cached config.
+ *******************************************************************************/
+public class QuickSearchIndexFieldTableCustomizer implements TableCustomizerInterface
+{
+   public static final int MAX_WEIGHT = 1000;
+
+   public static final List<String> OWNER_FIELDS    = List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel", "sortOrder");
+   public static final List<String> DOCUMENT_FIELDS = List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel");
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      Set<String> inBatch = new HashSet<>();
+      for(QRecord record : records)
+      {
+         Integer indexId = record.getValueInteger("quickSearchIndexId");
+         QRecord index   = indexRow(indexId);
+         if(index == null)
+         {
+            record.addError(new BadInputStatusMessage("Choose the search index this field belongs to"));
+            continue;
+         }
+
+         String fieldName = record.getValueString("fieldName");
+         if(!validateField(record, index, fieldName, null) || !inBatch.add(indexId + "." + fieldName))
+         {
+            if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+            {
+               record.addError(new BadInputStatusMessage("Field [" + fieldName + "] is listed twice"));
+            }
+            continue;
+         }
+
+         if(record.getValue("weight") == null)
+         {
+            record.setValue("weight", 1);
+         }
+         if(!StringUtils.hasContent(record.getValueString("role")))
+         {
+            record.setValue("role", QuickSearchFieldRole.BODY.name());
+         }
+         if(record.getValue("includeLabel") == null)
+         {
+            record.setValue("includeLabel", false);
+         }
+         if(record.getValue("sortOrder") == null)
+         {
+            record.setValue("sortOrder", nextSortOrder(indexId));
+         }
+         validateOptions(record);
+         record.setValue("source", QuickSearchIndexField.SOURCE_OWNER);
+         record.setValue("ownerEdited", true);
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old == null)
+         {
+            continue;
+         }
+         if(record.getValues().containsKey("quickSearchIndexId") && !Objects.equals(record.getValueInteger("quickSearchIndexId"), old.getValueInteger("quickSearchIndexId")))
+         {
+            record.addError(new BadInputStatusMessage("A field cannot be moved to another search index; remove it and add it there"));
+            continue;
+         }
+         if(!changes(record, old, OWNER_FIELDS))
+         {
+            continue;
+         }
+         if(record.getValues().containsKey("fieldName") && !Objects.equals(record.getValueString("fieldName"), old.getValueString("fieldName")))
+         {
+            validateField(record, indexRow(old.getValueInteger("quickSearchIndexId")), record.getValueString("fieldName"), old.getValue("id"));
+         }
+         validateOptions(record);
+         record.setValue("ownerEdited", true);
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate a field name against its index's table and the index's other
+    ** fields (excludeId: the row being edited). Adds an error and returns false
+    ** when it is not acceptable.
+    *******************************************************************************/
+   private static boolean validateField(QRecord record, QRecord index, String fieldName, Serializable excludeId) throws QException
+   {
+      if(index == null)
+      {
+         record.addError(new BadInputStatusMessage("The search index of this field no longer exists"));
+         return (false);
+      }
+      String         tableName = index.getValueString("tableName");
+      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+      String         problem   = QuickSearchFieldRules.whyNotIndexable(table, fieldName);
+      if(problem != null)
+      {
+         record.addError(new BadInputStatusMessage("Cannot index " + fieldName + ": " + problem));
+         return (false);
+      }
+
+      QQueryFilter filter = new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, index.getValue("id")))
+         .withCriteria(new QFilterCriteria("fieldName", QCriteriaOperator.EQUALS, fieldName));
+      if(excludeId != null)
+      {
+         filter.withCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, excludeId));
+      }
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(fieldTableName());
+      queryInput.setFilter(filter);
+      if(CollectionUtils.nullSafeHasContents(new QueryAction().execute(queryInput).getRecords()))
+      {
+         record.addError(new BadInputStatusMessage(fieldName + " is already a searchable field of " + tableName));
+         return (false);
+      }
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Weight 1 to MAX_WEIGHT; a known role and analyzer.
+    *******************************************************************************/
+   private static void validateOptions(QRecord record)
+   {
+      Integer weight = record.getValueInteger("weight");
+      if(record.getValues().containsKey("weight") && (weight == null || weight < 1 || weight > MAX_WEIGHT))
+      {
+         record.addError(new BadInputStatusMessage("Weight must be from 1 to " + MAX_WEIGHT));
+      }
+      String role = record.getValueString("role");
+      if(StringUtils.hasContent(role) && QuickSearchFieldRole.fromName(role, null) == null)
+      {
+         record.addError(new BadInputStatusMessage("Unknown role [" + role + "]"));
+      }
+      String analyzer = record.getValueString("analyzer");
+      if(StringUtils.hasContent(analyzer) && QuickSearchFieldAnalyzer.fromName(analyzer, null) == null)
+      {
+         record.addError(new BadInputStatusMessage("Unknown analyzer [" + analyzer + "]"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         for(QRecord record : succeeded(records))
+         {
+            QRecord index = indexRow(record.getValueInteger("quickSearchIndexId"));
+            if(index != null)
+            {
+               QuickSearchTombstones.remove(index.getValueString("tableName"), record.getValueString("fieldName"));
+               markNeedsReindex(index);
+            }
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+         Map<Integer, QRecord>      marked  = new HashMap<>();
+         for(QRecord record : succeeded(records))
+         {
+            QRecord old = oldById.get(record.getValue("id"));
+            if(old != null && changes(record, old, DOCUMENT_FIELDS))
+            {
+               Integer indexId = old.getValueInteger("quickSearchIndexId");
+               if(!marked.containsKey(indexId))
+               {
+                  QRecord index = indexRow(indexId);
+                  marked.put(indexId, index);
+                  markNeedsReindex(index);
+               }
+            }
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** An admin's removal leaves a tombstone (unless the whole index is going).
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         for(QRecord record : succeeded(records))
+         {
+            QRecord index = indexRow(record.getValueInteger("quickSearchIndexId"));
+            if(index != null)
+            {
+               QuickSearchTombstones.add(index.getValueString("tableName"), record.getValueString("fieldName"), "removed by an admin");
+               markNeedsReindex(index);
+            }
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Mark the index NEEDS_REINDEX (a system write), unless a build is already
+    ** queued or running.
+    *******************************************************************************/
+   static void markNeedsReindex(QRecord index) throws QException
+   {
+      if(index == null)
+      {
+         return;
+      }
+      String status = index.getValueString("status");
+      if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status)
+         || AbstractIndexingStep.STATUS_REBUILDING.equals(status) || AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+      {
+         return;
+      }
+      QuickSearchConfigWrites.asSystem(() ->
+      {
+         QuickSearchConfigPrimer.updateAll(QuickSearchTablePossibleValueProviders.config().getQuickSearchIndexTableName(),
+            List.of(new QRecord().withValue("id", index.getValue("id")).withValue("status", AbstractIndexingStep.STATUS_NEEDS_REINDEX)));
+         return (null);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the update changes any of the given fields.
+    *******************************************************************************/
+   private static boolean changes(QRecord record, QRecord old, List<String> fields)
+   {
+      for(String field : fields)
+      {
+         if(record.getValues().containsKey(field) && !Objects.equals(normalize(record.getValue(field)), normalize(old.getValue(field))))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare values as strings, blank as null.
+    *******************************************************************************/
+   private static String normalize(Serializable value)
+   {
+      String s = value == null ? null : String.valueOf(value);
+      return (StringUtils.hasContent(s) ? s : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The records without errors.
+    *******************************************************************************/
+   private static List<QRecord> succeeded(List<QRecord> records)
+   {
+      return (CollectionUtils.nonNullList(records).stream().filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** The index row, or null.
+    *******************************************************************************/
+   static QRecord indexRow(Integer indexId) throws QException
+   {
+      if(indexId == null)
+      {
+         return (null);
+      }
+      GetInput getInput = new GetInput(QuickSearchTablePossibleValueProviders.config().getQuickSearchIndexTableName());
+      getInput.setPrimaryKey(indexId);
+      return (new GetAction().execute(getInput).getRecord());
+   }
+
+
+
+   /*******************************************************************************
+    ** One past the index's highest sortOrder, in steps of 10.
+    *******************************************************************************/
+   private static Integer nextSortOrder(Integer indexId) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(fieldTableName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexId)));
+      int max = 0;
+      for(QRecord row : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+      {
+         max = Math.max(max, row.getValueInteger("sortOrder") == null ? 0 : row.getValueInteger("sortOrder"));
+      }
+      return (max + 10);
+   }
+
+
+
+   /*******************************************************************************
+    ** The field table's name.
+    *******************************************************************************/
+   private static String fieldTableName()
+   {
+      QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
+      return (config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchIndexTableCustomizer.java

@@ -0,0 +1,239 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+
+
+/*******************************************************************************
+ ** Customizer of quick_search_index, the searchable-table registry.
+ **
+ ** Rows are added by Add Search Index, which validates the table (eligible,
+ ** readable, not yet indexed), previews and queues the first build, and by
+ ** priming; the generic insert is turned off. Any other insert is marked
+ ** source OWNER and ownerEdited, and its schedule validated. An admin's edit of the
+ ** enabled flag, interval or timestamp field marks the row ownerEdited, so
+ ** priming never overwrites it; turning enabled back on queues a build. Any
+ ** change drops the cached config, so it takes effect at once on this server.
+ ** The QBit's own writes (QuickSearchConfigWrites) pass straight through.
+ *******************************************************************************/
+public class QuickSearchIndexTableCustomizer implements TableCustomizerInterface
+{
+   public static final List<String> CONFIG_FIELDS = List.of("enabled", "basepullIntervalMinutes", "basepullTimestampField");
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // rows are added by Add Search Index (which validates and previews) and  //
+      // by priming; the generic insert is turned off. Anything else inserting  //
+      // a row is the owner's doing: mark it so priming never overwrites it     //
+      ////////////////////////////////////////////////////////////////////////////
+      for(QRecord record : records)
+      {
+         if(record.getValue("source") == null)
+         {
+            record.setValue("source", QuickSearchIndex.SOURCE_OWNER);
+         }
+         if(record.getValue("ownerEdited") == null)
+         {
+            record.setValue("ownerEdited", true);
+         }
+         String tableName = record.getValueString("tableName");
+         if(QContext.getQInstance().getTable(tableName) != null)
+         {
+            validateSchedule(record, tableName);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      if(QuickSearchConfigWrites.isSystemWrite())
+      {
+         return (records);
+      }
+
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old == null || !changesConfig(record, old))
+         {
+            continue;
+         }
+
+         validateSchedule(record, old.getValueString("tableName"));
+         record.setValue("ownerEdited", true);
+
+         boolean wasEnabled = !Boolean.FALSE.equals(old.getValueBoolean("enabled"));
+         boolean nowEnabled = !Boolean.FALSE.equals(record.getValues().containsKey("enabled") ? record.getValueBoolean("enabled") : old.getValueBoolean("enabled"));
+         String  status     = old.getValueString("status");
+         if(!wasEnabled && nowEnabled && !AbstractIndexingStep.STATUS_REBUILDING.equals(status) && !AbstractIndexingStep.STATUS_BUILDING.equals(status))
+         {
+            record.setValue("status", AbstractIndexingStep.STATUS_PENDING_BUILD);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the update changes the enabled flag, interval or timestamp field.
+    *******************************************************************************/
+   static boolean changesConfig(QRecord record, QRecord old)
+   {
+      for(String field : CONFIG_FIELDS)
+      {
+         if(record.getValues().containsKey(field) && !Objects.equals(normalize(record.getValue(field)), normalize(old.getValue(field))))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare values as strings, blank as null (forms send "" and "true").
+    *******************************************************************************/
+   private static String normalize(Serializable value)
+   {
+      String s = value == null ? null : String.valueOf(value);
+      return (StringUtils.hasContent(s) ? s : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** A timestamp field must be a date-time field of the table; the interval
+    ** cannot be negative.
+    *******************************************************************************/
+   private static void validateSchedule(QRecord record, String tableName)
+   {
+      String timestampField = record.getValueString("basepullTimestampField");
+      if(StringUtils.hasContent(timestampField) && !QuickSearchFieldRules.isTimestampField(QContext.getQInstance().getTable(tableName), timestampField))
+      {
+         record.addError(new BadInputStatusMessage("Timestamp field [" + timestampField + "] is not a date-time field of " + tableName));
+      }
+      Integer interval = record.getValueInteger("basepullIntervalMinutes");
+      if(record.getValues().containsKey("basepullIntervalMinutes") && interval != null && interval < 0)
+      {
+         record.addError(new BadInputStatusMessage("Update interval cannot be negative (0 means every update)"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** An admin's delete leaves a tombstone, so code defaults do not add the
+    ** table back.
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      if(!QuickSearchConfigWrites.isSystemWrite())
+      {
+         for(QRecord record : CollectionUtils.nonNullList(records))
+         {
+            if(CollectionUtils.nullSafeIsEmpty(record.getErrors()) && record.getValueString("tableName") != null)
+            {
+               QuickSearchTombstones.add(record.getValueString("tableName"), null, "removed by an admin");
+            }
+         }
+      }
+      invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the runtime's cached config.
+    *******************************************************************************/
+   static void invalidate()
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime != null)
+      {
+         runtime.invalidateConfig();
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchSettingsTableCustomizer.java

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Customizer of quick_search_settings. Pausing stamps pausedAt and pausedBy
+ ** (the Quick Search home shows "Paused since ... by ..."); resuming clears
+ ** them and hands the changes deferred while paused (DEFERRED failed events)
+ ** to the next basepull (PENDING). Every change drops the cached settings, so
+ ** a pause takes effect at once on this server and within
+ ** QuickSearchConfigStore.CACHE_TTL_MILLIS on the others.
+ *******************************************************************************/
+public class QuickSearchSettingsTableCustomizer implements TableCustomizerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchSettingsTableCustomizer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old == null || !record.getValues().containsKey("indexingPaused"))
+         {
+            continue;
+         }
+         boolean wasPaused = Boolean.TRUE.equals(old.getValueBoolean("indexingPaused"));
+         boolean nowPaused = Boolean.TRUE.equals(record.getValueBoolean("indexingPaused"));
+         if(!wasPaused && nowPaused)
+         {
+            record.setValue("pausedAt", Instant.now());
+            record.setValue("pausedBy", QuickSearchConfigWrites.currentUserName());
+         }
+         else if(wasPaused && !nowPaused)
+         {
+            record.setValue("pausedAt", null);
+            record.setValue("pausedBy", null);
+            record.setValue("pauseReason", null);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
+      for(QRecord record : records)
+      {
+         QRecord old = oldById.get(record.getValue("id"));
+         if(old != null && CollectionUtils.nullSafeIsEmpty(record.getErrors()) && Boolean.TRUE.equals(old.getValueBoolean("indexingPaused"))
+            && record.getValues().containsKey("indexingPaused") && !Boolean.TRUE.equals(record.getValueBoolean("indexingPaused")))
+         {
+            releaseDeferredEvents();
+         }
+      }
+      QuickSearchIndexTableCustomizer.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Make the events deferred while paused PENDING, so the next basepull
+    ** replays them. Never throws (they stay DEFERRED; a reconcile covers them).
+    *******************************************************************************/
+   static void releaseDeferredEvents()
+   {
+      try
+      {
+         QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
+         String                table  = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+         if(QContext.getQInstance().getTable(table) == null)
+         {
+            return;
+         }
+
+         while(true)
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table);
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_DEFERRED)).withLimit(1000));
+            List<QRecord> deferred = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+            if(deferred.isEmpty())
+            {
+               return;
+            }
+            List<QRecord> updates = new ArrayList<>();
+            deferred.forEach(r -> updates.add(new QRecord().withValue("id", r.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING)));
+            QuickSearchConfigPrimer.updateAll(table, updates);
+            LOG.info("Released changes deferred while Quick Search indexing was paused", logPair("count", updates.size()));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not release the changes deferred while indexing was paused; the next reconcile covers them", e);
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchStatus.java

@@ -0,0 +1,208 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+
+
+/*******************************************************************************
+ ** The indexing status for the Quick Search home: the global pause ("Paused
+ ** since ... by ..."), the last self-priming report ("New: 2 tables added and
+ ** building..."), the tables queued or building, and those whose field
+ ** changes wait for Apply Search Index Changes. The QBit's diagnostics widget
+ ** shows it; hosts with their own home can use read() and bannerHtml().
+ **
+ ** @param paused             whether all indexing is paused
+ ** @param pausedAt           since when (null when not paused)
+ ** @param pausedBy           by whom
+ ** @param pauseReason        the reason given, if any
+ ** @param lastPrimedAt       when self-priming last ran
+ ** @param lastPrimingSummary what it last changed
+ ** @param building           labels of tables queued for or in a build
+ ** @param needsReindex       labels of tables with field changes to apply
+ ** @param recordsMode        whether the config comes from the records (else
+ **                           code: the config tables are missing or not primed)
+ *******************************************************************************/
+public record QuickSearchStatus(boolean paused, Instant pausedAt, String pausedBy, String pauseReason, Instant lastPrimedAt, String lastPrimingSummary,
+                                List<String> building, List<String> needsReindex, boolean recordsMode)
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchStatus.class);
+
+   private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
+
+
+
+   /*******************************************************************************
+    ** Read the status. Never throws; parts that cannot be read are empty.
+    *******************************************************************************/
+   public static QuickSearchStatus read(QuickSearchRuntime runtime)
+   {
+      List<String> building     = new ArrayList<>();
+      List<String> needsReindex = new ArrayList<>();
+      if(runtime == null)
+      {
+         return (new QuickSearchStatus(false, null, null, null, null, null, building, needsReindex, false));
+      }
+
+      try
+      {
+         String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+         if(QContext.getQInstance() != null && QContext.getQInstance().getTable(indexTable) != null)
+         {
+            for(QRecord row : QuickSearchConfigPrimer.queryAll(indexTable))
+            {
+               String status = row.getValueString("status");
+               String label  = labelOf(row.getValueString("tableName"));
+               if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
+               {
+                  continue;
+               }
+               if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status)
+                  || AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+               {
+                  building.add(label);
+               }
+               else if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+               {
+                  needsReindex.add(label);
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not read the index rows for the Quick Search status", e);
+      }
+
+      QRecord settings = runtime.getConfigStore().getSettings();
+      boolean paused   = settings != null && Boolean.TRUE.equals(settings.getValueBoolean("indexingPaused"));
+      return (new QuickSearchStatus(paused,
+         settings == null ? null : settings.getValueInstant("pausedAt"),
+         settings == null ? null : settings.getValueString("pausedBy"),
+         settings == null ? null : settings.getValueString("pauseReason"),
+         settings == null ? null : settings.getValueInstant("lastPrimedAt"),
+         settings == null ? null : settings.getValueString("lastPrimingSummary"),
+         building, needsReindex, runtime.getConfigStore().isRecordsMode()));
+   }
+
+
+
+   /*******************************************************************************
+    ** "Paused since 2026-10-12T14:03:00Z by Jane Admin (reason)".
+    *******************************************************************************/
+   public String pausedText()
+   {
+      return ("Paused since " + (pausedAt == null ? "an unknown time" : WHEN.format(pausedAt)) + " by " + (pausedBy == null ? "an unknown user" : pausedBy)
+         + (StringUtils.hasContent(pauseReason) ? " (" + pauseReason + ")" : "")
+         + ". Nothing is indexed; search keeps serving the current index, and changes are indexed when indexing resumes.");
+   }
+
+
+
+   /*******************************************************************************
+    ** The building line: "Building: Blog Post, Page", or null.
+    *******************************************************************************/
+   public String buildingText()
+   {
+      if(building.isEmpty())
+      {
+         return (null);
+      }
+      return ((StringUtils.hasContent(lastPrimingSummary) && lastPrimingSummary.startsWith("New:") ? lastPrimingSummary + " " : "")
+         + "Building now or with the next update: " + String.join(", ", building) + ".");
+   }
+
+
+
+   /*******************************************************************************
+    ** The pending-changes line, or null.
+    *******************************************************************************/
+   public String needsReindexText()
+   {
+      return (needsReindex.isEmpty() ? null : "Field changes waiting to be applied (Apply Search Index Changes): " + String.join(", ", needsReindex) + ".");
+   }
+
+
+
+   /*******************************************************************************
+    ** The status as standalone HTML, for a host's own home widget: one line
+    ** per item, empty when there is nothing to report.
+    *******************************************************************************/
+   public String bannerHtml()
+   {
+      StringBuilder html = new StringBuilder();
+      if(paused)
+      {
+         html.append(banner("#b26a00", "#fff4e5", pausedText()));
+      }
+      if(buildingText() != null)
+      {
+         html.append(banner("#0b5394", "#e8f0fe", buildingText()));
+      }
+      if(needsReindexText() != null)
+      {
+         html.append(banner("#5f6368", "#f1f3f4", needsReindexText()));
+      }
+      return (html.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** One banner line.
+    *******************************************************************************/
+   private static String banner(String color, String background, String text)
+   {
+      return ("<div style=\"padding:8px 12px;margin:0 0 8px 0;border-radius:6px;border-left:4px solid " + color + ";background:" + background + ";color:#202124\">"
+         + DiagnosticsHtml.escape(text) + "</div>");
+   }
+
+
+
+   /*******************************************************************************
+    ** A table's label, or its name.
+    *******************************************************************************/
+   private static String labelOf(String tableName)
+   {
+      var table = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+      return (table == null ? String.valueOf(tableName) : QuickSearchFieldRules.label(table));
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings table name for links ("/app/" + name + "/1").
+    *******************************************************************************/
+   public static String settingsPath(QuickSearchRuntime runtime)
+   {
+      return ("/app/" + runtime.getConfig().applyTablePrefix(QuickSearchSettings.TABLE_NAME) + "/" + QuickSearchSettings.SETTINGS_ID);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchTablePossibleValueProviders.java

@@ -0,0 +1,321 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+
+
+/*******************************************************************************
+ ** Custom possible-value providers over the instance's tables and fields, for
+ ** the config screens:
+ ** - SearchableTables: the indexed tables (quick_search_searchable_table),
+ **   shown by label wherever a table name is stored;
+ ** - EligibleTables: tables the current user could add (readable, eligible,
+ **   not yet indexed), for Add Search Index;
+ ** - IndexableFields and TimestampFields: the indexable or date-time fields
+ **   of the table the form is about, read from the form's other values
+ **   (tableName, or the field row's quickSearchIndexId).
+ *******************************************************************************/
+public final class QuickSearchTablePossibleValueProviders
+{
+
+   /*******************************************************************************
+    ** Utility holder - no instances.
+    *******************************************************************************/
+   private QuickSearchTablePossibleValueProviders()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The QBit's config: the runtime's, else the static facade's.
+    *******************************************************************************/
+   static QuickSearchQBitConfig config()
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig());
+   }
+
+
+
+   /*******************************************************************************
+    ** Table possible values whose id is the table name and label its label.
+    *******************************************************************************/
+   abstract static class AbstractTableProvider implements QCustomPossibleValueProvider<String>
+   {
+      /*******************************************************************************
+       ** The table names to offer.
+       *******************************************************************************/
+      abstract List<String> tableNames() throws QException;
+
+
+
+      /*******************************************************************************
+       ** Any table in the instance, by label.
+       *******************************************************************************/
+      @Override
+      public QPossibleValue<String> getPossibleValue(Serializable idValue)
+      {
+         String         tableName = ValueUtils.getValueAsString(idValue);
+         QTableMetaData table     = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+         return (tableName == null ? null : new QPossibleValue<>(tableName, table == null ? tableName : QuickSearchFieldRules.label(table)));
+      }
+
+
+
+      /*******************************************************************************
+       ** The offered tables matching the search term (or ids).
+       *******************************************************************************/
+      @Override
+      public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+      {
+         List<QPossibleValue<String>> values = new ArrayList<>();
+         for(String tableName : tableNames())
+         {
+            QPossibleValue<String> value = getPossibleValue(tableName);
+            if(matches(input, value))
+            {
+               values.add(value);
+            }
+         }
+         return (values);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The indexed tables.
+    *******************************************************************************/
+   public static class SearchableTables extends AbstractTableProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      List<String> tableNames()
+      {
+         QuickSearchRuntime runtime = QuickSearchRuntime.get();
+         List<String>       names   = new ArrayList<>();
+         List<QuickSearchableTableConfig> tables = runtime != null ? runtime.getDiscoveredTables() : QuickSearchQBitContext.getDiscoveredTables();
+         for(QuickSearchableTableConfig tableConfig : CollectionUtils.nonNullList(tables))
+         {
+            names.add(tableConfig.getTableName());
+         }
+         return (names);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables the current user could add to Quick Search.
+    *******************************************************************************/
+   public static class EligibleTables extends AbstractTableProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      List<String> tableNames() throws QException
+      {
+         QuickSearchQBitConfig config  = config();
+         Set<String>           indexed = new LinkedHashSet<>();
+         for(QRecord row : QuickSearchConfigPrimer.queryAll(config.getQuickSearchIndexTableName()))
+         {
+            indexed.add(row.getValueString("tableName"));
+         }
+         List<String> names = new ArrayList<>();
+         QuickSearchFieldRules.eligibleTables(QContext.getQInstance(), config, indexed).forEach(t -> names.add(t.getName()));
+         return (names);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Field possible values of the form's table; id is the field name.
+    *******************************************************************************/
+   abstract static class AbstractFieldProvider implements QCustomPossibleValueProvider<String>
+   {
+      /*******************************************************************************
+       ** Whether to offer a field.
+       *******************************************************************************/
+      abstract boolean offers(QTableMetaData table, QFieldMetaData field);
+
+
+
+      /*******************************************************************************
+       ** A field name is shown as itself: without the form's values the table is
+       ** not known here.
+       *******************************************************************************/
+      @Override
+      public QPossibleValue<String> getPossibleValue(Serializable idValue)
+      {
+         String fieldName = ValueUtils.getValueAsString(idValue);
+         return (fieldName == null ? null : new QPossibleValue<>(fieldName, fieldName));
+      }
+
+
+
+      /*******************************************************************************
+       ** The offered fields of the form's table, labeled "Label (name)".
+       *******************************************************************************/
+      @Override
+      public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+      {
+         List<QPossibleValue<String>> values = new ArrayList<>();
+         QTableMetaData               table  = resolveTable(input.getOtherValues());
+         if(table == null)
+         {
+            return (values);
+         }
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            if(!offers(table, field))
+            {
+               continue;
+            }
+            QPossibleValue<String> value = new QPossibleValue<>(field.getName(), (field.getLabel() == null ? field.getName() : field.getLabel()) + " (" + field.getName() + ")");
+            if(matches(input, value))
+            {
+               values.add(value);
+            }
+         }
+         return (values);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Fields that can be indexed.
+    *******************************************************************************/
+   public static class IndexableFields extends AbstractFieldProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      boolean offers(QTableMetaData table, QFieldMetaData field)
+      {
+         return (QuickSearchFieldRules.isIndexable(table, field.getName()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Date-time fields, for the basepull timestamp.
+    *******************************************************************************/
+   public static class TimestampFields extends AbstractFieldProvider
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      boolean offers(QTableMetaData table, QFieldMetaData field)
+      {
+         return (QuickSearchFieldRules.isTimestampField(table, field.getName()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The table a form is about: its searchTableName or tableName value, else
+    ** the table of its quickSearchIndexId row. Null when none resolves.
+    *******************************************************************************/
+   static QTableMetaData resolveTable(Map<String, Serializable> otherValues) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(otherValues == null || qInstance == null)
+      {
+         return (null);
+      }
+
+      ////////////////////////////////////////////////////////////////////////
+      // Add Search Index's searchTableName first: a process form also gets  //
+      // tableName, the table the process runs from                          //
+      ////////////////////////////////////////////////////////////////////////
+      String tableName = ValueUtils.getValueAsString(otherValues.get("searchTableName"));
+      if(!StringUtils.hasContent(tableName))
+      {
+         tableName = ValueUtils.getValueAsString(otherValues.get("tableName"));
+      }
+      if(!StringUtils.hasContent(tableName) && otherValues.get("quickSearchIndexId") != null)
+      {
+         GetInput getInput = new GetInput(config().getQuickSearchIndexTableName());
+         getInput.setPrimaryKey(ValueUtils.getValueAsInteger(otherValues.get("quickSearchIndexId")));
+         QRecord row = new GetAction().execute(getInput).getRecord();
+         tableName = row == null ? null : row.getValueString("tableName");
+      }
+      return (StringUtils.hasContent(tableName) ? qInstance.getTable(tableName) : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a value is wanted: in idList when one is given, else its label or
+    ** id contains the search term (any case).
+    *******************************************************************************/
+   static boolean matches(SearchPossibleValueSourceInput input, QPossibleValue<String> value)
+   {
+      if(value == null)
+      {
+         return (false);
+      }
+      if(CollectionUtils.nullSafeHasContents(input.getIdList()))
+      {
+         return (input.getIdList().stream().anyMatch(id -> value.getId().equals(ValueUtils.getValueAsString(id))));
+      }
+      String term = input.getSearchTerm();
+      if(!StringUtils.hasContent(term))
+      {
+         return (true);
+      }
+      String lower = term.toLowerCase(Locale.ROOT);
+      return (value.getLabel().toLowerCase(Locale.ROOT).contains(lower) || value.getId().toLowerCase(Locale.ROOT).contains(lower));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchTombstones.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.config;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+
+
+/*******************************************************************************
+ ** Reads and writes quick_search_config_tombstone: what an admin removed, so
+ ** priming does not add it back. Every write is a system write. With no
+ ** tombstone table in the instance these do nothing (and priming then cannot
+ ** tell a removed code table from a new one; add the table's DDL).
+ *******************************************************************************/
+public final class QuickSearchTombstones
+{
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private QuickSearchTombstones()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The tombstone table's name, or null when it is not in the instance.
+    *******************************************************************************/
+   static String tableName()
+   {
+      QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
+      String                name   = config == null ? null : config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME);
+      return (name != null && QContext.getQInstance() != null && QContext.getQInstance().getTable(name) != null ? name : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Remember that a table (fieldName null) or one of its fields was removed.
+    *******************************************************************************/
+   public static void add(String tableName, String fieldName, String reason) throws QException
+   {
+      String name = tableName();
+      if(name == null || !find(name, tableName, fieldName).isEmpty())
+      {
+         return;
+      }
+      QuickSearchConfigWrites.asSystem(() -> QuickSearchConfigPrimer.insertOne(name, new QRecord()
+         .withValue("tableName", tableName)
+         .withValue("fieldName", fieldName)
+         .withValue("removedBy", QuickSearchConfigWrites.currentUserName())
+         .withValue("reason", reason)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget a removal (the admin added the table or field back).
+    *******************************************************************************/
+   public static void remove(String tableName, String fieldName) throws QException
+   {
+      String name = tableName();
+      if(name == null)
+      {
+         return;
+      }
+      List<Serializable> ids = new ArrayList<>();
+      find(name, tableName, fieldName).forEach(r -> ids.add(r.getValue("id")));
+      if(ids.isEmpty())
+      {
+         return;
+      }
+      QuickSearchConfigWrites.asSystem(() ->
+      {
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(name);
+         deleteInput.setPrimaryKeys(ids);
+         return (new DeleteAction().execute(deleteInput));
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table (fieldName null) or field has a tombstone.
+    *******************************************************************************/
+   public static boolean exists(String tableName, String fieldName) throws QException
+   {
+      String name = tableName();
+      return (name != null && !find(name, tableName, fieldName).isEmpty());
+   }
+
+
+
+   /*******************************************************************************
+    ** The tombstones for one table (fieldName null) or field.
+    *******************************************************************************/
+   static List<QRecord> find(String name, String tableName, String fieldName) throws QException
+   {
+      QQueryFilter filter = new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName));
+      filter.addCriteria(fieldName == null
+         ? new QFilterCriteria("fieldName", QCriteriaOperator.IS_BLANK)
+         : new QFilterCriteria("fieldName", QCriteriaOperator.EQUALS, fieldName));
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(name);
+      queryInput.setFilter(filter);
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizer.java

@@ -1,106 +0,0 @@
-/*
- * Copyright 2024 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.quicksearch.customizers;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
-import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Post-delete table customizer that removes deleted records from the Quick
- ** Search OpenSearch index.
- **
- ** QQQ instantiates this class via reflection using the no-arg constructor.
- ** The publisher and table configuration are obtained from the static
- ** QuickSearchQBitContext at call time.
- *******************************************************************************/
-public class QuickSearchPostDeleteCustomizer implements TableCustomizerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(QuickSearchPostDeleteCustomizer.class);
-
-
-
-   /*******************************************************************************
-    ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
-    *******************************************************************************/
-   public QuickSearchPostDeleteCustomizer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Post-delete hook: builds DELETE events for each deleted record and publishes
-    ** them via the configured IndexEventPublisher.
-    **
-    ** If the publisher is null (QBit not yet initialized) or if publishing fails,
-    ** a warning is logged and the original records are returned without throwing,
-    ** so as never to block the underlying QQQ delete operation.
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
-   {
-      if(records == null || records.isEmpty())
-      {
-         return records;
-      }
-
-      IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
-      if(publisher == null)
-      {
-         LOG.warn("QuickSearch publisher is null; skipping post-delete indexing", logPair("tableName", deleteInput.getTableName()));
-         return records;
-      }
-
-      try
-      {
-         String tableName = deleteInput.getTableName();
-         QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
-         String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
-
-         List<IndexEvent> events = new ArrayList<>();
-         for(QRecord record : records)
-         {
-            events.add(new IndexEvent()
-               .withAction(IndexEventAction.DELETE)
-               .withTableName(tableName)
-               .withRecordId(record.getValueString(primaryKeyField)));
-         }
-
-         publisher.publishDeleteEvents(events);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to publish post-delete index events", e, logPair("tableName", deleteInput.getTableName()));
-      }
-
-      return records;
-   }
-
-}

src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizer.java

@@ -1,107 +0,0 @@
-/*
- * Copyright 2024 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.quicksearch.customizers;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
-import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Post-insert table customizer that indexes newly inserted records in the
- ** Quick Search OpenSearch index.
- **
- ** QQQ instantiates this class via reflection using the no-arg constructor.
- ** The publisher and table configuration are obtained from the static
- ** QuickSearchQBitContext at call time.
- *******************************************************************************/
-public class QuickSearchPostInsertCustomizer implements TableCustomizerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(QuickSearchPostInsertCustomizer.class);
-
-
-
-   /*******************************************************************************
-    ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
-    *******************************************************************************/
-   public QuickSearchPostInsertCustomizer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Post-insert hook: builds INDEX events for each inserted record and publishes
-    ** them via the configured IndexEventPublisher.
-    **
-    ** If the publisher is null (QBit not yet initialized) or if publishing fails,
-    ** a warning is logged and the original records are returned without throwing,
-    ** so as never to block the underlying QQQ insert operation.
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
-   {
-      if(records == null || records.isEmpty())
-      {
-         return records;
-      }
-
-      IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
-      if(publisher == null)
-      {
-         LOG.warn("QuickSearch publisher is null; skipping post-insert indexing", logPair("tableName", insertInput.getTableName()));
-         return records;
-      }
-
-      try
-      {
-         String tableName = insertInput.getTableName();
-         QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
-         String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
-
-         List<IndexEvent> events = new ArrayList<>();
-         for(QRecord record : records)
-         {
-            events.add(new IndexEvent()
-               .withAction(IndexEventAction.INDEX)
-               .withTableName(tableName)
-               .withRecordId(record.getValueString(primaryKeyField))
-               .withRecord(record));
-         }
-
-         publisher.publishIndexEvents(events);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to publish post-insert index events", e, logPair("tableName", insertInput.getTableName()));
-      }
-
-      return records;
-   }
-
-}

src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizer.java

@@ -1,108 +0,0 @@
-/*
- * Copyright 2024 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.quicksearch.customizers;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
-import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Post-update table customizer that re-indexes updated records in the Quick
- ** Search OpenSearch index.
- **
- ** QQQ instantiates this class via reflection using the no-arg constructor.
- ** The publisher and table configuration are obtained from the static
- ** QuickSearchQBitContext at call time.
- *******************************************************************************/
-public class QuickSearchPostUpdateCustomizer implements TableCustomizerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(QuickSearchPostUpdateCustomizer.class);
-
-
-
-   /*******************************************************************************
-    ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
-    *******************************************************************************/
-   public QuickSearchPostUpdateCustomizer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Post-update hook: builds INDEX events for each updated record and publishes
-    ** them via the configured IndexEventPublisher.
-    **
-    ** If the publisher is null (QBit not yet initialized) or if publishing fails,
-    ** a warning is logged and the original records are returned without throwing,
-    ** so as never to block the underlying QQQ update operation.
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      if(records == null || records.isEmpty())
-      {
-         return records;
-      }
-
-      IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
-      if(publisher == null)
-      {
-         LOG.warn("QuickSearch publisher is null; skipping post-update indexing", logPair("tableName", updateInput.getTableName()));
-         return records;
-      }
-
-      try
-      {
-         String tableName = updateInput.getTableName();
-         QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
-         String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
-
-         List<IndexEvent> events = new ArrayList<>();
-         for(QRecord record : records)
-         {
-            events.add(new IndexEvent()
-               .withAction(IndexEventAction.INDEX)
-               .withTableName(tableName)
-               .withRecordId(record.getValueString(primaryKeyField))
-               .withRecord(record));
-         }
-
-         publisher.publishIndexEvents(events);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to publish post-update index events", e, logPair("tableName", updateInput.getTableName()));
-      }
-
-      return records;
-   }
-
-}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/AnalyzerTester.java

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+
+
+/*******************************************************************************
+ ** The analyzer tester: run _analyze on the live index with its two
+ ** analyzers. The text is analyzed as it would be indexed (standard
+ ** tokenizer, lowercase, edge n-grams of 2 to 20 characters) and as a query
+ ** (standard tokenizer, lowercase); with a query, each query token is marked
+ ** found or not among the text's indexed tokens, which is how matching works
+ ** (every query token must be found). Read-only.
+ *******************************************************************************/
+public class AnalyzerTester
+{
+
+   /*******************************************************************************
+    ** The tokens of the text (both analyzers) and of the query.
+    *******************************************************************************/
+   public record Result(String text, String query, List<AnalyzedToken> indexTokens, List<AnalyzedToken> textAsQueryTokens, List<AnalyzedToken> queryTokens)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Analyze text (and an optional query) with the index's analyzers.
+    *******************************************************************************/
+   public Result analyze(QuickSearchRuntime runtime, String text, String query) throws QException
+   {
+      QuickSearchDiagnosticsOperations ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      String                           safeText = text == null ? "" : text;
+      return (new Result(safeText, query,
+         ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, safeText),
+         ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, safeText),
+         StringUtils.hasContent(query) ? ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, query) : null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render a result as HTML.
+    *******************************************************************************/
+   public static String render(Result result)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Analyzer tokens");
+      if(result.text().length() > QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH)
+      {
+         html.status(Level.WARN, "Only the first " + QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH + " characters were analyzed.");
+      }
+
+      Set<String> indexed = new LinkedHashSet<>();
+      result.indexTokens().forEach(t -> indexed.add(t.token()));
+
+      if(result.queryTokens() != null)
+      {
+         html.subheading("Does the query match this text?");
+         List<DiagnosticsHtml.Check> checks = new ArrayList<>();
+         boolean                     all    = !result.queryTokens().isEmpty();
+         for(AnalyzedToken token : result.queryTokens())
+         {
+            boolean found = indexed.contains(token.token());
+            all &= found;
+            String why = found ? "found among the indexed tokens"
+               : token.token().length() < 2 ? "one character: never indexed (n-grams start at 2)"
+               : token.token().length() > 20 ? "longer than 20 characters: never indexed (n-grams stop at 20)"
+               : "no word in the text starts with it";
+            checks.add(new DiagnosticsHtml.Check("\"" + token.token() + "\"", found ? Level.OK : Level.FAIL, why));
+         }
+         html.status(all ? Level.OK : Level.FAIL, all ? "Every query token is found: the searchable-text clause matches."
+            : "At least one query token is missing: with AND semantics, a document with only this text does not match.");
+         html.checks(checks);
+      }
+
+      html.subheading("As indexed (" + QuickSearchOpenSearchClient.INDEX_ANALYZER + "): " + result.indexTokens().size() + " token(s)");
+      html.paragraph(join(result.indexTokens()));
+      html.subheading("As a query (" + QuickSearchOpenSearchClient.SEARCH_ANALYZER + "): " + result.textAsQueryTokens().size() + " token(s)");
+      html.paragraph(join(result.textAsQueryTokens()));
+      if(result.queryTokens() != null)
+      {
+         html.subheading("Query tokens: " + result.queryTokens().size());
+         html.paragraph(join(result.queryTokens()));
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Tokens as "token (position)" separated by commas.
+    *******************************************************************************/
+   static String join(List<AnalyzedToken> tokens)
+   {
+      if(tokens == null || tokens.isEmpty())
+      {
+         return ("(none)");
+      }
+      List<String> parts = new ArrayList<>();
+      for(AnalyzedToken token : tokens)
+      {
+         parts.add(token.token() + " @" + token.position());
+      }
+      return (String.join(", ", parts));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticEventRecorder.java

@@ -0,0 +1,259 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.net.InetAddress;
+import java.time.Instant;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Writes quick_search_diagnostic_event rows: failed searches, diagnostic tool
+ ** runs (the audit trail) and refused runs.
+ ** Best effort and off the caller's thread: events are queued (at most
+ ** QUEUE_CAPACITY) and inserted by one daemon thread as the system user, so
+ ** a slow or failing database never slows or fails a search. At most
+ ** MAX_EVENTS_PER_MINUTE events are written per JVM per minute; the rest are
+ ** counted as dropped (see getDroppedCount), so a search outage cannot flood
+ ** the table. A no-op when the diagnostic tables are not configured.
+ *******************************************************************************/
+public class DiagnosticEventRecorder implements AutoCloseable
+{
+   private static final QLogger LOG = QLogger.getLogger(DiagnosticEventRecorder.class);
+
+   public static final int MAX_EVENTS_PER_MINUTE = 120;
+   public static final int QUEUE_CAPACITY        = 500;
+
+   private final QuickSearchQBitConfig config;
+   private final ThreadPoolExecutor    executor;
+   private final AtomicLong            droppedCount = new AtomicLong();
+   private final AtomicLong            failedCount  = new AtomicLong();
+   private final String                hostName;
+
+   private long windowStartMillis = 0L;
+   private int  windowCount       = 0;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public DiagnosticEventRecorder(QuickSearchQBitConfig config)
+   {
+      this.config   = config;
+      this.hostName = resolveHostName();
+      this.executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(QUEUE_CAPACITY), runnable ->
+      {
+         Thread thread = new Thread(runnable, "quick-search-diagnostic-events");
+         thread.setDaemon(true);
+         return (thread);
+      }, (runnable, pool) -> droppedCount.incrementAndGet());
+   }
+
+
+
+   /*******************************************************************************
+    ** Queue an event for insert. Fills in host name and create date, bounds
+    ** string lengths, and uses the current session's user when none is set.
+    *******************************************************************************/
+   public void record(QuickSearchDiagnosticEvent event)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      String    tableName = config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
+      if(event == null || qInstance == null || !Boolean.TRUE.equals(config.getEnableDiagnosticTools()) || qInstance.getTable(tableName) == null)
+      {
+         return;
+      }
+
+      if(!admit())
+      {
+         droppedCount.incrementAndGet();
+         return;
+      }
+
+      if(event.getUserName() == null)
+      {
+         event.setUserName(DiagnosticsSupport.currentUserName());
+      }
+
+      event.withHostName(truncate(hostName, 100))
+         .withCreateDate(Instant.now())
+         .withSource(truncate(event.getSource(), 50))
+         .withUserName(truncate(event.getUserName(), 250))
+         .withTableNames(truncate(event.getTableNames(), 1000))
+         .withSearchTerm(truncate(event.getSearchTerm(), 100))
+         .withErrorClass(truncate(event.getErrorClass(), 250))
+         .withErrorMessage(truncate(event.getErrorMessage(), 4000))
+         .withDetail(truncate(event.getDetail(), 4000));
+
+      executor.execute(() -> insert(qInstance, tableName, event));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one event as the system user. Failures are logged and counted.
+    *******************************************************************************/
+   private void insert(QInstance qInstance, String tableName, QuickSearchDiagnosticEvent event)
+   {
+      QSession session = new QSystemUserSession(qInstance);
+      try
+      {
+         QContext.init(qInstance, session);
+         new InsertAction().execute(new InsertInput(tableName).withRecordEntity(event));
+      }
+      catch(Exception e)
+      {
+         failedCount.incrementAndGet();
+         LOG.warn("Could not record a Quick Search diagnostic event", e, logPair("eventType", event.getEventType()), logPair("source", event.getSource()));
+      }
+      finally
+      {
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether one more event fits in the current minute's budget.
+    *******************************************************************************/
+   private synchronized boolean admit()
+   {
+      long now = System.currentTimeMillis();
+      if(now - windowStartMillis >= 60_000L)
+      {
+         windowStartMillis = now;
+         windowCount       = 0;
+      }
+      if(windowCount >= MAX_EVENTS_PER_MINUTE)
+      {
+         return (false);
+      }
+      windowCount++;
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Wait (up to timeoutMillis) until every queued event has been written.
+    ** For tests and orderly shutdown.
+    *******************************************************************************/
+   public void flush(long timeoutMillis)
+   {
+      try
+      {
+         executor.submit(() ->
+         {
+         }).get(timeoutMillis, TimeUnit.MILLISECONDS);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Diagnostic event flush did not complete", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Events not written because the queue or the per-minute budget was full.
+    *******************************************************************************/
+   public long getDroppedCount()
+   {
+      return (droppedCount.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Events whose insert failed.
+    *******************************************************************************/
+   public long getFailedCount()
+   {
+      return (failedCount.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop the writer after the queued events (up to 2 seconds).
+    *******************************************************************************/
+   @Override
+   public void close()
+   {
+      executor.shutdown();
+      try
+      {
+         if(!executor.awaitTermination(2, TimeUnit.SECONDS))
+         {
+            executor.shutdownNow();
+         }
+      }
+      catch(InterruptedException e)
+      {
+         executor.shutdownNow();
+         Thread.currentThread().interrupt();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** This JVM's host name (the pod name in Kubernetes).
+    *******************************************************************************/
+   static String resolveHostName()
+   {
+      String fromEnv = System.getenv("HOSTNAME");
+      if(fromEnv != null && !fromEnv.isBlank())
+      {
+         return (fromEnv);
+      }
+      try
+      {
+         return (InetAddress.getLocalHost().getHostName());
+      }
+      catch(Exception e)
+      {
+         return ("unknown");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   static String truncate(String value, int max)
+   {
+      return (value == null || value.length() <= max ? value : value.substring(0, max));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticRecordReads.java

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Source-table reads for the diagnostics, in the current session (so table
+ ** permissions and record security locks apply, as for any query the user
+ ** runs). Records are read the way indexing reads them: display values
+ ** generated and possible values translated, so a document built from them
+ ** is exactly the document indexing would write.
+ *******************************************************************************/
+public final class DiagnosticRecordReads
+{
+   public static final int MAX_IDS_PER_QUERY = 1_000;
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private DiagnosticRecordReads()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Records of a table by primary key (as strings), keyed by primary key
+    ** string, read as indexing reads them. Ids that do not exist or that the
+    ** session cannot read are absent. Ids that do not convert to the key's
+    ** type are skipped.
+    *******************************************************************************/
+   public static Map<String, QRecord> loadForIndexing(String tableName, Collection<String> recordIds) throws QException
+   {
+      QTableMetaData       table   = requireTable(tableName);
+      String               pk      = table.getPrimaryKeyField();
+      Map<String, QRecord> records = new LinkedHashMap<>();
+      List<Serializable>   ids     = typedIds(table, recordIds);
+
+      for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
+         queryInput.setShouldGenerateDisplayValues(true);
+         queryInput.setShouldTranslatePossibleValues(true);
+         for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+         {
+            records.put(record.getValueString(pk), record);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Up to limit records of a table ordered by primary key (ascending or
+    ** descending), read as indexing reads them.
+    *******************************************************************************/
+   public static List<QRecord> loadEdge(String tableName, boolean ascending, int limit) throws QException
+   {
+      QTableMetaData table      = requireTable(tableName);
+      QueryInput     queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(table.getPrimaryKeyField(), ascending)).withLimit(limit));
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldTranslatePossibleValues(true);
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The primary keys (as strings) among recordIds that exist and that the
+    ** session can read.
+    *******************************************************************************/
+   public static List<String> existingIds(String tableName, Collection<String> recordIds) throws QException
+   {
+      QTableMetaData     table  = requireTable(tableName);
+      String             pk     = table.getPrimaryKeyField();
+      List<String>       found  = new ArrayList<>();
+      List<Serializable> ids    = typedIds(table, recordIds);
+
+      for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
+         for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+         {
+            found.add(record.getValueString(pk));
+         }
+      }
+      return (found);
+   }
+
+
+
+   /*******************************************************************************
+    ** Row count of a table in the current session.
+    *******************************************************************************/
+   public static Integer count(String tableName) throws QException
+   {
+      CountInput countInput = new CountInput();
+      countInput.setTableName(tableName);
+      return (new CountAction().execute(countInput).getCount());
+   }
+
+
+
+   /*******************************************************************************
+    ** Ids converted to the primary key's type; ids that do not convert are
+    ** skipped (they cannot exist).
+    *******************************************************************************/
+   static List<Serializable> typedIds(QTableMetaData table, Collection<String> recordIds)
+   {
+      QFieldMetaData     pkField = table.getField(table.getPrimaryKeyField());
+      List<Serializable> ids     = new ArrayList<>();
+      for(String recordId : CollectionUtils.nonNullCollection(recordIds))
+      {
+         try
+         {
+            Serializable id = ValueUtils.getValueAsFieldType(pkField.getType(), recordId);
+            if(id != null && !ids.contains(id))
+            {
+               ids.add(id);
+            }
+         }
+         catch(Exception e)
+         {
+            //////////////////////////////////////////////////////////
+            // e.g. "abc" for an integer key: no such record exists //
+            //////////////////////////////////////////////////////////
+         }
+      }
+      return (ids);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QTableMetaData requireTable(String tableName) throws QException
+   {
+      QTableMetaData table = QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw (new QException("Table [" + tableName + "] is not in the instance"));
+      }
+      return (table);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsHtml.java

@@ -0,0 +1,362 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.List;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+
+/*******************************************************************************
+ ** A small HTML builder for the diagnostic screens (process HTML blocks, the
+ ** validation report field and the diagnostics widget). Every text value is
+ ** escaped. Layout rules: one column of blocks capped at a readable width,
+ ** no fixed widths, and long values (JSON, ids, text) wrap instead of
+ ** scrolling, so a screen reads from 360px to 3200px without an inner or
+ ** horizontal scrollbar. Colours are mid-tone so they read in light and dark
+ ** themes; status is always also spelled out in text.
+ *******************************************************************************/
+public class DiagnosticsHtml
+{
+   public static final int MAX_JSON_CHARS = 60_000;
+
+   private static final ObjectMapper PRETTY = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+
+   private static final String WRAP = "overflow-wrap:anywhere;word-break:break-word;";
+
+   private final StringBuilder html = new StringBuilder();
+
+
+
+   /*******************************************************************************
+    ** Severity of a check or a line, rendered as a coloured label.
+    *******************************************************************************/
+   public enum Level
+   {
+      OK("OK", "#16a34a"),
+      INFO("INFO", "#2563eb"),
+      WARN("WARN", "#d97706"),
+      FAIL("FAIL", "#dc2626");
+
+      private final String label;
+      private final String color;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       *******************************************************************************/
+      Level(String label, String color)
+      {
+         this.label = label;
+         this.color = color;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for label
+       *******************************************************************************/
+      public String getLabel()
+      {
+         return (label);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Escape text for HTML element content and attribute values.
+    *******************************************************************************/
+   public static String escape(Object value)
+   {
+      if(value == null)
+      {
+         return ("");
+      }
+      String        text = String.valueOf(value);
+      StringBuilder out  = new StringBuilder(text.length() + 16);
+      for(int i = 0; i < text.length(); i++)
+      {
+         char c = text.charAt(i);
+         switch(c)
+         {
+            case '<' -> out.append("&lt;");
+            case '>' -> out.append("&gt;");
+            case '&' -> out.append("&amp;");
+            case '"' -> out.append("&quot;");
+            case '\'' -> out.append("&#39;");
+            default -> out.append(c);
+         }
+      }
+      return (out.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Escape text but keep OpenSearch highlight tags (em) as mark elements.
+    *******************************************************************************/
+   public static String highlight(String fragment)
+   {
+      return (escape(fragment).replace("&lt;em&gt;", "<mark>").replace("&lt;/em&gt;", "</mark>"));
+   }
+
+
+
+   /*******************************************************************************
+    ** A coloured status label.
+    *******************************************************************************/
+   public static String badge(Level level)
+   {
+      return ("<strong style=\"color:" + level.color + "\">" + level.label + "</strong>");
+   }
+
+
+
+   /*******************************************************************************
+    ** Open the root block.
+    *******************************************************************************/
+   public DiagnosticsHtml open()
+   {
+      html.append("<div style=\"max-width:72rem;").append(WRAP).append("\">");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Close the root block.
+    *******************************************************************************/
+   public DiagnosticsHtml close()
+   {
+      html.append("</div>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A section heading.
+    *******************************************************************************/
+   public DiagnosticsHtml heading(String text)
+   {
+      html.append("<h3>").append(escape(text)).append("</h3>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A sub-heading.
+    *******************************************************************************/
+   public DiagnosticsHtml subheading(String text)
+   {
+      html.append("<h4>").append(escape(text)).append("</h4>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A paragraph of escaped text.
+    *******************************************************************************/
+   public DiagnosticsHtml paragraph(String text)
+   {
+      html.append("<p>").append(escape(text)).append("</p>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A paragraph starting with a status label.
+    *******************************************************************************/
+   public DiagnosticsHtml status(Level level, String text)
+   {
+      html.append("<p>").append(badge(level)).append(" ").append(escape(text)).append("</p>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A list of checks: each line a status label and text.
+    *******************************************************************************/
+   public DiagnosticsHtml checks(List<Check> checks)
+   {
+      if(checks == null || checks.isEmpty())
+      {
+         return (this);
+      }
+      html.append("<ul>");
+      for(Check check : checks)
+      {
+         html.append("<li>").append(badge(check.level())).append(" <strong>").append(escape(check.name())).append("</strong>");
+         if(check.message() != null)
+         {
+            html.append(": ").append(escape(check.message()));
+         }
+         html.append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A list of label: value lines (values escaped). Null values show as
+    ** "none".
+    *******************************************************************************/
+   public DiagnosticsHtml facts(List<String[]> facts)
+   {
+      html.append("<ul>");
+      for(String[] fact : facts)
+      {
+         html.append("<li><strong>").append(escape(fact[0])).append(":</strong> ")
+            .append(fact.length > 1 && fact[1] != null ? escape(fact[1]) : "<em>none</em>").append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A bulleted list of pre-built HTML items (callers escape their parts).
+    *******************************************************************************/
+   public DiagnosticsHtml rawList(List<String> itemsHtml)
+   {
+      if(itemsHtml == null || itemsHtml.isEmpty())
+      {
+         return (this);
+      }
+      html.append("<ul>");
+      for(String item : itemsHtml)
+      {
+         html.append("<li>").append(item).append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A preformatted block that wraps (no horizontal scrolling).
+    *******************************************************************************/
+   public DiagnosticsHtml pre(String text)
+   {
+      html.append("<pre style=\"white-space:pre-wrap;").append(WRAP).append("\">").append(escape(text)).append("</pre>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A collapsed block with a summary line and pretty-printed JSON inside.
+    *******************************************************************************/
+   public DiagnosticsHtml jsonDetails(String summary, String json)
+   {
+      html.append("<details><summary>").append(escape(summary)).append("</summary>");
+      pre(prettyJson(json));
+      html.append("</details>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A collapsed block with a summary line and preformatted text inside.
+    *******************************************************************************/
+   public DiagnosticsHtml textDetails(String summary, String text)
+   {
+      html.append("<details><summary>").append(escape(summary)).append("</summary>");
+      pre(text);
+      html.append("</details>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Append pre-built HTML (callers escape their parts).
+    *******************************************************************************/
+   public DiagnosticsHtml raw(String rawHtml)
+   {
+      html.append(rawHtml);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Pretty-print JSON (or return the text as-is when it is not JSON),
+    ** truncated to MAX_JSON_CHARS.
+    *******************************************************************************/
+   public static String prettyJson(String json)
+   {
+      if(json == null)
+      {
+         return ("");
+      }
+      String pretty;
+      try
+      {
+         pretty = PRETTY.writeValueAsString(PRETTY.readTree(json));
+      }
+      catch(Exception e)
+      {
+         pretty = json;
+      }
+      return (pretty.length() > MAX_JSON_CHARS ? pretty.substring(0, MAX_JSON_CHARS) + "\n... (truncated at " + MAX_JSON_CHARS + " characters)" : pretty);
+   }
+
+
+
+   /*******************************************************************************
+    ** Pretty-print a JSON node.
+    *******************************************************************************/
+   public static String prettyJson(JsonNode node)
+   {
+      return (node == null ? "" : prettyJson(node.toString()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The built HTML.
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return (html.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** One check: name, level and message.
+    *******************************************************************************/
+   public record Check(String name, Level level, String message)
+   {
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsRateLimiter.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.LongSupplier;
+
+
+/*******************************************************************************
+ ** Sliding one-minute window of diagnostic tool runs per key (user and
+ ** tool), per JVM. tryAcquire admits a run when fewer than the limit ran in
+ ** the last 60 seconds.
+ *******************************************************************************/
+public class DiagnosticsRateLimiter
+{
+   private static final long WINDOW_MILLIS = 60_000L;
+
+   private final Map<String, Deque<Long>> runsByKey = new ConcurrentHashMap<>();
+   private final LongSupplier             clock;
+
+
+
+   /*******************************************************************************
+    ** Constructor (system clock)
+    *******************************************************************************/
+   public DiagnosticsRateLimiter()
+   {
+      this(System::currentTimeMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor with a clock (tests)
+    *******************************************************************************/
+   public DiagnosticsRateLimiter(LongSupplier clock)
+   {
+      this.clock = clock;
+   }
+
+
+
+   /*******************************************************************************
+    ** Admit one run for the key when fewer than perMinute ran in the last
+    ** minute; a null or non-positive limit admits everything.
+    *******************************************************************************/
+   public boolean tryAcquire(String key, Integer perMinute)
+   {
+      if(perMinute == null || perMinute <= 0)
+      {
+         return (true);
+      }
+
+      long        now  = clock.getAsLong();
+      Deque<Long> runs = runsByKey.computeIfAbsent(key, k -> new ArrayDeque<>());
+      synchronized(runs)
+      {
+         while(!runs.isEmpty() && now - runs.peekFirst() >= WINDOW_MILLIS)
+         {
+            runs.pollFirst();
+         }
+         if(runs.size() >= perMinute)
+         {
+            return (false);
+         }
+         runs.addLast(now);
+         return (true);
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsSupport.java

@@ -0,0 +1,249 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.Collection;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Shared pieces of the diagnostic tools: tool names, the current user, the
+ ** per-user rate limit, and recording tool runs and search failures as
+ ** quick_search_diagnostic_event rows (the audit trail).
+ *******************************************************************************/
+public final class DiagnosticsSupport
+{
+   private static final QLogger LOG = QLogger.getLogger(DiagnosticsSupport.class);
+
+   public static final String TOOL_PLAYGROUND      = "SEARCH_PLAYGROUND";
+   public static final String TOOL_INSPECT_RECORD  = "INSPECT_RECORD";
+   public static final String TOOL_REINDEX_RECORDS = "REINDEX_RECORDS";
+   public static final String TOOL_VALIDATE_INDEX  = "VALIDATE_INDEX";
+   public static final String TOOL_ANALYZE_TEXT    = "ANALYZE_TEXT";
+
+
+
+   /*******************************************************************************
+    ** Latest saved validation, shared by the home widget and host status API.
+    ** This does not run validation or scan source tables.
+    *******************************************************************************/
+   public static QRecord latestValidation(QuickSearchQBitConfig config) throws QException
+   {
+      String tableName = config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         return (null);
+      }
+      var input = new QueryInput(tableName)
+         .withFilter(new QQueryFilter()
+            .withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
+      var rows = new QueryAction().execute(input).getRecords();
+      return (rows == null || rows.isEmpty() ? null : rows.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private DiagnosticsSupport()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The session's user id reference (else full name), or null.
+    *******************************************************************************/
+   public static String currentUserName()
+   {
+      QSession session = QContext.getQSession();
+      if(session == null || session.getUser() == null)
+      {
+         return (null);
+      }
+      return (session.getUser().getIdReference() != null ? session.getUser().getIdReference() : session.getUser().getFullName());
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime of the Quick Search QBit, or a user-facing error saying the
+    ** tool is unavailable.
+    *******************************************************************************/
+   public static QuickSearchRuntime requireRuntime() throws QUserFacingException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         throw (new QUserFacingException("Quick Search is not configured in this application, so its diagnostic tools are unavailable."));
+      }
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Admit one run of a tool for the current user, or refuse it (recording
+    ** the refusal) when the user ran diagnosticRunsPerMinute tools in the last
+    ** minute.
+    *******************************************************************************/
+   public static void checkRateLimit(QuickSearchRuntime runtime, String tool) throws QUserFacingException
+   {
+      String user = currentUserName();
+      if(runtime.getDiagnosticsRateLimiter().tryAcquire((user == null ? "anonymous" : user) + "|" + tool, runtime.getConfig().getDiagnosticRunsPerMinute()))
+      {
+         return;
+      }
+
+      runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+         .withEventType(QuickSearchDiagnosticEvent.TYPE_RATE_LIMITED)
+         .withSource(tool)
+         .withOutcome(QuickSearchDiagnosticEvent.OUTCOME_REFUSED));
+      throw (new QUserFacingException("Too many Quick Search diagnostic runs in the last minute (limit " + runtime.getConfig().getDiagnosticRunsPerMinute()
+         + " per tool). Wait a minute and try again."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a tool run (audit): who ran which tool on which tables with which
+    ** term, how long it took, and its outcome.
+    *******************************************************************************/
+   public static void recordToolRun(QuickSearchRuntime runtime, String tool, String eventType, Collection<String> tableNames, String term, long startMillis, Throwable error, String detail)
+   {
+      try
+      {
+         runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+            .withEventType(eventType)
+            .withSource(tool)
+            .withOutcome(error == null ? QuickSearchDiagnosticEvent.OUTCOME_OK : QuickSearchDiagnosticEvent.OUTCOME_ERROR)
+            .withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
+            .withSearchTerm(term)
+            .withDurationMillis((int) Math.min(Integer.MAX_VALUE, System.currentTimeMillis() - startMillis))
+            .withErrorClass(error == null ? null : error.getClass().getName())
+            .withErrorMessage(error == null ? null : error.getMessage())
+            .withDetail(detail));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not queue a diagnostic event", e);
+      }
+
+      LOG.info("Quick Search diagnostic tool run", logPair("tool", tool), logPair("user", currentUserName()), logPair("tables", tableNames),
+         logPair("millis", System.currentTimeMillis() - startMillis), logPair("error", error == null ? null : error.getMessage()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a failed search (OpenSearch error or timeout). userTerm is the
+    ** user's term: never stored, and redacted from the message and detail.
+    *******************************************************************************/
+   public static void recordSearchFailure(QuickSearchRuntime runtime, String source, Collection<String> tableNames, String userTerm, long durationMillis, Throwable error, String detail)
+   {
+      if(runtime == null)
+      {
+         return;
+      }
+      try
+      {
+         Throwable cause = rootCause(error);
+         runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+            .withEventType(QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED)
+            .withSource(source)
+            .withOutcome(QuickSearchDiagnosticEvent.OUTCOME_ERROR)
+            .withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
+            .withDurationMillis((int) Math.min(Integer.MAX_VALUE, durationMillis))
+            .withErrorClass(cause == null ? null : cause.getClass().getName())
+            .withErrorMessage(redactTerm(error == null ? null : error.getMessage(), userTerm))
+            .withDetail(redactTerm(detail, userTerm)));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not queue a search failure event", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The text with the user's search term, and each of its words of three or
+    ** more characters, replaced by a marker (case-insensitive): error messages
+    ** (ours and OpenSearch's) can echo the term, which may hold personal data.
+    *******************************************************************************/
+   public static String redactTerm(String text, String userTerm)
+   {
+      if(text == null || userTerm == null || userTerm.isBlank())
+      {
+         return (text);
+      }
+      String redacted = text.replaceAll("(?i)" + java.util.regex.Pattern.quote(userTerm.trim()), "[search term]");
+      for(String word : userTerm.trim().split("\\s+"))
+      {
+         if(word.length() >= 3)
+         {
+            redacted = redacted.replaceAll("(?i)" + java.util.regex.Pattern.quote(word), "[search term]");
+         }
+      }
+      return (redacted);
+   }
+
+
+
+   /*******************************************************************************
+    ** The innermost cause of a throwable (the one that names the real error
+    ** class: timeout, connection refused, OpenSearch error).
+    *******************************************************************************/
+   public static Throwable rootCause(Throwable error)
+   {
+      Throwable cause = error;
+      int       depth = 0;
+      while(cause != null && cause.getCause() != null && cause.getCause() != cause && depth++ < 20)
+      {
+         cause = cause.getCause();
+      }
+      return (cause);
+   }
+
+
+
+   /*******************************************************************************
+    ** A user-facing error for a tool whose OpenSearch call failed: the tool
+    ** reports the failure honestly instead of showing partial data.
+    *******************************************************************************/
+   public static QUserFacingException unavailable(String what, QException e)
+   {
+      return (new QUserFacingException(what + " is unavailable: " + e.getMessage(), e));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/IndexValidator.java

@@ -0,0 +1,792 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.RawResponse;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Validate the Quick Search index against the database and its expected
+ ** shape, and report every finding with a level (OK, INFO, WARN, FAIL):
+ ** - cluster: reachable, cluster and index health, the alias pointing at one
+ **   physical index, leftover physical indexes from interrupted rebuilds;
+ ** - mapping and settings: mapping version, field types and analyzers of the
+ **   fixed fields, the fieldValues dynamic template and every fieldValues.*
+ **   field mapped as text, the two analyzers and the edge n-gram filter;
+ ** - per table: database rows vs indexed documents, a sample of records
+ **   (the lowest and highest primary keys) checked present and current in
+ **   the index, a random sample of documents checked to still have a record,
+ **   the quick_search_index row's status and last run, failed events waiting.
+ ** Database reads run as the system user, so counts compare like with like
+ ** (the index holds every row whatever the record locks). The report holds
+ ** counts, ids and field names, never record values. Read-only, except that
+ ** persist() saves the report as a quick_search_validation_run row.
+ *******************************************************************************/
+public class IndexValidator
+{
+   private static final QLogger LOG = QLogger.getLogger(IndexValidator.class);
+
+   private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
+
+   public static final String SCOPE_CLUSTER = "cluster";
+   public static final String SCOPE_MAPPING = "mapping";
+
+   private static final int MAX_LISTED_IDS = 25;
+
+
+
+   /*******************************************************************************
+    ** One finding.
+    *******************************************************************************/
+   public record Finding(String scope, String check, Level level, String message)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Per-table numbers.
+    *******************************************************************************/
+   public record TableStats(String tableName, Integer databaseCount, Long indexCount, Integer sampled, List<String> missingIds, Map<String, List<String>> staleFields,
+                            Integer documentsSampled, List<String> extraIds)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A validation report.
+    *******************************************************************************/
+   public record Report(String status, Instant startTime, Instant endTime, Integer sampleSize, List<String> tableNames, List<Finding> findings, List<TableStats> tableStats)
+   {
+      /*******************************************************************************
+       ** Count of findings at a level.
+       *******************************************************************************/
+      public int count(Level level)
+      {
+         return ((int) findings.stream().filter(f -> f.level() == level).count());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate the named tables (empty for every indexed table) with the given
+    ** sample size per table.
+    *******************************************************************************/
+   public Report validate(QuickSearchRuntime runtime, List<String> requestedTables, int sampleSize) throws QException
+   {
+      Instant          start     = Instant.now();
+      List<Finding>    findings  = new ArrayList<>();
+      List<TableStats> stats     = new ArrayList<>();
+      List<String>     tables    = resolveTables(runtime, requestedTables, findings);
+      int              boundedN  = Math.max(1, Math.min(sampleSize, QuickSearchQBitConfig.MAX_VALIDATION_SAMPLE_SIZE));
+      QInstance        qInstance = QContext.getQInstance();
+
+      QContext.withTemporaryContext(new CapturedContext(qInstance, new QSystemUserSession()), () ->
+      {
+         QuickSearchDiagnosticsOperations ops;
+         try
+         {
+            ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+         }
+         catch(QException e)
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Client", Level.FAIL, "The OpenSearch client could not be built: " + e.getMessage()));
+            return;
+         }
+
+         if(!checkCluster(runtime, ops, findings))
+         {
+            return;
+         }
+         checkMapping(ops, findings);
+         checkSettings(ops, findings);
+
+         for(String tableName : tables)
+         {
+            stats.add(checkTable(runtime, ops, runtime.getTableConfig(tableName), boundedN, findings));
+         }
+      });
+
+      String status = findings.stream().anyMatch(f -> f.level() == Level.FAIL) ? QuickSearchValidationRun.STATUS_FAIL
+         : findings.stream().anyMatch(f -> f.level() == Level.WARN) ? QuickSearchValidationRun.STATUS_WARN : QuickSearchValidationRun.STATUS_PASS;
+      return (new Report(status, start, Instant.now(), boundedN, tables, findings, stats));
+   }
+
+
+
+   /*******************************************************************************
+    ** The requested indexed tables (unknown ones reported), else every indexed
+    ** table.
+    *******************************************************************************/
+   private static List<String> resolveTables(QuickSearchRuntime runtime, List<String> requested, List<Finding> findings)
+   {
+      List<String> tables = new ArrayList<>();
+      if(CollectionUtils.nullSafeIsEmpty(requested))
+      {
+         runtime.getDiscoveredTables().forEach(t -> tables.add(t.getTableName()));
+         return (tables);
+      }
+      for(String tableName : requested)
+      {
+         if(runtime.getTableConfig(tableName) == null)
+         {
+            findings.add(new Finding(tableName, "Configured", Level.WARN, "Quick Search does not index this table; skipped."));
+         }
+         else if(!tables.contains(tableName))
+         {
+            tables.add(tableName);
+         }
+      }
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health, alias and physical indexes. Returns false when the index
+    ** cannot be reached (the other checks are then skipped).
+    *******************************************************************************/
+   boolean checkCluster(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse health;
+      try
+      {
+         health = ops.clusterHealth();
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "OpenSearch did not answer: " + e.getMessage()));
+         return (false);
+      }
+
+      if(health.status() == 404)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Index", Level.FAIL, "The index alias [" + ops.getIndexName() + "] does not exist. Run the full reindex to create it."));
+         return (false);
+      }
+      if(!health.isOk())
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "Cluster health returned status " + health.status() + ": " + QuickSearchDiagnosticsOperations.errorReason(health.body())));
+         return (false);
+      }
+
+      findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.OK, "OpenSearch answered (" + health.body().path("cluster_name").asText("cluster") + ", "
+         + health.body().path("number_of_nodes").asInt() + " node(s))."));
+      findings.add(healthFinding("Cluster health", health.body().path("status").asText("unknown")));
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = health.body().path("indices").fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index = indexes.next();
+         findings.add(healthFinding("Index health " + index.getKey(), index.getValue().path("status").asText("unknown")));
+      }
+
+      try
+      {
+         Set<String> physical = runtime.getClient().getPhysicalIndexes();
+         if(physical.isEmpty())
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.WARN, "[" + ops.getIndexName() + "] is a concrete index, not an alias (created before 1.0). The next full reindex replaces it with an alias."));
+         }
+         else if(physical.size() > 1)
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "The alias points at " + physical.size() + " physical indexes " + physical + "; searches and counts see documents twice. Run the full reindex."));
+         }
+         else
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.OK, "The alias points at " + physical.iterator().next() + "."));
+         }
+
+         RawResponse cat = ops.physicalIndexes();
+         if(cat.isOk() && cat.body() != null)
+         {
+            for(JsonNode row : cat.body())
+            {
+               String name = row.path("index").asText();
+               if(!physical.contains(name))
+               {
+                  findings.add(new Finding(SCOPE_CLUSTER, "Leftover index", Level.WARN, "Physical index " + name + " (" + row.path("docs.count").asText("?") + " docs, "
+                     + row.path("store.size").asText("?") + ") is not behind the alias: left by an interrupted rebuild. Delete it to free space."));
+               }
+            }
+         }
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "Could not read the alias: " + e.getMessage()));
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static Finding healthFinding(String check, String status)
+   {
+      return (switch(status)
+      {
+         case "green" -> new Finding(SCOPE_CLUSTER, check, Level.OK, "green.");
+         case "yellow" -> new Finding(SCOPE_CLUSTER, check, Level.WARN, "yellow: replica shards are unassigned (expected on a single-node cluster when the index asks for replicas).");
+         default -> new Finding(SCOPE_CLUSTER, check, Level.FAIL, status + ": primary shards are unassigned; searches and writes may fail.");
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Shared cluster health interpretation for validation and the host status API.
+    *******************************************************************************/
+   public static Level healthLevel(String status)
+   {
+      return (healthFinding("Health", status).level());
+   }
+
+
+
+   /*******************************************************************************
+    ** The live mapping against the mapping this version creates.
+    *******************************************************************************/
+   void checkMapping(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse response;
+      try
+      {
+         response = ops.mapping();
+         QuickSearchDiagnosticsOperations.requireOk(response, "Mapping read failed");
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_MAPPING, "Mapping", Level.FAIL, e.getMessage()));
+         return;
+      }
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index    = indexes.next();
+         JsonNode                    mappings = index.getValue().path("mappings");
+         String                      prefix   = index.getKey() + ": ";
+
+         int version = mappings.path("_meta").path(QuickSearchOpenSearchClient.META_MAPPING_VERSION).asInt(1);
+         findings.add(version >= QuickSearchOpenSearchClient.MAPPING_VERSION
+            ? new Finding(SCOPE_MAPPING, "Mapping version", Level.OK, prefix + "version " + version + ".")
+            : new Finding(SCOPE_MAPPING, "Mapping version", Level.FAIL, prefix + "version " + version + ", this release writes " + QuickSearchOpenSearchClient.MAPPING_VERSION + ". Run the full reindex."));
+
+         JsonNode properties = mappings.path("properties");
+         List<String> problems = new ArrayList<>();
+         expectType(properties, "sourceTable", "keyword", problems);
+         expectType(properties, "recordId", "keyword", problems);
+         expectType(properties, "indexedAt", "date", problems);
+         expectText(properties.path("searchableText"), "searchableText", problems);
+         expectText(properties.path("recordLabel"), "recordLabel", problems);
+
+         boolean template = false;
+         for(JsonNode entry : mappings.path("dynamic_templates"))
+         {
+            template |= entry.has("fieldValuesAsText");
+         }
+         if(!template)
+         {
+            problems.add("the fieldValuesAsText dynamic template is missing, so new fields may be mapped as numbers or dates");
+         }
+         if(mappings.path("date_detection").asBoolean(true) || mappings.path("numeric_detection").asBoolean(false))
+         {
+            problems.add("date or numeric detection is on");
+         }
+
+         Iterator<Map.Entry<String, JsonNode>> fields = properties.path("fieldValues").path("properties").fields();
+         int                                   count  = 0;
+         while(fields.hasNext())
+         {
+            Map.Entry<String, JsonNode> field = fields.next();
+            expectText(field.getValue(), "fieldValues." + field.getKey(), problems);
+            count++;
+         }
+
+         findings.add(problems.isEmpty()
+            ? new Finding(SCOPE_MAPPING, "Field mappings", Level.OK, prefix + "fixed fields and " + count + " fieldValues field(s) mapped as expected.")
+            : new Finding(SCOPE_MAPPING, "Field mappings", Level.FAIL, prefix + String.join("; ", problems) + ". Run the full reindex to rebuild the mapping."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void expectType(JsonNode properties, String field, String type, List<String> problems)
+   {
+      String actual = properties.path(field).path("type").asText(null);
+      if(!type.equals(actual))
+      {
+         problems.add(field + " is " + (actual == null ? "not mapped" : "mapped as " + actual) + ", expected " + type);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void expectText(JsonNode mapping, String field, List<String> problems)
+   {
+      String type     = mapping.path("type").asText(null);
+      String analyzer = mapping.path("analyzer").asText(null);
+      String search   = mapping.path("search_analyzer").asText(null);
+      if(!"text".equals(type))
+      {
+         problems.add(field + " is " + (type == null ? "not mapped" : "mapped as " + type) + ", expected text");
+      }
+      else if(!QuickSearchOpenSearchClient.INDEX_ANALYZER.equals(analyzer) || !QuickSearchOpenSearchClient.SEARCH_ANALYZER.equals(search))
+      {
+         problems.add(field + " uses analyzer " + analyzer + " / search analyzer " + search + ", expected " + QuickSearchOpenSearchClient.INDEX_ANALYZER + " / " + QuickSearchOpenSearchClient.SEARCH_ANALYZER);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The live analysis settings against the ones this version creates.
+    *******************************************************************************/
+   void checkSettings(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse response;
+      try
+      {
+         response = ops.settings();
+         QuickSearchDiagnosticsOperations.requireOk(response, "Settings read failed");
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, e.getMessage()));
+         return;
+      }
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index    = indexes.next();
+         JsonNode                    settings = index.getValue().path("settings").path("index");
+         JsonNode                    analysis = settings.path("analysis");
+         List<String>                problems = new ArrayList<>();
+
+         JsonNode indexAnalyzer  = analysis.path("analyzer").path(QuickSearchOpenSearchClient.INDEX_ANALYZER);
+         JsonNode searchAnalyzer = analysis.path("analyzer").path(QuickSearchOpenSearchClient.SEARCH_ANALYZER);
+         JsonNode ngram          = analysis.path("filter").path("edge_ngram_filter");
+
+         if(!"standard".equals(indexAnalyzer.path("tokenizer").asText()) || !List.of("lowercase", "edge_ngram_filter").equals(textList(indexAnalyzer.path("filter"))))
+         {
+            problems.add(QuickSearchOpenSearchClient.INDEX_ANALYZER + " is " + (indexAnalyzer.isMissingNode() ? "missing" : indexAnalyzer.toString()));
+         }
+         if(!"standard".equals(searchAnalyzer.path("tokenizer").asText()) || !List.of("lowercase").equals(textList(searchAnalyzer.path("filter"))))
+         {
+            problems.add(QuickSearchOpenSearchClient.SEARCH_ANALYZER + " is " + (searchAnalyzer.isMissingNode() ? "missing" : searchAnalyzer.toString()));
+         }
+         if(!"edge_ngram".equals(ngram.path("type").asText()) || ngram.path("min_gram").asInt() != 2 || ngram.path("max_gram").asInt() != 20)
+         {
+            problems.add("edge_ngram_filter is " + (ngram.isMissingNode() ? "missing" : ngram.toString()));
+         }
+
+         findings.add(problems.isEmpty()
+            ? new Finding(SCOPE_MAPPING, "Analysis settings", Level.OK, index.getKey() + ": analyzers and edge n-grams (2 to 20) as expected; "
+            + settings.path("number_of_shards").asText("?") + " shard(s), " + settings.path("number_of_replicas").asText("?") + " replica(s).")
+            : new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, index.getKey() + ": " + String.join("; ", problems) + ". Run the full reindex."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static List<String> textList(JsonNode array)
+   {
+      List<String> values = new ArrayList<>();
+      array.forEach(v -> values.add(v.asText()));
+      return (values);
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts, sampled freshness and orphans of one table, plus its index row
+    ** and failed events.
+    *******************************************************************************/
+   TableStats checkTable(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, int sampleSize, List<Finding> findings)
+   {
+      String tableName = tableConfig.getTableName();
+      Integer databaseCount = null;
+      Long    indexCount    = null;
+      try
+      {
+         databaseCount = DiagnosticRecordReads.count(tableName);
+         indexCount    = runtime.getClient().countDocumentsForTable(tableName);
+         long difference = indexCount - databaseCount;
+         long tolerance  = Math.max(5, databaseCount / 100);
+         Level level = difference == 0 ? Level.OK : (Math.abs(difference) <= tolerance ? Level.WARN : Level.FAIL);
+         findings.add(new Finding(tableName, "Counts", level, "database " + databaseCount + ", index " + indexCount
+            + (difference == 0 ? "." : " (" + (difference > 0 ? difference + " more in the index" : (-difference) + " missing from the index") + "). Reconcile fixes counts.")));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Counts", Level.FAIL, e.getMessage()));
+      }
+
+      List<String>              missing     = new ArrayList<>();
+      Map<String, List<String>> staleFields = new LinkedHashMap<>();
+      int                       sampled     = 0;
+      try
+      {
+         Map<String, QRecord> sample = new LinkedHashMap<>();
+         for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, true, (sampleSize + 1) / 2))
+         {
+            sample.put(record.getValueString(tableConfig.getPrimaryKeyField()), record);
+         }
+         for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, false, sampleSize / 2))
+         {
+            sample.putIfAbsent(record.getValueString(tableConfig.getPrimaryKeyField()), record);
+         }
+         sampled = sample.size();
+
+         Map<String, JsonNode> indexed = ops.getDocuments(tableName, sample.keySet());
+         for(Map.Entry<String, QRecord> entry : sample.entrySet())
+         {
+            JsonNode source = indexed.get(entry.getKey());
+            if(source == null)
+            {
+               missing.add(entry.getKey());
+               continue;
+            }
+            List<String> differing = differingFields(tableConfig, IndexingUtils.buildDocument(entry.getValue(), tableConfig), source);
+            if(!differing.isEmpty())
+            {
+               staleFields.put(entry.getKey(), differing);
+            }
+         }
+
+         findings.add(missing.isEmpty() && staleFields.isEmpty()
+            ? new Finding(tableName, "Sampled records", Level.OK, sampled + " record(s) checked: all indexed and current.")
+            : new Finding(tableName, "Sampled records", Level.FAIL, sampled + " record(s) checked: " + missing.size() + " missing" + listIds(missing) + ", "
+            + staleFields.size() + " stale" + listIds(new ArrayList<>(staleFields.keySet())) + ". Reindex those records, or run reconcile for the table."));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Sampled records", Level.FAIL, e.getMessage()));
+      }
+
+      List<String> extra            = new ArrayList<>();
+      int          documentsSampled = 0;
+      try
+      {
+         List<String> documentIds = ops.sampleRecordIds(tableName, sampleSize);
+         documentsSampled = documentIds.size();
+         Set<String> existing = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, documentIds));
+         for(String recordId : documentIds)
+         {
+            if(!existing.contains(recordId))
+            {
+               extra.add(recordId);
+            }
+         }
+         findings.add(extra.isEmpty()
+            ? new Finding(tableName, "Sampled documents", Level.OK, documentsSampled + " random document(s) checked: every one has its record.")
+            : new Finding(tableName, "Sampled documents", Level.FAIL, documentsSampled + " random document(s) checked: " + extra.size() + " have no record (orphans)"
+            + listIds(extra) + ". Reconcile removes them."));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Sampled documents", Level.FAIL, e.getMessage()));
+      }
+
+      checkIndexRow(runtime, tableName, findings);
+      checkFailedEvents(runtime, tableName, findings);
+
+      return (new TableStats(tableName, databaseCount, indexCount, sampled, missing, staleFields, documentsSampled, extra));
+   }
+
+
+
+   /*******************************************************************************
+    ** Names of the values that differ between an expected document and an
+    ** indexed source (recordLabel, searchableText, fieldValues.x).
+    *******************************************************************************/
+   static List<String> differingFields(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, JsonNode source)
+   {
+      return (RecordInspector.diff(tableConfig, expected, new QuickSearchDiagnosticsOperations.DocumentView(true, null, null, source)).stream()
+         .filter(d -> !d.isSame())
+         .map(RecordInspector.FieldDiff::name)
+         .toList());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void checkIndexRow(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
+   {
+      try
+      {
+         String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+         if(QContext.getQInstance().getTable(indexTable) == null)
+         {
+            return;
+         }
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(indexTable);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "No Quick Search Index row yet: no basepull, reconcile or reindex has run for this table."));
+            return;
+         }
+
+         QRecord row     = rows.get(0);
+         String  status  = row.getValueString("status");
+         String  lastRun = row.getValueString("lastRunStatus");
+         if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.INFO, "Disabled: Quick Search neither indexes nor searches this table, so differences are expected."));
+         }
+         if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "Status NEEDS_REINDEX: the configured fields changed since the last reindex. Rebuild the table."));
+         }
+         else if(AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "Status REBUILDING: a full reindex is running. If it was killed, basepull returns the table to ACTIVE once its run records show no sign of life for fullReindexStaleMinutes."));
+         }
+         if(AbstractIndexingStep.RUN_FAILED.equals(lastRun))
+         {
+            findings.add(new Finding(tableName, "Last run", Level.WARN, "The last indexing run failed: " + row.getValueString("lastErrorMessage")));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read the Quick Search Index row during validation", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void checkFailedEvents(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
+   {
+      try
+      {
+         String failedTable = runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+         if(QContext.getQInstance().getTable(failedTable) == null)
+         {
+            return;
+         }
+         CountInput countInput = new CountInput();
+         countInput.setTableName(failedTable);
+         countInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+            new QFilterCriteria("status", QCriteriaOperator.IN, List.of(QuickSearchFailedEvent.STATUS_PENDING, QuickSearchFailedEvent.STATUS_EXHAUSTED))));
+         Integer count = new CountAction().execute(countInput).getCount();
+         if(count != null && count > 0)
+         {
+            findings.add(new Finding(tableName, "Failed events", Level.WARN, count + " real-time index event(s) failed and wait for replay (or are exhausted). See Quick Search Failed Event."));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not count failed events during validation", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Save a report as a quick_search_validation_run row (as the system user);
+    ** returns its id, or null when the table is not configured or the insert
+    ** fails (logged).
+    *******************************************************************************/
+   public Integer persist(QuickSearchRuntime runtime, Report report, String triggerType, String userName)
+   {
+      String tableName = runtime.getConfig().applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
+      if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
+      {
+         return (null);
+      }
+
+      QuickSearchValidationRun run = new QuickSearchValidationRun()
+         .withStatus(report.status())
+         .withTriggerType(triggerType)
+         .withUserName(userName)
+         .withTableNames(DiagnosticEventRecorder.truncate(String.join(",", report.tableNames()), 1000))
+         .withSampleSize(report.sampleSize())
+         .withTablesChecked(report.tableStats().size())
+         .withFailCount(report.count(Level.FAIL))
+         .withWarnCount(report.count(Level.WARN))
+         .withSummary(DiagnosticEventRecorder.truncate(summary(report), 250))
+         .withReportHtml(render(report))
+         .withReportJson(toJson(report))
+         .withStartTime(report.startTime())
+         .withEndTime(report.endTime())
+         .withCreateDate(Instant.now());
+
+      Integer[] id = new Integer[1];
+      try
+      {
+         QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
+         {
+            InsertOutput output = new InsertAction().execute(new InsertInput(tableName).withRecordEntity(run));
+            id[0] = output.getRecords().get(0).getValueInteger("id");
+         });
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not save the Quick Search validation report", e);
+      }
+      return (id[0]);
+   }
+
+
+
+   /*******************************************************************************
+    ** One line: status and counts.
+    *******************************************************************************/
+   public static String summary(Report report)
+   {
+      return (report.status() + ": " + report.tableStats().size() + " table(s), " + report.count(Level.FAIL) + " failed check(s), " + report.count(Level.WARN) + " warning(s).");
+   }
+
+
+
+   /*******************************************************************************
+    ** The report as JSON.
+    *******************************************************************************/
+   public static String toJson(Report report)
+   {
+      try
+      {
+         return (MAPPER.writeValueAsString(report));
+      }
+      catch(Exception e)
+      {
+         return ("{\"error\":\"report could not be serialized\"}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The report as HTML: summary, then the findings grouped by scope.
+    *******************************************************************************/
+   public static String render(Report report)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      Level overall = switch(report.status())
+      {
+         case QuickSearchValidationRun.STATUS_PASS -> Level.OK;
+         case QuickSearchValidationRun.STATUS_WARN -> Level.WARN;
+         default -> Level.FAIL;
+      };
+      html.heading("Index validation");
+      html.status(overall, summary(report));
+      html.paragraph("Ran " + report.startTime() + " to " + report.endTime() + ", " + report.sampleSize() + " sampled record(s) and document(s) per table. Database reads ran as the system user.");
+
+      Map<String, List<DiagnosticsHtml.Check>> byScope = new LinkedHashMap<>();
+      for(Finding finding : report.findings())
+      {
+         byScope.computeIfAbsent(finding.scope(), k -> new ArrayList<>()).add(new DiagnosticsHtml.Check(finding.check(), finding.level(), finding.message()));
+      }
+      for(Map.Entry<String, List<DiagnosticsHtml.Check>> scope : byScope.entrySet())
+      {
+         String heading = switch(scope.getKey())
+         {
+            case SCOPE_CLUSTER -> "Cluster and index";
+            case SCOPE_MAPPING -> "Mapping and analyzers";
+            default -> "Table " + scope.getKey();
+         };
+         html.subheading(heading);
+         html.checks(scope.getValue());
+      }
+
+      for(TableStats stats : report.tableStats())
+      {
+         if(!stats.staleFields().isEmpty())
+         {
+            List<String> items = new ArrayList<>();
+            stats.staleFields().entrySet().stream().limit(MAX_LISTED_IDS).forEach(e ->
+               items.add("#" + DiagnosticsHtml.escape(e.getKey()) + ": " + DiagnosticsHtml.escape(String.join(", ", e.getValue()))));
+            html.subheading("Stale fields in " + stats.tableName());
+            html.rawList(items);
+         }
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String listIds(List<String> ids)
+   {
+      if(ids.isEmpty())
+      {
+         return ("");
+      }
+      return (" (" + String.join(", ", ids.subList(0, Math.min(MAX_LISTED_IDS, ids.size()))) + (ids.size() > MAX_LISTED_IDS ? ", ..." : "") + ")");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/QuickSearchDeveloperPanelProvider.java

@@ -0,0 +1,535 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanel;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelAction;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelBlock;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelContext;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelKeyValue;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelLink;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelProviderInterface;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelTone;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The "Quick Search" panel of a record's developer view (QQQ's
+ ** DeveloperPanelProviderInterface, /app/{table}/{id}/dev).
+ **
+ ** Shown for tables this QBit indexes, to users who may read the Quick Search
+ ** Index table. It shows the table's index row (enabled, status, counts, last
+ ** basepull and full reindex, searchable fields), the latest index run, the
+ ** record's indexed document compared with the database (Inspect Search
+ ** Document's comparison; needs the debug permission), the record's failed
+ ** index events, and the Inspect / Reindex record actions and admin links the
+ ** user may use. Every read checks the user's permissions; an unreachable
+ ** OpenSearch shows a warning instead of failing the panel.
+ *******************************************************************************/
+public class QuickSearchDeveloperPanelProvider implements DeveloperPanelProviderInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchDeveloperPanelProvider.class);
+
+   public static final String ID    = "quickSearch";
+   public static final String TITLE = "Quick Search";
+
+   public static final String STATUS_INDEXED     = "Indexed";
+   public static final String STATUS_STALE       = "Stale";
+   public static final String STATUS_NOT_INDEXED = "Not indexed";
+   public static final String STATUS_DISABLED    = "Disabled";
+   public static final String STATUS_ENABLED     = "Enabled";
+   public static final String STATUS_UNAVAILABLE = "OpenSearch unavailable";
+
+   static final int MAX_FAILED_EVENTS = 20;
+   static final int MAX_VALUE_LENGTH  = 300;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getId()
+   {
+      return (ID);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getTitle()
+   {
+      return (TITLE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Integer getOrder()
+   {
+      return (400);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getIcon()
+   {
+      return ("search");
+   }
+
+
+
+   /*******************************************************************************
+    ** Available for a table this QBit indexes, when the user may read the Quick
+    ** Search Index table (the panel shows admin-level indexing state).
+    *******************************************************************************/
+   @Override
+   public boolean isAvailable(DeveloperPanelContext context)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || context == null || runtime.getTableConfig(context.getTableName()) == null)
+      {
+         return (false);
+      }
+      return (mayRead(runtime.getConfig().getQuickSearchIndexTableName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the panel.
+    *******************************************************************************/
+   @Override
+   public DeveloperPanel build(DeveloperPanelContext context) throws QException
+   {
+      QuickSearchRuntime          runtime     = QuickSearchRuntime.getOrThrow();
+      QuickSearchQBitConfig       config      = runtime.getConfig();
+      String                      tableName   = context.getTableName();
+      QuickSearchableTableConfig  tableConfig = runtime.getTableConfig(tableName);
+      Serializable                primaryKey  = context.getPrimaryKey();
+      String                      recordId    = primaryKey == null ? null : String.valueOf(primaryKey);
+      String                      indexTable  = config.getQuickSearchIndexTableName();
+
+      DeveloperPanel panel = new DeveloperPanel()
+         .withDescription("How Quick Search (OpenSearch) indexes this table, and this record's search document.");
+
+      //////////////////////////////////////////////////////////////
+      // the table's index row (readable: checked by isAvailable) //
+      //////////////////////////////////////////////////////////////
+      QRecord indexRow = null;
+      if(mayRead(indexTable))
+      {
+         indexRow = queryFirst(indexTable, new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+      }
+      boolean enabled = indexRow == null || !Boolean.FALSE.equals(indexRow.getValueBoolean("enabled"));
+      panel.withBlock(indexBlock(config, tableConfig, indexTable, indexRow, enabled));
+
+      //////////////////////
+      // latest index run //
+      //////////////////////
+      String runTable = config.getQuickSearchIndexRunTableName();
+      if(indexRow != null && mayRead(runTable))
+      {
+         QRecord run = queryFirst(runTable, new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexRow.getValue("id")))
+            .withOrderBy(new QFilterOrderBy("id", false)));
+         panel.withBlock(run == null
+            ? DeveloperPanelBlock.text("No index runs are recorded for this table yet.", DeveloperPanelTone.INFO).withLabel("Latest index run")
+            : runBlock(runTable, run));
+      }
+
+      /////////////////////////////////////////
+      // this record's document vs. database //
+      /////////////////////////////////////////
+      String verdict = null;
+      boolean openSearchFailed = false;
+      if(recordId != null)
+      {
+         if(!Boolean.TRUE.equals(config.getEnableDiagnosticTools()))
+         {
+            panel.withBlock(DeveloperPanelBlock.text("Quick Search diagnostic tools are disabled (enableDiagnosticTools), so this record's search document is not compared.", DeveloperPanelTone.INFO)
+               .withLabel("Search document"));
+         }
+         else if(!mayInspect(config, tableName))
+         {
+            panel.withBlock(DeveloperPanelBlock.text("You do not have the Quick Search debug permission, so this record's search document is not shown.", DeveloperPanelTone.INFO)
+               .withLabel("Search document"));
+         }
+         else
+         {
+            try
+            {
+               RecordInspector.Inspection inspection = inspect(runtime, tableName, recordId, new QueryInput(tableName));
+               verdict = inspection.verdict();
+               addInspectionBlocks(panel, inspection);
+            }
+            catch(Exception e)
+            {
+               openSearchFailed = true;
+               LOG.info("Quick Search developer panel could not read the search document", logPair("tableName", tableName), logPair("recordId", recordId), logPair("error", e.getMessage()));
+               panel.withBlock(DeveloperPanelBlock.text("OpenSearch could not be reached, so this record's search document cannot be shown: "
+                  + DiagnosticsSupport.rootCause(e).getMessage(), DeveloperPanelTone.WARNING).withLabel("Search document"));
+            }
+         }
+      }
+
+      /////////////////////////////////
+      // failed index events, if any //
+      /////////////////////////////////
+      String failedTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
+      if(recordId != null && mayRead(failedTable))
+      {
+         DeveloperPanelBlock failedBlock = failedEventsBlock(failedTable, tableName, recordId);
+         if(failedBlock != null)
+         {
+            panel.withBlock(failedBlock);
+         }
+      }
+
+      //////////////////////////////
+      // actions the user may run //
+      //////////////////////////////
+      if(recordId != null)
+      {
+         DeveloperPanelBlock actions = DeveloperPanelBlock.actions("Actions");
+         addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName), tableName, "Inspect search document",
+            "Compare the indexed document with the database, and see why the record does or does not match a term.");
+         addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config, tableName), tableName, "Reindex this record",
+            "Rebuild this record's search document from the database now.");
+         if(!actions.getActions().isEmpty())
+         {
+            panel.withBlock(actions);
+         }
+      }
+
+      ///////////
+      // links //
+      ///////////
+      DeveloperPanelBlock links   = DeveloperPanelBlock.links("Links");
+      String              appName = config.applyPrefix(QuickSearchAdminAppMetaDataProducer.APP_NAME);
+      if(QContext.getQInstance().getApp(appName) != null && PermissionsHelper.hasAppPermission(new QueryInput(indexTable), appName))
+      {
+         links.withLink(new DeveloperPanelLink().withLabel("Quick Search Admin").withHref("/app/" + appName).withDescription("Index status, runs, failed events and diagnostic tools."));
+      }
+      if(indexRow != null)
+      {
+         links.withLink(new DeveloperPanelLink().withLabel("Quick Search Index row for " + tableName).withHref(recordHref(indexTable, indexRow.getValue("id"))));
+      }
+      if(!links.getLinks().isEmpty())
+      {
+         panel.withBlock(links);
+      }
+
+      setStatus(panel, enabled, verdict, openSearchFailed);
+      return (panel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect one record (overridable for tests).
+    *******************************************************************************/
+   protected RecordInspector.Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, AbstractActionInput permissionInput) throws QException
+   {
+      return (new RecordInspector().inspect(runtime, tableName, recordId, null, permissionInput));
+   }
+
+
+
+   /*******************************************************************************
+    ** The table's index row as key/values.
+    *******************************************************************************/
+   private DeveloperPanelBlock indexBlock(QuickSearchQBitConfig config, QuickSearchableTableConfig tableConfig, String indexTable, QRecord indexRow, boolean enabled)
+   {
+      DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Index");
+      if(indexRow == null)
+      {
+         block.withItem("Index row", "none yet (the first indexing run creates it)");
+      }
+      else
+      {
+         block.withItem(new DeveloperPanelKeyValue().withLabel("Index row").withValue("#" + indexRow.getValueString("id")).withHref(recordHref(indexTable, indexRow.getValue("id"))));
+      }
+      block.withItem("Enabled", enabled ? "Yes" : "No");
+      block.withItem("Status", indexRow == null ? null : indexRow.getValueString("status"));
+      block.withItem("Indexed records", indexRow == null ? null : indexRow.getValueString("recordCount"));
+      block.withItem("Last basepull", indexRow == null ? null : indexRow.getValueString("lastBasepullTime"));
+      block.withItem("Last full reindex", indexRow == null ? null : indexRow.getValueString("lastFullReindexTime"));
+      block.withItem("Searchable fields", String.join(", ", CollectionUtils.nonNullList(tableConfig.getSearchableFields())));
+      block.withItem(new DeveloperPanelKeyValue().withLabel("OpenSearch index (alias)").withValue(config.getOpensearchIndexName()).withMonospace(true));
+      block.withItem("Real-time indexing", Boolean.TRUE.equals(config.getEnableRealTimeIndexing()) ? "On" : "Off");
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** The latest index run as key/values.
+    *******************************************************************************/
+   private DeveloperPanelBlock runBlock(String runTable, QRecord run)
+   {
+      DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Latest index run")
+         .withItem(new DeveloperPanelKeyValue().withLabel("Run").withValue("#" + run.getValueString("id")).withHref(recordHref(runTable, run.getValue("id"))))
+         .withItem("Type", run.getValueString("runType"))
+         .withItem("Status", run.getValueString("status"))
+         .withItem("Started", run.getValueString("startTime"))
+         .withItem("Ended", run.getValueString("endTime"))
+         .withItem("Processed", run.getValueString("recordsProcessed"))
+         .withItem("Indexed", run.getValueString("recordsIndexed"))
+         .withItem("Errors", run.getValueString("errorCount"));
+      if(run.getValueString("errorMessage") != null)
+      {
+         block.withItem("Error", truncate(run.getValueString("errorMessage")));
+      }
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** The inspection: verdict text, document facts, field comparison table.
+    *******************************************************************************/
+   private void addInspectionBlocks(DeveloperPanel panel, RecordInspector.Inspection inspection)
+   {
+      if(inspection.error() != null)
+      {
+         panel.withBlock(DeveloperPanelBlock.text(inspection.error(), RecordInspector.VERDICT_NOT_INDEXED.equals(inspection.verdict()) ? DeveloperPanelTone.INFO : DeveloperPanelTone.WARNING)
+            .withLabel("Search document"));
+         return;
+      }
+
+      switch(inspection.verdict())
+      {
+         case RecordInspector.VERDICT_CURRENT -> panel.withBlock(DeveloperPanelBlock.text("Indexed and current: the search document matches the database.", DeveloperPanelTone.SUCCESS)
+            .withLabel("Search document"));
+         case RecordInspector.VERDICT_STALE -> panel.withBlock(DeveloperPanelBlock.text("Indexed but stale: some values differ from the database. Reindex this record to refresh it.", DeveloperPanelTone.WARNING)
+            .withLabel("Search document"));
+         default -> panel.withBlock(DeveloperPanelBlock.text("Not in the index: the global search cannot find this record. Reindex this record to add it (or wait for the next basepull or reconcile).",
+            DeveloperPanelTone.ERROR).withLabel("Search document"));
+      }
+
+      DocumentView indexed = inspection.indexed();
+      panel.withBlock(DeveloperPanelBlock.keyValues("Document")
+         .withItem(new DeveloperPanelKeyValue().withLabel("Document id").withValue(OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId())).withMonospace(true))
+         .withItem(new DeveloperPanelKeyValue().withLabel("Physical index").withValue(indexed == null ? null : indexed.physicalIndex()).withMonospace(true))
+         .withItem("Indexed version", indexed == null || indexed.version() == null ? null : String.valueOf(indexed.version()))
+         .withItem("Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)" : String.valueOf(inspection.expected().getVersion()))
+         .withItem("Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null)));
+
+      DeveloperPanelBlock table = DeveloperPanelBlock.table("Database vs index", List.of("Field", "Database", "Index", "Match"));
+      for(RecordInspector.FieldDiff diff : CollectionUtils.nonNullList(inspection.diffs()))
+      {
+         table.withRow(row(diff.name(), truncate(diff.databaseValue()), truncate(diff.indexValue()), diff.isSame() ? "yes" : "NO"));
+      }
+      panel.withBlock(table);
+   }
+
+
+
+   /*******************************************************************************
+    ** The record's failed index events, or null when there are none.
+    *******************************************************************************/
+   private DeveloperPanelBlock failedEventsBlock(String failedTable, String tableName, String recordId) throws QException
+   {
+      QueryInput queryInput = new QueryInput(failedTable);
+      queryInput.setFilter(new QQueryFilter(
+         new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+         new QFilterCriteria("recordId", QCriteriaOperator.EQUALS, recordId))
+         .withOrderBy(new QFilterOrderBy("id", false))
+         .withLimit(MAX_FAILED_EVENTS));
+      List<QRecord> events = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+      if(events.isEmpty())
+      {
+         return (null);
+      }
+
+      DeveloperPanelBlock block = DeveloperPanelBlock.table("Failed index events" + (events.size() >= MAX_FAILED_EVENTS ? " (latest " + MAX_FAILED_EVENTS + ")" : ""),
+         List.of("Id", "Action", "Status", "Attempts", "Created", "Error"));
+      for(QRecord event : events)
+      {
+         block.withRow(row(event.getValueString("id"), event.getValueString("action"), event.getValueString("status"), event.getValueString("attempts"),
+            event.getValueString("createDate"), truncate(event.getValueString("errorMessage"))));
+      }
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add a record action when the process exists and the user may run it.
+    *******************************************************************************/
+   private void addActionIfPermitted(DeveloperPanelBlock actions, String processName, String tableName, String label, String description)
+   {
+      if(QContext.getQInstance().getProcess(processName) != null && PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName))
+      {
+         actions.withAction(new DeveloperPanelAction().withLabel(label).withProcessName(processName).withTableName(tableName).withDescription(description));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the user may inspect records: the Inspect Search Document
+    ** process's permission (debugPermissionRules), evaluated on the
+    ** registered process, or on an unregistered copy when the record actions
+    ** are turned off.
+    *******************************************************************************/
+   private boolean mayInspect(QuickSearchQBitConfig config, String tableName)
+   {
+      String processName = QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName);
+      if(QContext.getQInstance().getProcess(processName) != null)
+      {
+         return (PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName));
+      }
+      return (PermissionsHelper.getPermissionCheckResult(new QueryInput(tableName), QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(config, tableName))
+         == PermissionCheckResult.ALLOW);
+   }
+
+
+
+   /*******************************************************************************
+    ** Status label and tone from the enabled flag and the inspection verdict.
+    *******************************************************************************/
+   private void setStatus(DeveloperPanel panel, boolean enabled, String verdict, boolean openSearchFailed)
+   {
+      if(!enabled)
+      {
+         panel.withStatusLabel(STATUS_DISABLED).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(openSearchFailed)
+      {
+         panel.withStatusLabel(STATUS_UNAVAILABLE).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(RecordInspector.VERDICT_CURRENT.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_INDEXED).withStatusTone(DeveloperPanelTone.SUCCESS);
+      }
+      else if(RecordInspector.VERDICT_STALE.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_STALE).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(RecordInspector.VERDICT_MISSING.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_NOT_INDEXED).withStatusTone(DeveloperPanelTone.ERROR);
+      }
+      else
+      {
+         panel.withStatusLabel(STATUS_ENABLED).withStatusTone(DeveloperPanelTone.INFO);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the table exists and the user may read it.
+    *******************************************************************************/
+   private static boolean mayRead(String tableName)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      return (qInstance != null && qInstance.getTable(tableName) != null && PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
+   }
+
+
+
+   /*******************************************************************************
+    ** First row of a query (as the user), or null.
+    *******************************************************************************/
+   private static QRecord queryFirst(String tableName, QQueryFilter filter) throws QException
+   {
+      QueryInput queryInput = new QueryInput(tableName);
+      queryInput.setFilter(filter.withLimit(1));
+      List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeHasContents(records) ? records.get(0) : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** App-relative link to a record.
+    *******************************************************************************/
+   private static String recordHref(String tableName, Serializable id)
+   {
+      return ("/app/" + tableName + "/" + id);
+   }
+
+
+
+   /*******************************************************************************
+    ** A table row (null cells as empty strings).
+    *******************************************************************************/
+   private static List<String> row(String... cells)
+   {
+      List<String> row = new ArrayList<>();
+      for(String cell : cells)
+      {
+         row.add(cell == null ? "" : cell);
+      }
+      return (row);
+   }
+
+
+
+   /*******************************************************************************
+    ** Long values cut for display.
+    *******************************************************************************/
+   static String truncate(String value)
+   {
+      if(value == null || value.length() <= MAX_VALUE_LENGTH)
+      {
+         return (value);
+      }
+      return (value.substring(0, MAX_VALUE_LENGTH) + "... (" + value.length() + " characters)");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/QuickSearchDiagnosticsWidgetRenderer.java

@@ -0,0 +1,253 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** The Search Diagnostics widget: the record-search provider's state on the
+ ** server that renders it, the most recent search failures (from every
+ ** server, via Quick Search Diagnostic Event), the latest validation run,
+ ** and where the tools are. Every part says plainly when it has nothing to
+ ** show or cannot be read.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchDiagnosticsWidgetRenderer.class);
+
+   public static final int RECENT_FAILURES = 10;
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", "<p>Quick Search is not configured in this application.</p>")));
+      }
+      return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", renderHtml(runtime))));
+   }
+
+
+
+   /*******************************************************************************
+    ** The widget's HTML.
+    *******************************************************************************/
+   static String renderHtml(QuickSearchRuntime runtime)
+   {
+      QuickSearchQBitConfig config = runtime.getConfig();
+      DiagnosticsHtml       html   = new DiagnosticsHtml().open();
+
+      renderIndexingStatus(html, runtime);
+
+      SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(QContext.getQInstance(), runtime);
+      html.subheading("Global search on this server");
+      if(!Boolean.TRUE.equals(state.quickSearchProvider()) || !Boolean.TRUE.equals(state.serveCoreRecordSearch()))
+      {
+         html.status(Level.INFO, "Quick Search does not serve the global search here (" + (state.registeredProvider() == null ? "no provider registered" : "provider " + state.registeredProvider())
+            + "); core searches every table itself.");
+      }
+      else if(state.backoffRemainingMillis() > 0)
+      {
+         html.status(Level.FAIL, "Backing off for " + (state.backoffRemainingMillis() / 1000) + " s after an OpenSearch failure: the global search uses core's database search meanwhile. Last error: "
+            + state.lastFailureMessage());
+      }
+      else
+      {
+         html.status(Level.OK, "Served by OpenSearch through Quick Search (timeout " + state.timeoutMillis() + " ms).");
+      }
+      html.facts(List.of(
+         new String[] { "Failed OpenSearch searches since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) },
+         new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) },
+         new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() }));
+
+      renderRecentFailures(html, config);
+      renderLastValidation(html, config);
+
+      html.subheading("Tools");
+      html.paragraph("On Quick Search Index (Actions): Add Search Index, Apply Search Index Changes, Disable and Remove Search Index, "
+         + "Search Playground, Validate Search Index, Test Search Analyzer. Searchable fields are edited on each index record. "
+         + "On Quick Search Settings: Pause and Resume Indexing; on Quick Search Index Run: Cancel Index Run. "
+         + "On any indexed record and on Quick Search Failed Event rows: Inspect Search Document and Reindex in Search. "
+         + "Every run is recorded in Quick Search Diagnostic Event.");
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Indexing: paused (since when, by whom), what self-priming added and is
+    ** building, and field changes waiting to be applied.
+    *******************************************************************************/
+   static void renderIndexingStatus(DiagnosticsHtml html, QuickSearchRuntime runtime)
+   {
+      QuickSearchStatus status = QuickSearchStatus.read(runtime);
+      html.subheading("Indexing");
+      String banner = status.bannerHtml();
+      if(!banner.isEmpty())
+      {
+         html.raw(banner);
+      }
+      if(status.paused())
+      {
+         html.raw("<p><a href=\"" + DiagnosticsHtml.escape(QuickSearchStatus.settingsPath(runtime)) + "\">Quick Search Settings</a>: Resume Indexing (Actions).</p>");
+      }
+      else
+      {
+         html.status(Level.OK, "Indexing is running (real-time and scheduled). Pause it from "
+            + "Quick Search Settings (Actions: Pause Indexing); search keeps serving while paused.");
+      }
+      if(!status.recordsMode())
+      {
+         html.status(Level.INFO, "The searchable tables come from code: the Quick Search config tables are not in this application yet, or not primed. "
+            + "See docs/MIGRATION-1.1.md (data-driven config).");
+      }
+      else if(status.lastPrimedAt() != null)
+      {
+         html.paragraph("Self-priming ran " + status.lastPrimedAt() + ": " + (status.lastPrimingSummary() == null ? "no changes." : status.lastPrimingSummary()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderRecentFailures(DiagnosticsHtml html, QuickSearchQBitConfig config)
+   {
+      String tableName = config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
+      html.subheading("Recent search failures (all servers)");
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         html.paragraph("Not recorded: the diagnostic event table is not configured.");
+         return;
+      }
+
+      try
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("eventType", QCriteriaOperator.EQUALS, QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED))
+            .withOrderBy(new QFilterOrderBy("id", false))
+            .withLimit(RECENT_FAILURES));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            html.status(Level.OK, "None recorded (failures are kept for " + config.getRunHistoryRetentionDays() + " days).");
+            return;
+         }
+
+         List<String> items = new ArrayList<>();
+         for(QRecord row : rows)
+         {
+            items.add("<a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
+               + DiagnosticsHtml.escape(row.getValueString("createDate")) + "</a> on " + DiagnosticsHtml.escape(row.getValueString("hostName"))
+               + ": " + DiagnosticsHtml.escape(row.getValueString("source")) + ", " + DiagnosticsHtml.escape(row.getValueString("durationMillis")) + " ms, "
+               + DiagnosticsHtml.escape(shortClass(row.getValueString("errorClass"))) + ": " + DiagnosticsHtml.escape(abbreviate(row.getValueString("errorMessage"), 240)));
+         }
+         html.rawList(items);
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read recent search failures for the diagnostics widget", e);
+         html.status(Level.WARN, "Could not read the diagnostic events: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderLastValidation(DiagnosticsHtml html, QuickSearchQBitConfig config)
+   {
+      String tableName = config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
+      html.subheading("Latest index validation");
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         html.paragraph("Not kept: the validation run table is not configured.");
+         return;
+      }
+
+      try
+      {
+         QRecord row = DiagnosticsSupport.latestValidation(config);
+         if(row == null)
+         {
+            html.paragraph("None yet. Run Validate Search Index from Quick Search Index (Actions)"
+               + (config.getValidationCronExpression() == null ? "." : "; it is also scheduled (" + config.getValidationCronExpression() + ")."));
+            return;
+         }
+
+         String  status = row.getValueString("status");
+         Level   level  = QuickSearchValidationRun.STATUS_PASS.equals(status) ? Level.OK : (QuickSearchValidationRun.STATUS_WARN.equals(status) ? Level.WARN : Level.FAIL);
+         html.raw("<p>" + DiagnosticsHtml.badge(level) + " <a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
+            + DiagnosticsHtml.escape(row.getValueString("startTime")) + "</a> (" + DiagnosticsHtml.escape(row.getValueString("triggerType")) + "): "
+            + DiagnosticsHtml.escape(row.getValueString("summary")) + "</p>");
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read the latest validation run for the diagnostics widget", e);
+         html.status(Level.WARN, "Could not read the validation runs: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String shortClass(String className)
+   {
+      return (className == null ? "error" : className.substring(className.lastIndexOf('.') + 1));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String abbreviate(String text, int max)
+   {
+      return (text == null ? "" : (text.length() > max ? text.substring(0, max) + "..." : text));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/RecordInspector.java

@@ -0,0 +1,410 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Check;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** Inspect one record's search document: the document as indexed next to the
+ ** document indexing would write from the database now (field by field),
+ ** and, given a term, why the record does or does not match it.
+ ** The match analysis walks the same gates as the global search: term
+ ** length, table indexed and enabled, read permission, record readable,
+ ** document present and current, every query token present among the
+ ** document's indexed tokens (edge n-grams of 2 to 20 characters, AND
+ ** semantics), explain, and the record's rank against the provider's
+ ** per-table id window. Read-only.
+ *******************************************************************************/
+public class RecordInspector
+{
+   public static final String VERDICT_CURRENT     = "CURRENT";
+   public static final String VERDICT_STALE       = "STALE";
+   public static final String VERDICT_MISSING     = "MISSING";
+   public static final String VERDICT_NOT_INDEXED = "NOT_INDEXED";
+   public static final String VERDICT_NO_RECORD   = "NO_RECORD";
+
+   private static final int RANK_WINDOW = RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE;
+
+
+
+   /*******************************************************************************
+    ** One compared value: name, value from the database, value in the index.
+    *******************************************************************************/
+   public record FieldDiff(String name, String databaseValue, String indexValue)
+   {
+      /*******************************************************************************
+       ** Whether the two values are equal.
+       *******************************************************************************/
+      public boolean isSame()
+      {
+         return (Objects.equals(databaseValue, indexValue));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The inspection of one record.
+    *******************************************************************************/
+   public record Inspection(String tableName, String recordId, String verdict, List<FieldDiff> diffs, DocumentView indexed, OpenSearchDocument expected,
+                            List<Check> matchChecks, ExplainResult explain, String error)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect one record, with an optional term to analyze.
+    *******************************************************************************/
+   public Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, String term, AbstractActionInput permissionInput) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         return (new Inspection(tableName, recordId, VERDICT_NOT_INDEXED, List.of(), null, null, List.of(), null, "Quick Search does not index table [" + tableName + "]."));
+      }
+
+      Map<String, QRecord> records = DiagnosticRecordReads.loadForIndexing(tableName, List.of(recordId));
+      QRecord              record  = records.get(recordId);
+      if(record == null && !records.isEmpty())
+      {
+         record = records.values().iterator().next();
+      }
+      if(record == null)
+      {
+         return (new Inspection(tableName, recordId, VERDICT_NO_RECORD, List.of(), null, null, List.of(), null,
+            "Record " + recordId + " does not exist, or you cannot read it."));
+      }
+
+      QuickSearchDiagnosticsOperations ops      = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      OpenSearchDocument               expected = IndexingUtils.buildDocument(record, tableConfig);
+      DocumentView                     indexed  = ops.getDocument(tableName, recordId);
+      List<FieldDiff>                  diffs    = diff(tableConfig, expected, indexed);
+
+      String verdict = !Boolean.TRUE.equals(indexed.found()) ? VERDICT_MISSING : (diffs.stream().allMatch(FieldDiff::isSame) ? VERDICT_CURRENT : VERDICT_STALE);
+
+      List<Check>   checks  = new ArrayList<>();
+      ExplainResult explain = null;
+      String        cleaned = IndexingUtils.normalizeSearchText(term);
+      if(StringUtils.hasContent(cleaned))
+      {
+         explain = analyzeMatch(runtime, ops, tableConfig, recordId, cleaned, verdict, expected, indexed, checks, permissionInput);
+      }
+
+      return (new Inspection(tableName, recordId, verdict, diffs, indexed, expected, checks, explain, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare the expected document with the indexed one: label, searchable
+    ** text, then each configured field (in configuration order) and any extra
+    ** field only the index holds.
+    *******************************************************************************/
+   static List<FieldDiff> diff(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, DocumentView indexed)
+   {
+      JsonNode source = indexed == null || indexed.source() == null ? null : indexed.source();
+
+      List<FieldDiff> diffs = new ArrayList<>();
+      diffs.add(new FieldDiff("recordLabel", expected == null ? null : blankToNull(expected.getRecordLabel()), text(source, "recordLabel")));
+      diffs.add(new FieldDiff("searchableText", expected == null ? null : blankToNull(expected.getSearchableText()), text(source, "searchableText")));
+
+      Set<String> fieldNames = new LinkedHashSet<>(CollectionUtils.nonNullList(tableConfig.getSearchableFields()));
+      if(source != null && source.has("fieldValues"))
+      {
+         source.get("fieldValues").fieldNames().forEachRemaining(fieldNames::add);
+      }
+      for(String field : fieldNames)
+      {
+         Object expectedValue = expected == null || expected.getFieldValues() == null ? null : expected.getFieldValues().get(field);
+         diffs.add(new FieldDiff("fieldValues." + field, expectedValue == null ? null : String.valueOf(expectedValue),
+            source == null ? null : text(source.path("fieldValues"), field)));
+      }
+      return (diffs);
+   }
+
+
+
+   /*******************************************************************************
+    ** Walk the global search's gates for this record and term, adding a check
+    ** per gate; returns the explain result when the document exists.
+    *******************************************************************************/
+   private ExplainResult analyzeMatch(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, String recordId, String term, String verdict,
+      OpenSearchDocument expected, DocumentView indexed, List<Check> checks, AbstractActionInput permissionInput) throws QException
+   {
+      String tableName = tableConfig.getTableName();
+
+      checks.add(term.length() < QuickSearchAction.MIN_TERM_LENGTH
+         ? new Check("Term length", Level.FAIL, "Terms shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters never reach OpenSearch.")
+         : new Check("Term length", Level.OK, term.length() + " characters."));
+
+      boolean enabled = runtime.isTableEnabled(tableName);
+      checks.add(enabled
+         ? new Check("Table enabled", Level.OK, "The table is enabled in Quick Search Index.")
+         : new Check("Table enabled", Level.FAIL, "The table is disabled in Quick Search Index, so Quick Search does not search it (core searches it with its own database search)."));
+
+      boolean readable = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
+      checks.add(readable
+         ? new Check("Table permission", Level.OK, "You can read this table.")
+         : new Check("Table permission", Level.FAIL, "You cannot read this table, so the global search skips it for you."));
+
+      checks.add(new Check("Record access", Level.OK, "You can read this record (record security locks allow it)."));
+
+      switch(verdict)
+      {
+         case VERDICT_MISSING -> checks.add(new Check("Indexed document", Level.FAIL,
+            "The record has no document in the index, so OpenSearch cannot return it. Use Reindex in Search on the record (or wait for the next basepull / reconcile)."));
+         case VERDICT_STALE -> checks.add(new Check("Indexed document", Level.WARN,
+            "The indexed document differs from the database (see the comparison); the search matches the indexed values. Reindex the record to refresh it."));
+         default -> checks.add(new Check("Indexed document", Level.OK, "The indexed document matches the database."));
+      }
+
+      List<AnalyzedToken> queryTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
+      Set<String>         indexedGrams = Boolean.TRUE.equals(indexed.found()) ? tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, text(indexed.source(), "searchableText")))
+         : Set.of();
+      Set<String>         expectedGrams = expected == null ? Set.of() : tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, expected.getSearchableText()));
+
+      if(queryTokens.isEmpty())
+      {
+         checks.add(new Check("Term tokens", Level.FAIL, "The term has no searchable characters after analysis."));
+      }
+      for(AnalyzedToken token : queryTokens)
+      {
+         String value = token.token();
+         if(value.length() < 2)
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "One character: the index holds edge n-grams of 2 to 20 characters, so a single character never matches, and every token must match."));
+         }
+         else if(value.length() > 20)
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Longer than 20 characters: the index holds edge n-grams of at most 20 characters, so this token never matches. Search a shorter prefix."));
+         }
+         else if(indexedGrams.contains(value))
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.OK, "Found in the indexed text (as a word or a word prefix)."));
+         }
+         else if(expectedGrams.contains(value))
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.WARN, "Not in the indexed text, but in the database's current values: it will match once the record is reindexed."));
+         }
+         else
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Not in this record's searchable text: no searchable field holds a word starting with it. Every token must match (AND)."));
+         }
+      }
+
+      ExplainResult explain = null;
+      if(Boolean.TRUE.equals(indexed.found()))
+      {
+         explain = ops.explain(term, tableName, recordId, runtime.getDiscoveredTables());
+         checks.add(Boolean.TRUE.equals(explain.matched())
+            ? new Check("OpenSearch explain", Level.OK, "The search query matches this document with score " + String.format("%.4f", explain.score()) + ".")
+            : new Check("OpenSearch explain", Level.FAIL, "The search query does not match this document."));
+
+         if(Boolean.TRUE.equals(explain.matched()))
+         {
+            checks.add(rankCheck(runtime, ops, tableName, recordId, term));
+         }
+      }
+      return (explain);
+   }
+
+
+
+   /*******************************************************************************
+    ** Where the record ranks among the table's matches, against the number of
+    ** ids the provider asks for with the default per-table limit.
+    *******************************************************************************/
+   private Check rankCheck(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, String tableName, String recordId, String term) throws QException
+   {
+      int         window = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
+      SearchProbe probe  = ops.probeSearch(term, tableName, RANK_WINDOW, runtime.getDiscoveredTables());
+      int         rank   = 0;
+      for(ProbeHit hit : probe.hits())
+      {
+         rank++;
+         if(recordId.equals(hit.recordId()))
+         {
+            if(rank <= RecordSearchAction.DEFAULT_LIMIT_PER_TABLE)
+            {
+               return (new Check("Rank", Level.OK, "#" + rank + " of " + probe.totalHits() + " matches in this table: within the global search's default " + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " results."));
+            }
+            if(rank <= window)
+            {
+               return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: Quick Search returns it, but the global search shows only the first "
+                  + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " readable results per table by default. A more specific term ranks it higher."));
+            }
+            return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: beyond the top " + window + " ids the provider requests per table, so the global search does not show it."));
+         }
+      }
+      return (new Check("Rank", Level.WARN, "Not in the top " + RANK_WINDOW + " of " + probe.totalHits() + " matches in this table, so the global search does not show it."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render inspections as HTML.
+    *******************************************************************************/
+   public static String render(List<Inspection> inspections, String term)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      for(Inspection inspection : inspections)
+      {
+         html.heading("Search document for " + inspection.tableName() + " #" + inspection.recordId());
+         if(inspection.error() != null)
+         {
+            html.status(Level.FAIL, inspection.error());
+            continue;
+         }
+
+         switch(inspection.verdict())
+         {
+            case VERDICT_CURRENT -> html.status(Level.OK, "Indexed and current: the document matches the database.");
+            case VERDICT_STALE -> html.status(Level.WARN, "Indexed but stale: some values differ from the database. Reindex in Search refreshes it.");
+            default -> html.status(Level.FAIL, "Not in the index. Reindex in Search adds it.");
+         }
+
+         DocumentView indexed = inspection.indexed();
+         List<String[]> facts = new ArrayList<>();
+         facts.add(new String[] { "Document id", OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId()) });
+         facts.add(new String[] { "Physical index", indexed == null ? null : indexed.physicalIndex() });
+         facts.add(new String[] { "Indexed version", indexed == null || indexed.version() == null ? null : indexed.version() + versionNote(indexed.version()) });
+         facts.add(new String[] { "Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)"
+            : inspection.expected().getVersion() + versionNote(inspection.expected().getVersion()) });
+         facts.add(new String[] { "Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null) });
+         html.facts(facts);
+
+         html.subheading("Database vs index");
+         List<String> items = new ArrayList<>();
+         for(FieldDiff diff : inspection.diffs())
+         {
+            items.add(DiagnosticsHtml.badge(diff.isSame() ? Level.OK : Level.WARN) + " <strong>" + DiagnosticsHtml.escape(diff.name()) + "</strong>"
+               + (diff.isSame() ? ": " + valueHtml(diff.databaseValue())
+               : "<br>database: " + valueHtml(diff.databaseValue()) + "<br>index: " + valueHtml(diff.indexValue())));
+         }
+         html.rawList(items);
+
+         if(!inspection.matchChecks().isEmpty())
+         {
+            html.subheading("Why does (or doesn't) this record match \"" + term + "\"?");
+            html.checks(inspection.matchChecks());
+            if(inspection.explain() != null && inspection.explain().explanationText() != null)
+            {
+               html.textDetails("Score explanation", inspection.explain().explanationText());
+            }
+         }
+
+         if(indexed != null && indexed.source() != null)
+         {
+            html.jsonDetails("Indexed document (JSON)", indexed.source().toString());
+         }
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String valueHtml(String value)
+   {
+      if(value == null)
+      {
+         return ("<em>none</em>");
+      }
+      String shown = value.length() > 500 ? value.substring(0, 500) + "... (" + value.length() + " characters)" : value;
+      return ("<code>" + DiagnosticsHtml.escape(shown) + "</code>");
+   }
+
+
+
+   /*******************************************************************************
+    ** " (time)" for a version that looks like epoch millis.
+    *******************************************************************************/
+   private static String versionNote(Long version)
+   {
+      return (version != null && version > 946_684_800_000L ? " (" + java.time.Instant.ofEpochMilli(version) + ")" : "");
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static Set<String> tokenSet(List<AnalyzedToken> tokens)
+   {
+      Set<String> set = new LinkedHashSet<>();
+      for(AnalyzedToken token : tokens)
+      {
+         set.add(token.token());
+      }
+      return (set);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String text(JsonNode node, String field)
+   {
+      if(node == null || !node.has(field) || node.get(field).isNull())
+      {
+         return (null);
+      }
+      return (blankToNull(node.get(field).isTextual() ? node.get(field).asText() : node.get(field).toString()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String blankToNull(String value)
+   {
+      return (value == null || value.isEmpty() ? null : value);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/RecordReindexer.java

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** Reindex chosen records of one table now: each record the user can read is
+ ** rebuilt from the database and written to the index (versioned, so a newer
+ ** document is never overwritten); a record that no longer exists has its
+ ** document removed; a record that exists but that the user cannot read is
+ ** skipped. The one diagnostic tool that writes to the index; it never
+ ** changes a record. A disabled table is refused.
+ *******************************************************************************/
+public class RecordReindexer
+{
+   public static final String OUTCOME_INDEXED = "INDEXED";
+   public static final String OUTCOME_DELETED = "DELETED";
+   public static final String OUTCOME_SKIPPED = "SKIPPED";
+   public static final String OUTCOME_FAILED  = "FAILED";
+
+
+
+   /*******************************************************************************
+    ** What happened to one record.
+    *******************************************************************************/
+   public record Outcome(String recordId, String outcome, String message)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Reindex the records of a table by id.
+    *******************************************************************************/
+   public List<Outcome> reindex(QuickSearchRuntime runtime, String tableName, List<String> recordIds) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Quick Search does not index table [" + tableName + "]"));
+      }
+      if(!runtime.isTableEnabled(tableName))
+      {
+         throw (new QException("Table [" + tableName + "] is disabled in Quick Search Index; enable it before reindexing its records"));
+      }
+      if(runtime.isIndexingPaused())
+      {
+         throw (new QException("Quick Search indexing is paused; resume it in Quick Search Settings before reindexing records"));
+      }
+
+      Set<String>          ids      = new LinkedHashSet<>(recordIds);
+      Map<String, QRecord> readable = DiagnosticRecordReads.loadForIndexing(tableName, ids);
+      Set<String>          validIds = new HashSet<>();
+      DiagnosticRecordReads.typedIds(QContext.getQInstance().getTable(tableName), ids).forEach(id -> validIds.add(String.valueOf(id)));
+
+      ////////////////////////////////////////////////////////////////////
+      // ids the user cannot read: exists at all (system view)? if not, //
+      // the record is gone and its document should go too              //
+      ////////////////////////////////////////////////////////////////////
+      List<String> unreadable = ids.stream().filter(id -> !readable.containsKey(id)).toList();
+      Set<String>  existing   = new HashSet<>();
+      if(!unreadable.isEmpty())
+      {
+         List<String> found = new ArrayList<>();
+         QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
+            found.addAll(DiagnosticRecordReads.existingIds(tableName, unreadable)));
+         existing.addAll(found);
+      }
+
+      QuickSearchOpenSearchClient client   = runtime.getClient();
+      List<Outcome>               outcomes = new ArrayList<>();
+
+      List<OpenSearchDocument> docs = new ArrayList<>();
+      for(String id : ids)
+      {
+         QRecord record = readable.get(id);
+         if(record != null)
+         {
+            OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
+            if(doc != null)
+            {
+               docs.add(doc);
+            }
+         }
+      }
+
+      BulkIndexResult indexResult = client.indexDocuments(docs, Math.max(1, runtime.getConfig().getBulkBatchSize() == null ? 500 : runtime.getConfig().getBulkBatchSize()));
+      Set<String>     failedIds   = new HashSet<>();
+      for(String failure : indexResult.getErrors())
+      {
+         ////////////////////////////////////////////////////
+         // failures read "table:recordId: reason"          //
+         ////////////////////////////////////////////////////
+         int    reasonStart = failure.indexOf(": ");
+         String documentId  = reasonStart < 0 ? failure : failure.substring(0, reasonStart);
+         failedIds.add(documentId.substring(documentId.indexOf(':') + 1));
+      }
+
+      List<String> toDelete = new ArrayList<>();
+      for(String id : ids)
+      {
+         if(readable.containsKey(id))
+         {
+            outcomes.add(failedIds.contains(id) ? new Outcome(id, OUTCOME_FAILED, "OpenSearch rejected the document; see the run's errors")
+               : new Outcome(id, OUTCOME_INDEXED, "Rebuilt from the database and written to the index."));
+         }
+         else if(!validIds.contains(id))
+         {
+            outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "Not a valid id for this table."));
+         }
+         else if(existing.contains(id))
+         {
+            outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "The record exists but you cannot read it, so it was not reindexed."));
+         }
+         else
+         {
+            toDelete.add(OpenSearchDocument.buildDocumentId(tableName, id));
+            outcomes.add(new Outcome(id, OUTCOME_DELETED, "The record no longer exists; its document (if any) was removed."));
+         }
+      }
+
+      if(!toDelete.isEmpty())
+      {
+         BulkIndexResult deleteResult = client.deleteDocuments(toDelete, 500);
+         if(!deleteResult.getErrors().isEmpty())
+         {
+            throw (new QException("Removing documents of deleted records failed: " + String.join("; ", deleteResult.getErrors())));
+         }
+      }
+
+      if(!indexResult.getErrors().isEmpty())
+      {
+         outcomes.add(new Outcome("-", OUTCOME_FAILED, "Index errors: " + String.join("; ", indexResult.getErrors())));
+      }
+
+      client.refreshIndex();
+      return (outcomes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Render outcomes as HTML.
+    *******************************************************************************/
+   public static String render(String tableName, List<Outcome> outcomes)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Reindex in search: " + tableName);
+      long indexed = outcomes.stream().filter(o -> OUTCOME_INDEXED.equals(o.outcome())).count();
+      long deleted = outcomes.stream().filter(o -> OUTCOME_DELETED.equals(o.outcome())).count();
+      long skipped = outcomes.stream().filter(o -> OUTCOME_SKIPPED.equals(o.outcome())).count();
+      long failed  = outcomes.stream().filter(o -> OUTCOME_FAILED.equals(o.outcome())).count();
+      html.status(failed > 0 ? Level.FAIL : (skipped > 0 ? Level.WARN : Level.OK),
+         indexed + " reindexed, " + deleted + " removed (record gone), " + skipped + " skipped, " + failed + " failed. The documents are searchable now.");
+
+      List<String> items = new ArrayList<>();
+      for(Outcome outcome : outcomes)
+      {
+         Level level = switch(outcome.outcome())
+         {
+            case OUTCOME_INDEXED, OUTCOME_DELETED -> Level.OK;
+            case OUTCOME_SKIPPED -> Level.WARN;
+            default -> Level.FAIL;
+         };
+         items.add(DiagnosticsHtml.badge(level) + " #" + DiagnosticsHtml.escape(outcome.recordId()) + ": " + DiagnosticsHtml.escape(outcome.message()));
+      }
+      html.rawList(items);
+      return (html.close().toString());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/SearchPathDiagnosis.java

@@ -0,0 +1,178 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+
+
+/*******************************************************************************
+ ** Which path the global search (core RecordSearchAction, POST
+ ** /qqq/v1/search) takes for each table, and why: OpenSearch through the
+ ** Quick Search record-search provider, core's own database search, or not
+ ** searched at all. Mirrors RecordSearchAction's table selection and
+ ** QuickSearchRecordSearchProvider.claimsTable without calling them, so a
+ ** diagnosis changes no state (no backoff, no counters).
+ *******************************************************************************/
+public final class SearchPathDiagnosis
+{
+   public static final String PATH_OPENSEARCH = "OpenSearch (Quick Search provider)";
+   public static final String PATH_CORE       = "Core database search";
+   public static final String PATH_NONE       = "Not searched";
+
+
+
+   /*******************************************************************************
+    ** The record-search provider's state in this JVM.
+    *******************************************************************************/
+   public record ProviderState(String registeredProvider, Boolean quickSearchProvider, Boolean serveCoreRecordSearch, Long backoffRemainingMillis,
+                               Long failureCount, Instant lastFailureTime, String lastFailureMessage, Long backoffDeclinedClaims, Integer timeoutMillis, Instant runtimeStartedAt)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's path and the reason for it.
+    *******************************************************************************/
+   public record TablePath(String tableName, String tableLabel, Boolean indexed, Boolean enabled, Boolean readable, Boolean coreSearchable, String path, String reason)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private SearchPathDiagnosis()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The provider's state from the instance and the runtime.
+    *******************************************************************************/
+   public static ProviderState providerState(QInstance qInstance, QuickSearchRuntime runtime)
+   {
+      QCodeReference        provider = qInstance.getRecordSearchProvider();
+      QuickSearchQBitConfig config   = runtime.getConfig();
+      return (new ProviderState(
+         provider == null ? null : provider.getName(),
+         provider != null && QuickSearchRecordSearchProvider.class.getName().equals(provider.getName()),
+         Boolean.TRUE.equals(config.getServeCoreRecordSearch()),
+         runtime.getRecordSearchBackoffRemainingMillis(),
+         runtime.getRecordSearchFailureCount(),
+         runtime.getLastRecordSearchFailureTime(),
+         runtime.getLastRecordSearchFailureMessage(),
+         runtime.getBackoffDeclinedClaimCount(),
+         config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis(),
+         runtime.getStartedAt()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The path of each named table (unknown names are reported as not
+    ** searched), as seen by the session of permissionInput.
+    *******************************************************************************/
+   public static List<TablePath> diagnose(QInstance qInstance, QuickSearchRuntime runtime, List<String> tableNames, AbstractActionInput permissionInput)
+   {
+      ProviderState   state = providerState(qInstance, runtime);
+      List<TablePath> paths = new ArrayList<>();
+      for(String tableName : CollectionUtils.nonNullList(tableNames))
+      {
+         paths.add(diagnoseTable(qInstance, runtime, state, tableName, permissionInput));
+      }
+      return (paths);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static TablePath diagnoseTable(QInstance qInstance, QuickSearchRuntime runtime, ProviderState state, String tableName, AbstractActionInput permissionInput)
+   {
+      QTableMetaData table = qInstance.getTable(tableName);
+      if(table == null)
+      {
+         return (new TablePath(tableName, tableName, false, false, false, false, PATH_NONE, "There is no table with this name."));
+      }
+
+      boolean indexed        = runtime.getTableConfig(tableName) != null;
+      boolean enabled        = indexed && runtime.isTableEnabled(tableName);
+      boolean readable       = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
+      boolean coreSearchable = !CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) && table.isCapabilityEnabled(qInstance.getBackendForTable(tableName), Capability.TABLE_QUERY);
+      String  label          = table.getLabel() == null ? tableName : table.getLabel();
+
+      if(!coreSearchable)
+      {
+         return (new TablePath(tableName, label, indexed, enabled, readable, false, PATH_NONE,
+            "Global search skips this table: it has no core searchFields (or does not support query)."));
+      }
+      if(!readable)
+      {
+         return (new TablePath(tableName, label, indexed, enabled, false, true, PATH_NONE,
+            "Global search skips this table for you: you do not have read permission on it."));
+      }
+      if(!indexed)
+      {
+         return (new TablePath(tableName, label, false, false, true, true, PATH_CORE, "Quick Search does not index this table."));
+      }
+      if(!Boolean.TRUE.equals(state.serveCoreRecordSearch()))
+      {
+         return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, "serveCoreRecordSearch is off in the Quick Search configuration."));
+      }
+      if(!Boolean.TRUE.equals(state.quickSearchProvider()))
+      {
+         String reason = state.registeredProvider() == null
+            ? "No record search provider is registered on the instance."
+            : "Another record search provider is registered (" + state.registeredProvider() + "), so Quick Search does not serve global search.";
+         return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, reason));
+      }
+      if(!enabled)
+      {
+         return (new TablePath(tableName, label, true, false, true, true, PATH_CORE, "The table is disabled in Quick Search Index (enabled = false)."));
+      }
+      if(state.backoffRemainingMillis() != null && state.backoffRemainingMillis() > 0)
+      {
+         return (new TablePath(tableName, label, true, true, true, true, PATH_CORE,
+            "Quick Search is backing off for " + Math.max(1, state.backoffRemainingMillis() / 1000) + " more second(s) after an OpenSearch failure"
+               + (state.lastFailureTime() == null ? "" : " at " + state.lastFailureTime())
+               + (state.lastFailureMessage() == null ? "" : ": " + state.lastFailureMessage())));
+      }
+
+      return (new TablePath(tableName, label, true, true, true, true, PATH_OPENSEARCH,
+         "Claimed by the Quick Search provider. If OpenSearch fails or takes longer than " + state.timeoutMillis()
+            + " ms, core searches this table itself for that request and Quick Search backs off for "
+            + (QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS / 1000) + " seconds."));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/SearchPlayground.java

@@ -0,0 +1,654 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** The search playground: runs a term the way the global search does, as the
+ ** current user, and shows everything behind it.
+ ** 1. Which path serves each table and why (SearchPathDiagnosis).
+ ** 2. The provider's own OpenSearch request (_msearch of record ids), timed,
+ **    without starting a backoff when it fails.
+ ** 3. The real global search (core RecordSearchAction) as the current user:
+ **    what users see, per table, timed. Like any user search, a provider
+ **    failure here starts the provider's normal backoff.
+ ** 4. Per indexed table, the production query with scores, highlights and
+ **    matched clauses, which hits users can read (record locks), which ones
+ **    the global search shows, the query DSL and the raw response (hits the
+ **    user cannot read are redacted to their id).
+ ** 5. Optionally, explain for one hit, and the term's tokens.
+ ** Read-only: no index or record is changed.
+ *******************************************************************************/
+public class SearchPlayground
+{
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+
+
+   /*******************************************************************************
+    ** What to run: term, tables (empty for every searchable table), results per
+    ** table as in the global search, and an optional hit to explain
+    ** ("table:recordId").
+    *******************************************************************************/
+   public record Input(String term, List<String> tableNames, Integer limitPerTable, String explainDocumentId)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's probe: the search, which hit ids the user can read, which
+    ** the global search showed, and an error when the probe failed.
+    *******************************************************************************/
+   public record TableProbe(String tableName, SearchProbe probe, Set<String> readableIds, Set<String> shownIds, Integer hiddenCount, String error)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Everything the playground found.
+    *******************************************************************************/
+   public record Report(String term, List<String> notices, List<AnalyzedToken> termTokens, String termTokensError,
+                        SearchPathDiagnosis.ProviderState providerState, List<SearchPathDiagnosis.TablePath> paths,
+                        Map<String, List<String>> providerIds, Map<String, Integer> providerHiddenCounts, Long providerMillis, String providerError,
+                        Map<String, List<RecordSearchResult>> coreResults, Long coreMillis, String coreError,
+                        List<TableProbe> probes, String explainDocumentId, ExplainResult explain, String explainError,
+                        Integer limitPerTable, Integer idsPerTable)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the playground for the input as the current session.
+    *******************************************************************************/
+   public Report run(QuickSearchRuntime runtime, Input input, AbstractActionInput permissionInput)
+   {
+      QInstance             qInstance = QContext.getQInstance();
+      QuickSearchQBitConfig config    = runtime.getConfig();
+      List<String>          notices   = new ArrayList<>();
+
+      String term          = IndexingUtils.normalizeSearchText(input.term());
+      int    limitPerTable = input.limitPerTable() == null || input.limitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : Math.min(input.limitPerTable(), RecordSearchAction.MAX_LIMIT_PER_TABLE);
+      int    idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
+
+      List<String> tables = resolveTables(qInstance, runtime, input.tableNames(), notices);
+      List<SearchPathDiagnosis.TablePath> paths = SearchPathDiagnosis.diagnose(qInstance, runtime, tables, permissionInput);
+      SearchPathDiagnosis.ProviderState   state = SearchPathDiagnosis.providerState(qInstance, runtime);
+
+      if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         notices.add("The term is shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters: the Quick Search provider returns nothing for it, so only core's own search can match.");
+      }
+      if(term.length() > RecordSearchAction.MAX_SEARCH_TERM_LENGTH)
+      {
+         notices.add("The term is longer than " + RecordSearchAction.MAX_SEARCH_TERM_LENGTH + " characters: the global search rejects it.");
+      }
+
+      QuickSearchDiagnosticsOperations ops = null;
+      String                           opsError = null;
+      try
+      {
+         ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      }
+      catch(QException e)
+      {
+         opsError = e.getMessage();
+         notices.add("The OpenSearch client could not be built: " + e.getMessage());
+      }
+
+      ////////////////////////
+      // term tokens        //
+      ////////////////////////
+      List<AnalyzedToken> termTokens      = null;
+      String              termTokensError = opsError;
+      if(ops != null && StringUtils.hasContent(term))
+      {
+         try
+         {
+            termTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
+         }
+         catch(QException e)
+         {
+            termTokensError = e.getMessage();
+         }
+      }
+
+      ///////////////////////////////////////////////////////////
+      // the provider's own request, for the tables it claims  //
+      ///////////////////////////////////////////////////////////
+      List<String> claimed = paths.stream().filter(p -> SearchPathDiagnosis.PATH_OPENSEARCH.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
+      Map<String, List<String>> providerIds    = null;
+      Map<String, Integer>      providerHidden = new LinkedHashMap<>();
+      Long                      providerMillis = null;
+      String                    providerError  = null;
+      if(!claimed.isEmpty() && ops != null && term.length() >= QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         int  timeout = state.timeoutMillis();
+         long start   = System.currentTimeMillis();
+         try
+         {
+            providerIds = runtime.getClient().searchRecordIdsPerTable(term, claimed, idsPerTable, runtime.getDiscoveredTables(), timeout);
+         }
+         catch(QException | RuntimeException e)
+         {
+            providerError = e.getClass().getSimpleName() + ": " + e.getMessage();
+         }
+         providerMillis = System.currentTimeMillis() - start;
+         providerIds    = providerIds == null ? null : readableOnly(providerIds, providerHidden);
+      }
+
+      ///////////////////////////////////////////////////////
+      // the real global search, as the current user       //
+      ///////////////////////////////////////////////////////
+      Map<String, List<RecordSearchResult>> coreResults = new LinkedHashMap<>();
+      Long                                  coreMillis  = null;
+      String                                coreError   = null;
+      List<String> searched = paths.stream().filter(p -> !SearchPathDiagnosis.PATH_NONE.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
+      if(!searched.isEmpty() && StringUtils.hasContent(term))
+      {
+         long start = System.currentTimeMillis();
+         try
+         {
+            RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm(term).withTableNames(searched).withLimitPerTable(limitPerTable));
+            for(RecordSearchResult result : CollectionUtils.nonNullList(output.getResults()))
+            {
+               coreResults.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
+            }
+         }
+         catch(Exception e)
+         {
+            coreError = e.getClass().getSimpleName() + ": " + e.getMessage();
+         }
+         coreMillis = System.currentTimeMillis() - start;
+      }
+
+      //////////////////////////////////////////
+      // per indexed table, the probe search  //
+      //////////////////////////////////////////
+      List<TableProbe> probes = new ArrayList<>();
+      for(SearchPathDiagnosis.TablePath path : paths)
+      {
+         if(!Boolean.TRUE.equals(path.indexed()) || !Boolean.TRUE.equals(path.readable()) || term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+         {
+            continue;
+         }
+         if(ops == null)
+         {
+            probes.add(new TableProbe(path.tableName(), null, Set.of(), Set.of(), 0, opsError));
+            continue;
+         }
+         probes.add(probeTable(ops, runtime, term, path.tableName(), idsPerTable, coreResults.get(path.tableName())));
+      }
+
+      //////////////////////
+      // explain one hit  //
+      //////////////////////
+      ExplainResult explain      = null;
+      String        explainError = null;
+      String        explainId    = StringUtils.hasContent(input.explainDocumentId()) ? input.explainDocumentId().trim() : null;
+      if(explainId != null && ops != null)
+      {
+         try
+         {
+            explain = explainHit(ops, runtime, term, explainId);
+         }
+         catch(QException e)
+         {
+            explainError = e.getMessage();
+         }
+      }
+
+      return (new Report(term, notices, termTokens, termTokensError, state, paths, providerIds, providerHidden, providerMillis, providerError,
+         coreResults, coreMillis, coreError, probes, explainId, explain, explainError, limitPerTable, idsPerTable));
+   }
+
+
+
+   /*******************************************************************************
+    ** The requested tables that exist (else every table with core searchFields
+    ** or a Quick Search index), in instance order.
+    *******************************************************************************/
+   static List<String> resolveTables(QInstance qInstance, QuickSearchRuntime runtime, List<String> requested, List<String> notices)
+   {
+      List<String> tables = new ArrayList<>();
+      if(!CollectionUtils.nullSafeIsEmpty(requested))
+      {
+         for(String tableName : new LinkedHashSet<>(requested))
+         {
+            if(qInstance.getTable(tableName) == null)
+            {
+               notices.add("Ignored unknown table [" + tableName + "].");
+            }
+            else
+            {
+               tables.add(tableName);
+            }
+         }
+         return (tables);
+      }
+
+      for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values())
+      {
+         if(!CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) || runtime.getTableConfig(table.getName()) != null)
+         {
+            tables.add(table.getName());
+         }
+      }
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Probe one table and mark which hits the user can read and which ones the
+    ** global search showed.
+    *******************************************************************************/
+   private TableProbe probeTable(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String tableName, int size, List<RecordSearchResult> shown)
+   {
+      try
+      {
+         SearchProbe  probe = ops.probeSearch(term, tableName, size, runtime.getDiscoveredTables());
+         List<String> ids   = probe.hits().stream().map(ProbeHit::recordId).toList();
+
+         Set<String> readable = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, ids));
+         Set<String> shownIds = new HashSet<>();
+         for(RecordSearchResult result : CollectionUtils.nonNullList(shown))
+         {
+            shownIds.add(String.valueOf(result.getRecordId()));
+         }
+
+         List<ProbeHit> visible = probe.hits().stream().filter(h -> readable.contains(h.recordId())).toList();
+         int            hidden  = probe.hits().size() - visible.size();
+         SearchProbe redacted = new SearchProbe(probe.tableName(), probe.requestJson(), redactResponse(probe.responseJson(), readable),
+            probe.tookMillis(), probe.wallMillis(), hidden == 0 ? probe.totalHits() : null, probe.totalIsLowerBound(), visible);
+         return (new TableProbe(tableName, redacted, readable, shownIds, hidden, null));
+      }
+      catch(QException e)
+      {
+         return (new TableProbe(tableName, null, Set.of(), Set.of(), 0, e.getMessage()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The provider's ids per table, keeping only records the user can read;
+    ** how many were dropped goes in hiddenCounts. Ids of records the user
+    ** cannot read are never shown (they would reveal what a locked record
+    ** contains).
+    *******************************************************************************/
+   static Map<String, List<String>> readableOnly(Map<String, List<String>> idsByTable, Map<String, Integer> hiddenCounts)
+   {
+      Map<String, List<String>> readable = new LinkedHashMap<>();
+      for(Map.Entry<String, List<String>> entry : idsByTable.entrySet())
+      {
+         List<String> ids = entry.getValue() == null ? List.of() : entry.getValue();
+         Set<String>  ok;
+         try
+         {
+            ok = new HashSet<>(DiagnosticRecordReads.existingIds(entry.getKey(), ids));
+         }
+         catch(QException e)
+         {
+            ok = Set.of();
+         }
+         Set<String> finalOk = ok;
+         readable.put(entry.getKey(), ids.stream().filter(finalOk::contains).toList());
+         hiddenCounts.put(entry.getKey(), (int) ids.stream().filter(id -> !finalOk.contains(id)).count());
+      }
+      return (readable);
+   }
+
+
+
+   /*******************************************************************************
+    ** The raw response without the hits the user cannot read: they are
+    ** removed whole (id, score, source, highlights, matched queries), and
+    ** when any was removed the total and max score are redacted too, so the
+    ** response reveals nothing about records the user cannot read.
+    *******************************************************************************/
+   static String redactResponse(String responseJson, Set<String> readableIds)
+   {
+      try
+      {
+         JsonNode root    = MAPPER.readTree(responseJson);
+         JsonNode hitList = root.path("hits").path("hits");
+         int      removed = 0;
+         if(hitList.isArray())
+         {
+            ArrayNode array = (ArrayNode) hitList;
+            for(int i = array.size() - 1; i >= 0; i--)
+            {
+               String recordId = array.get(i).path("_source").path("recordId").asText(null);
+               if(recordId == null || !readableIds.contains(recordId))
+               {
+                  array.remove(i);
+                  removed++;
+               }
+            }
+         }
+         if(removed > 0 && root.path("hits") instanceof ObjectNode hits)
+         {
+            hits.put("total", "redacted: " + removed + " hit(s) you cannot read were removed");
+            hits.remove("max_score");
+         }
+         return (root.toString());
+      }
+      catch(Exception e)
+      {
+         return ("{\"redacted\":\"response could not be parsed for redaction\"}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Explain one hit ("table:recordId") when the table is indexed and the user
+    ** can read the record.
+    *******************************************************************************/
+   private ExplainResult explainHit(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String documentId) throws QException
+   {
+      int colon = documentId.indexOf(':');
+      if(colon <= 0 || colon == documentId.length() - 1)
+      {
+         throw (new QException("Give the hit as table:recordId (its document id), for example blogPost:42"));
+      }
+      String tableName = documentId.substring(0, colon);
+      String recordId  = documentId.substring(colon + 1);
+
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Table [" + tableName + "] is not indexed by Quick Search"));
+      }
+      if(DiagnosticRecordReads.existingIds(tableName, List.of(recordId)).isEmpty())
+      {
+         throw (new QException("Record " + recordId + " of [" + tableName + "] does not exist or you cannot read it"));
+      }
+      return (ops.explain(term, tableName, recordId, runtime.getDiscoveredTables()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render a report as HTML.
+    *******************************************************************************/
+   public static String render(Report report)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Search playground: \"" + report.term() + "\"");
+      html.paragraph("Run as you, at " + Instant.now() + ". The global search shows up to " + report.limitPerTable() + " results per table; the Quick Search provider asks OpenSearch for the top "
+         + report.idsPerTable() + " ids per table and core keeps the first " + report.limitPerTable() + " you can read.");
+
+      for(String notice : report.notices())
+      {
+         html.status(Level.WARN, notice);
+      }
+
+      renderProviderState(html, report.providerState());
+
+      html.subheading("Path per table");
+      List<String> pathItems = new ArrayList<>();
+      for(SearchPathDiagnosis.TablePath path : report.paths())
+      {
+         Level level = SearchPathDiagnosis.PATH_OPENSEARCH.equals(path.path()) ? Level.OK : (SearchPathDiagnosis.PATH_CORE.equals(path.path()) ? Level.INFO : Level.WARN);
+         pathItems.add(DiagnosticsHtml.badge(level) + " <strong>" + DiagnosticsHtml.escape(path.tableLabel()) + "</strong> (" + DiagnosticsHtml.escape(path.tableName()) + "): "
+            + DiagnosticsHtml.escape(path.path()) + ". " + DiagnosticsHtml.escape(path.reason()));
+      }
+      if(pathItems.isEmpty())
+      {
+         html.paragraph("No table is searchable by the global search.");
+      }
+      html.rawList(pathItems);
+
+      html.subheading("Term tokens (search analyzer)");
+      if(report.termTokensError() != null)
+      {
+         html.status(Level.FAIL, "Could not analyze the term: " + report.termTokensError());
+      }
+      else if(report.termTokens() != null)
+      {
+         html.paragraph(tokenSummary(report.termTokens()));
+         for(AnalyzedToken token : report.termTokens())
+         {
+            if(token.token().length() < 2)
+            {
+               html.status(Level.WARN, "\"" + token.token() + "\" is one character: the index holds edge n-grams of 2 to 20 characters, so it can never match, and every term must match.");
+            }
+            else if(token.token().length() > 20)
+            {
+               html.status(Level.WARN, "\"" + token.token() + "\" is longer than 20 characters: the index holds edge n-grams of at most 20 characters, so it can never match.");
+            }
+         }
+      }
+
+      html.subheading("Quick Search provider request");
+      if(report.providerMillis() == null)
+      {
+         html.paragraph("Not run: the provider claims none of these tables (see the paths above), or the term is too short.");
+      }
+      else if(report.providerError() != null)
+      {
+         html.status(Level.FAIL, "Failed after " + report.providerMillis() + " ms: " + report.providerError() + ". A user search hitting this falls back to core's database search for these tables.");
+      }
+      else
+      {
+         html.status(Level.OK, "Answered in " + report.providerMillis() + " ms (wall clock, limit " + report.providerState().timeoutMillis() + " ms).");
+         List<String> items = new ArrayList<>();
+         for(Map.Entry<String, List<String>> entry : report.providerIds().entrySet())
+         {
+            Integer hidden = report.providerHiddenCounts().getOrDefault(entry.getKey(), 0);
+            items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + entry.getValue().size() + " id(s) you can read"
+               + (hidden > 0 ? " (" + hidden + " more you cannot read, not shown)" : "")
+               + (entry.getValue().isEmpty() ? "" : ": " + DiagnosticsHtml.escape(String.join(", ", entry.getValue().subList(0, Math.min(20, entry.getValue().size()))))
+               + (entry.getValue().size() > 20 ? ", ..." : "")));
+         }
+         html.rawList(items);
+      }
+
+      html.subheading("Global search results (what you see)");
+      if(report.coreMillis() == null)
+      {
+         html.paragraph("Not run: no table is searchable, or the term is empty.");
+      }
+      else if(report.coreError() != null)
+      {
+         html.status(Level.FAIL, "The global search failed after " + report.coreMillis() + " ms: " + report.coreError());
+      }
+      else
+      {
+         int total = report.coreResults().values().stream().mapToInt(List::size).sum();
+         html.status(total == 0 ? Level.INFO : Level.OK, total + " result(s) in " + report.coreMillis() + " ms.");
+         List<String> items = new ArrayList<>();
+         for(Map.Entry<String, List<RecordSearchResult>> entry : report.coreResults().entrySet())
+         {
+            List<String> labels = new ArrayList<>();
+            for(RecordSearchResult result : entry.getValue())
+            {
+               labels.add(DiagnosticsHtml.escape(result.getRecordLabel() == null ? String.valueOf(result.getRecordId()) : result.getRecordLabel()) + " (#" + DiagnosticsHtml.escape(result.getRecordId()) + ")");
+            }
+            items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + String.join("; ", labels));
+         }
+         html.rawList(items);
+      }
+
+      html.subheading("OpenSearch detail per indexed table");
+      if(report.probes().isEmpty())
+      {
+         html.paragraph("No indexed table you can read was searched, or the term is too short.");
+      }
+      for(TableProbe tableProbe : report.probes())
+      {
+         renderProbe(html, tableProbe);
+      }
+
+      if(report.explainDocumentId() != null)
+      {
+         html.subheading("Explain " + report.explainDocumentId());
+         if(report.explainError() != null)
+         {
+            html.status(Level.FAIL, report.explainError());
+         }
+         else if(report.explain() != null && !Boolean.TRUE.equals(report.explain().documentFound()))
+         {
+            html.status(Level.FAIL, "There is no document " + report.explainDocumentId() + " in the index.");
+         }
+         else if(report.explain() != null)
+         {
+            html.status(Boolean.TRUE.equals(report.explain().matched()) ? Level.OK : Level.WARN,
+               Boolean.TRUE.equals(report.explain().matched()) ? "Matches with score " + report.explain().score() + "." : "Does not match this term.");
+            html.textDetails("Score explanation", report.explain().explanationText());
+            html.jsonDetails("Explain response (JSON)", report.explain().json());
+         }
+      }
+
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderProviderState(DiagnosticsHtml html, SearchPathDiagnosis.ProviderState state)
+   {
+      html.subheading("Record search provider (this server)");
+      List<String[]> facts = new ArrayList<>();
+      facts.add(new String[] { "Registered provider", state.registeredProvider() == null ? null : state.registeredProvider() + (Boolean.TRUE.equals(state.quickSearchProvider()) ? " (Quick Search)" : " (not Quick Search)") });
+      facts.add(new String[] { "serveCoreRecordSearch", String.valueOf(state.serveCoreRecordSearch()) });
+      facts.add(new String[] { "Timeout", state.timeoutMillis() + " ms" });
+      facts.add(new String[] { "Backing off", state.backoffRemainingMillis() > 0 ? "yes, " + (state.backoffRemainingMillis() / 1000) + " s left" : "no" });
+      facts.add(new String[] { "Failures since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) });
+      facts.add(new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) });
+      facts.add(new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() });
+      html.facts(facts);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderProbe(DiagnosticsHtml html, TableProbe tableProbe)
+   {
+      html.raw("<h4>" + DiagnosticsHtml.escape(tableProbe.tableName()) + "</h4>");
+      if(tableProbe.error() != null)
+      {
+         html.status(Level.FAIL, "OpenSearch search failed: " + tableProbe.error());
+         return;
+      }
+
+      SearchProbe probe = tableProbe.probe();
+      html.paragraph((probe.totalHits() == null ? "Matching documents: not shown (some are records you cannot read)"
+         : probe.totalHits() + (Boolean.TRUE.equals(probe.totalIsLowerBound()) ? "+" : "") + " matching document(s)")
+         + "; OpenSearch took " + probe.tookMillis() + " ms (" + probe.wallMillis() + " ms wall clock). Showing the " + probe.hits().size() + " top hit(s) you can read.");
+      if(tableProbe.hiddenCount() != null && tableProbe.hiddenCount() > 0)
+      {
+         html.status(Level.INFO, tableProbe.hiddenCount() + " of the top hits are records you cannot read (deleted, or hidden by a record security lock); they are not shown.");
+      }
+
+      List<String> items    = new ArrayList<>();
+      int          position = 0;
+      for(ProbeHit hit : probe.hits())
+      {
+         position++;
+         StringBuilder item = new StringBuilder();
+         item.append("#").append(position).append(" <strong>")
+            .append(DiagnosticsHtml.escape(hit.recordLabel() == null ? "(no label)" : hit.recordLabel()))
+            .append("</strong> &middot; document <code>").append(DiagnosticsHtml.escape(OpenSearchDocument.buildDocumentId(tableProbe.tableName(), hit.recordId()))).append("</code>")
+            .append(" &middot; score ").append(hit.score() == null ? "n/a" : String.format("%.3f", hit.score()))
+            .append(" &middot; ").append(tableProbe.shownIds().contains(hit.recordId()) ? "shown in global search" : "not shown (beyond the per-table limit)");
+         if(!hit.matchedClauses().isEmpty())
+         {
+            List<String> fields = hit.matchedClauses().stream().map(c -> c.startsWith(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX) ? c.substring(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX.length()) : c).toList();
+            item.append("<br>Matched: ").append(DiagnosticsHtml.escape(String.join(", ", fields)));
+         }
+         for(Map.Entry<String, List<String>> highlight : hit.highlights().entrySet())
+         {
+            item.append("<br><em>").append(DiagnosticsHtml.escape(highlight.getKey())).append(":</em> ")
+               .append(DiagnosticsHtml.highlight(String.join(" ... ", highlight.getValue())));
+         }
+         items.add(item.toString());
+      }
+      if(items.isEmpty())
+      {
+         html.paragraph("No document of this table that you can read matches every term.");
+      }
+      html.rawList(items);
+      html.jsonDetails("Query DSL", probe.requestJson());
+      html.jsonDetails("Raw response (hits you cannot read removed)", probe.responseJson());
+   }
+
+
+
+   /*******************************************************************************
+    ** "a, b, c" of token strings.
+    *******************************************************************************/
+   static String tokenSummary(List<AnalyzedToken> tokens)
+   {
+      if(tokens.isEmpty())
+      {
+         return ("No tokens: the term has no searchable characters.");
+      }
+      List<String> parts = new ArrayList<>();
+      for(AnalyzedToken token : tokens)
+      {
+         parts.add("\"" + token.token() + "\"");
+      }
+      return (tokens.size() + " token(s), all of which must match: " + String.join(", ", parts));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java

@@ -0,0 +1,170 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.ReentrantLock;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+
+
+/*******************************************************************************
+ ** Instance-owned five-second cache with one in-flight refresh. A stuck backend
+ ** cannot grow a queue or create more workers. Runtime shutdown stops the worker.
+ *******************************************************************************/
+public class StatusCache implements AutoCloseable
+{
+   private final ReentrantLock lock = new ReentrantLock();
+   private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable ->
+   {
+      Thread thread = new Thread(runnable, "quick-search-status");
+      thread.setDaemon(true);
+      return (thread);
+   });
+   private Future<QuickSearchStatus> refresh;
+   private QuickSearchStatus cached;
+   private long expiresAt;
+   private long refreshStartedAt;
+
+
+   /*******************************************************************************
+    ** Authorization belongs to the public service, before reaching this cache.
+    ** Only refreshes consume the existing per-user diagnostic rate limit.
+    *******************************************************************************/
+   public QuickSearchStatus get(QuickSearchRuntime runtime) throws QException
+   {
+      return (get(runtime, Instant.now().plusSeconds(2)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bound both lock contention and the shared refresh wait by the caller's deadline.
+    *******************************************************************************/
+   public QuickSearchStatus get(QuickSearchRuntime runtime, Instant deadline) throws QException
+   {
+      try
+      {
+         long remaining = Duration.between(Instant.now(), deadline).toNanos();
+         if(remaining <= 0 || !lock.tryLock(remaining, TimeUnit.NANOSECONDS))
+         {
+            throw (new QException("Status deadline exceeded"));
+         }
+         try
+         {
+            return (read(runtime, deadline));
+         }
+         finally
+         {
+            lock.unlock();
+         }
+      }
+      catch(InterruptedException e)
+      {
+         Thread.currentThread().interrupt();
+         throw (new QException("Status refresh interrupted", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Read or refresh while holding the deadline-aware cache lock.
+    *******************************************************************************/
+   private QuickSearchStatus read(QuickSearchRuntime runtime, Instant deadline) throws QException
+   {
+      if(cached != null && System.nanoTime() < expiresAt)
+      {
+         return (cached);
+      }
+      String homeHref = "/app/" + runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
+      // A refresh that finished while nobody was polling must not resurrect an old snapshot.
+      if(refresh != null && refresh.isDone() && System.nanoTime() - refreshStartedAt >= TimeUnit.SECONDS.toNanos(5))
+      {
+         refresh = null;
+      }
+      if(refresh == null)
+      {
+         DiagnosticsSupport.checkRateLimit(runtime, "STATUS");
+         QInstance instance = QContext.getQInstance();
+         refreshStartedAt = System.nanoTime();
+         refresh = executor.submit(() ->
+         {
+            QContext.init(instance, new QSystemUserSession(instance));
+            try
+            {
+               return (StatusDiagnostics.read(runtime));
+            }
+            finally
+            {
+               QContext.clear();
+            }
+         });
+      }
+      try
+      {
+         long remaining = Math.max(0, Duration.between(Instant.now(), deadline).toNanos());
+         cached = refresh.get(Math.min(TimeUnit.SECONDS.toNanos(2), remaining), TimeUnit.NANOSECONDS);
+         refresh = null;
+      }
+      catch(TimeoutException e)
+      {
+         // Leave the one worker in flight; never pile up abandoned requests.
+         cached = StatusDiagnostics.unavailable(QuickSearchStatus.State.DEGRADED, "Status refresh timed out; measurements are unknown.", homeHref);
+      }
+      catch(InterruptedException e)
+      {
+         Thread.currentThread().interrupt();
+         throw (new QException("Status refresh interrupted", e));
+      }
+      catch(ExecutionException e)
+      {
+         refresh = null;
+         cached = StatusDiagnostics.unavailable(QuickSearchStatus.State.DEGRADED, "Status diagnostics could not be read.", homeHref);
+      }
+      expiresAt = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+      return (cached);
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop the bounded worker when its runtime closes.
+    *******************************************************************************/
+   @Override
+   public void close()
+   {
+      executor.shutdownNow();
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusDiagnostics.java

@@ -0,0 +1,249 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.State;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Cheap projection of existing diagnostics: one health request, bounded metadata
+ ** reads, saved IndexValidator counts. No source scans or per-table remote calls.
+ *******************************************************************************/
+public final class StatusDiagnostics
+{
+   public static final int MAX_TABLES = 100;
+   public static final int MAX_BACKLOG = 1000;
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private StatusDiagnostics()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Read as the trusted diagnostics worker after the caller's admin check.
+    *******************************************************************************/
+   public static QuickSearchStatus read(QuickSearchRuntime runtime) throws Exception
+   {
+      String homeHref = "/app/" + runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
+      List<QRecord> rows = query(runtime.getConfig().getQuickSearchIndexTableName(), new QQueryFilter()
+         .withOrderBy(new QFilterOrderBy("id")).withLimit(MAX_TABLES + 1));
+      boolean truncated = rows.size() > MAX_TABLES;
+      QRecord validation = DiagnosticsSupport.latestValidation(runtime.getConfig());
+      String json = validation == null ? null : validation.getValueString("reportJson");
+      JsonNode report = json == null || json.length() > 1_000_000 ? MAPPER.createObjectNode() : MAPPER.readTree(json);
+      Instant countsAt = validation == null ? null : validation.getValueInstant("endTime");
+      List<IndexRow> indexes = new ArrayList<>();
+      Instant lastSync = null;
+      String lastError = runtime.getLastRecordSearchFailureMessage();
+      Instant lastErrorAt = runtime.getLastRecordSearchFailureTime();
+      boolean degraded = truncated || runtime.isRecordSearchBackingOff() || (rows.isEmpty() && !runtime.getCodeTables().isEmpty());
+      for(QRecord row : rows.subList(0, Math.min(rows.size(), MAX_TABLES)))
+      {
+         String tableName = row.getValueString("tableName");
+         Long documents = row.getValueLong("documentCount");
+         Long source = null;
+         Instant measuredAt = null;
+         for(JsonNode stats : report.path("tableStats"))
+         {
+            if(tableName.equals(stats.path("tableName").asText()))
+            {
+               documents = stats.path("indexCount").isNumber() ? stats.path("indexCount").longValue() : null;
+               source = stats.path("databaseCount").isNumber() ? stats.path("databaseCount").longValue() : null;
+               measuredAt = countsAt;
+               break;
+            }
+         }
+         Instant sync = latest(row.getValueInstant("lastBasepullTime"), latest(row.getValueInstant("lastFullReindexTime"), row.getValueInstant("lastReconcileTime")));
+         lastSync = latest(lastSync, sync);
+         String error = row.getValueString("lastErrorMessage");
+         if(error != null && !error.isBlank() && (lastErrorAt == null || (row.getValueInstant("modifyDate") != null && row.getValueInstant("modifyDate").isAfter(lastErrorAt))))
+         {
+            lastError = error;
+            lastErrorAt = row.getValueInstant("modifyDate");
+         }
+         boolean enabled = !Boolean.FALSE.equals(row.getValueBoolean("enabled"));
+         Integer realTimeErrors = row.getValueInteger("realTimeErrorCount");
+         degraded |= enabled && (!"ACTIVE".equals(row.getValueString("status")) || "FAILED".equals(row.getValueString("lastRunStatus"))
+            || (realTimeErrors != null && realTimeErrors > 0) || (source != null && documents != null && !source.equals(documents)));
+         indexes.add(new IndexRow(tableName, enabled, row.getValueString("status"), documents, source, measuredAt, sync, error, realTimeErrors));
+      }
+      List<QRecord> backlog = query(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME),
+         new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("PENDING", "EXHAUSTED", "AWAITING_REINDEX")))
+            .withOrderBy(new QFilterOrderBy("id")).withLimit(MAX_BACKLOG + 1));
+      int pending = 0;
+      int exhausted = 0;
+      for(QRecord row : backlog.subList(0, Math.min(backlog.size(), MAX_BACKLOG)))
+      {
+         if("EXHAUSTED".equals(row.getValueString("status")))
+         {
+            exhausted++;
+         }
+         else
+         {
+            pending++;
+         }
+      }
+      boolean paused = runtime.isIndexingPaused();
+      degraded |= paused || pending > 0 || exhausted > 0;
+      Boolean reachable = null;
+      String health = null;
+      Integer nodes = null;
+      Integer active = null;
+      Integer unassigned = null;
+      Integer unassignedReplicas = null;
+      Integer targetReplicas = null;
+      Boolean replicaSettingsFailed = null;
+      State state = degraded ? State.DEGRADED : State.OK;
+      if(!truncated && !indexes.isEmpty() && indexes.stream().noneMatch(row -> Boolean.TRUE.equals(row.enabled())))
+      {
+         state = State.DISABLED;
+      }
+      else
+      {
+         try
+         {
+            var response = new QuickSearchDiagnosticsOperations(runtime.getClient()).clusterHealth();
+            reachable = true;
+            QuickSearchDiagnosticsOperations.requireOk(response, "Cluster health unavailable");
+            JsonNode body = response.body();
+            health = body.path("status").asText("unknown");
+            nodes = number(body, "number_of_nodes");
+            active = number(body, "active_shards");
+            unassigned = number(body, "unassigned_shards");
+            unassignedReplicas = unassignedReplicaShards(body, unassigned);
+            var replicaStatus = runtime.getClient().getReplicaStatus();
+            if(replicaStatus != null)
+            {
+               targetReplicas = replicaStatus.targetReplicas();
+               replicaSettingsFailed = replicaStatus.failed();
+            }
+            boolean singleNodeReplicas = QuickSearchStatus.isSingleNodeReplicaYellow(health, nodes, unassignedReplicas);
+            if((IndexValidator.healthLevel(health) != DiagnosticsHtml.Level.OK && !singleNodeReplicas) || body.path("timed_out").asBoolean())
+            {
+               state = "red".equals(health) ? State.DOWN : State.DEGRADED;
+            }
+         }
+         catch(QException e)
+         {
+            if(reachable == null)
+            {
+               reachable = false;
+            }
+            state = State.DOWN;
+            lastError = "OpenSearch health is unavailable; see Quick Search diagnostics.";
+         }
+      }
+      return (new QuickSearchStatus(state, reachable, health, nodes, active, unassigned, unassignedReplicas, targetReplicas, replicaSettingsFailed,
+         paused, indexes, pending, exhausted, backlog.size() > MAX_BACKLOG,
+         truncated, lastSync, lastError, Instant.now(), homeHref));
+   }
+
+
+
+   /*******************************************************************************
+    ** Unknown measurements for disabled or failed snapshots.
+    *******************************************************************************/
+   public static QuickSearchStatus unavailable(State state, String error, String homeHref)
+   {
+      return (new QuickSearchStatus(state, null, null, null, null, null, null, List.of(), null, null, false, false, null, error, Instant.now(), homeHref));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bounded metadata queries; failures propagate rather than looking healthy.
+    *******************************************************************************/
+   private static List<QRecord> query(String tableName, QQueryFilter filter) throws QException
+   {
+      return (CollectionUtils.nonNullList(new QueryAction().execute(new QueryInput(tableName).withFilter(filter)).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** From index-level cluster health: the unassigned shards when nothing is
+    ** initializing and every index has all its primaries active (so every
+    ** unassigned shard is a replica); null otherwise or when unknown.
+    *******************************************************************************/
+   static Integer unassignedReplicaShards(JsonNode body, Integer unassigned)
+   {
+      if(unassigned == null || !body.path("indices").isObject() || !Integer.valueOf(0).equals(number(body, "initializing_shards")))
+      {
+         return (null);
+      }
+      for(JsonNode index : body.path("indices"))
+      {
+         Integer primaries = number(index, "number_of_shards");
+         if(primaries == null || !primaries.equals(number(index, "active_primary_shards")))
+         {
+            return (null);
+         }
+      }
+      return (unassigned);
+   }
+
+
+
+   /*******************************************************************************
+    ** Preserve unknown counters from incomplete cluster responses.
+    *******************************************************************************/
+   private static Integer number(JsonNode body, String field)
+   {
+      return (body.path(field).isNumber() ? body.path(field).intValue() : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Newest non-null timestamp.
+    *******************************************************************************/
+   private static Instant latest(Instant first, Instant second)
+   {
+      return (first == null ? second : second == null || first.isAfter(second) ? first : second);
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecks.java

@@ -0,0 +1,290 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.health;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
+import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthDetails;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+
+
+/*******************************************************************************
+ ** Central health projections of the existing, shared diagnostic snapshot.
+ ** Free-form errors and remote strings never cross this boundary.
+ *******************************************************************************/
+public class QuickSearchHealthChecks implements HealthCheckSupplier
+{
+   /*******************************************************************************
+    ** Register once, even if the producer is run again on the same instance.
+    *******************************************************************************/
+   public static void register(QInstance instance)
+   {
+      HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance);
+      if(registry.getChecks().stream().noneMatch(check -> "quick_search.opensearch".equals(check.getName())))
+      {
+         registry.withCheck(definition("quick_search.opensearch", input -> run(input, QuickSearchHealthChecks::opensearch)));
+      }
+      if(registry.getChecks().stream().noneMatch(check -> "quick_search.backlog".equals(check.getName())))
+      {
+         registry.withCheck(definition("quick_search.backlog", input -> run(input, QuickSearchHealthChecks::backlog)));
+      }
+      if(registry.getSuppliers().stream().noneMatch(supplier -> "quick_search.index".equals(supplier.getNamePrefix())))
+      {
+         registry.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix("quick_search.index").withComponent("quick_search")
+            .withSupplier(new QCodeReference(QuickSearchHealthChecks.class)));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Enumerate persisted index identities, including disabled indexes. Execution
+    ** reads a fresh cached snapshot, never a row captured at supplier resolution.
+    *******************************************************************************/
+   @Override
+   public List<HealthCheckMetaData> supply(QInstance instance) throws QException
+   {
+      QuickSearchRuntime runtime = runtime(instance);
+      if(runtime == null)
+      {
+         return (List.of());
+      }
+      List<HealthCheckMetaData> checks = new ArrayList<>();
+      for(QRecord row : QuickSearchConfigPrimer.queryAll(runtime.getConfig().getQuickSearchIndexTableName()))
+      {
+         String tableName = row.getValueString("tableName");
+         String name = tableName.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]+", "_").replaceAll("[._-]{2,}", "_").replaceAll("^[._-]|[._-]$", "");
+         if(name.isEmpty())
+         {
+            throw (new QException("Index name cannot form a health check identifier"));
+         }
+         checks.add(definition("quick_search.index." + name, input -> run(input, status -> index(status, tableName))));
+      }
+      return (checks);
+   }
+
+
+
+   /*******************************************************************************
+    ** All Quick Search checks are dependencies; none gate Kubernetes readiness.
+    *******************************************************************************/
+   private static HealthCheckMetaData definition(String name, HealthCheck check)
+   {
+      return (new HealthCheckMetaData().withName(name).withLabel(name).withComponent("quick_search")
+         .withCategory(HealthCategory.SEARCH).withKind(HealthCheckKind.DEPENDENCY)
+         .withIntervalSeconds(60).withTimeoutMs(5000).withCheckInstance(check));
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve from the supplied instance, not a static or previously closed runtime.
+    *******************************************************************************/
+   private static QuickSearchRuntime runtime(QInstance instance)
+   {
+      if(instance != null && instance.getQBits() != null)
+      {
+         for(QBitMetaData qbit : instance.getQBits().values())
+         {
+            if(qbit.getConfig() instanceof QuickSearchQBitConfig config)
+            {
+               return (config.getRuntime());
+            }
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Share the diagnostic worker and respect the deadline even on a cache hit.
+    ** Expected diagnostic failures use fixed messages, never exception payloads.
+    *******************************************************************************/
+   private static HealthCheckOutcome run(HealthCheckInput input, Function<QuickSearchStatus, HealthCheckOutcome> project)
+   {
+      if(!Instant.now().isBefore(input.getDeadline()))
+      {
+         return (HealthCheckOutcome.down("Quick Search health deadline exceeded"));
+      }
+      QuickSearchRuntime runtime = runtime(input.getQInstance());
+      if(runtime == null)
+      {
+         return (HealthCheckOutcome.unknown("Quick Search is not configured"));
+      }
+      try
+      {
+         QuickSearchStatus status = runtime.getStatusCache().get(runtime, input.getDeadline());
+         if(!Instant.now().isBefore(input.getDeadline()))
+         {
+            return (HealthCheckOutcome.down("Quick Search health deadline exceeded"));
+         }
+         return (project.apply(status));
+      }
+      catch(QException e)
+      {
+         return (HealthCheckOutcome.down("Quick Search diagnostics unavailable"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health is independent of backlog and per-index degradation.
+    *******************************************************************************/
+   private static HealthCheckOutcome opensearch(QuickSearchStatus status)
+   {
+      if(status.state() == QuickSearchStatus.State.DISABLED)
+      {
+         return (HealthCheckOutcome.unknown("Quick Search is disabled"));
+      }
+      HealthCheckOutcome outcome;
+      if(status.state() == QuickSearchStatus.State.DOWN || Boolean.FALSE.equals(status.clusterReachable()) || "red".equals(status.clusterHealth()))
+      {
+         outcome = HealthCheckOutcome.down("OpenSearch is unavailable or red");
+      }
+      else if(Boolean.TRUE.equals(status.clusterReachable()) && "green".equals(status.clusterHealth()))
+      {
+         outcome = HealthCheckOutcome.up("OpenSearch is green");
+      }
+      else if(Boolean.TRUE.equals(status.clusterReachable()) && QuickSearchStatus.isSingleNodeReplicaYellow(status.clusterHealth(), status.nodes(), status.unassignedReplicaShards()))
+      {
+         outcome = HealthCheckOutcome.up("OpenSearch is yellow only because replicas cannot be assigned on a single node");
+         outcome.getDetails().withValue("note", "replicas cannot be assigned on a single-node cluster");
+      }
+      else
+      {
+         outcome = HealthCheckOutcome.degraded("OpenSearch health is yellow or incomplete");
+      }
+      detail(outcome, "reachable", status.clusterReachable());
+      detail(outcome, "nodes", status.nodes());
+      detail(outcome, "activeShards", status.activeShards());
+      detail(outcome, "unassignedShards", status.unassignedShards());
+      detail(outcome, "unassignedReplicaShards", status.unassignedReplicaShards());
+      detail(outcome, "targetReplicas", status.targetReplicas());
+      detail(outcome, "replicaSettingsFailed", status.replicaSettingsFailed());
+      return (outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts are lower bounds when truncated; do not declare an incomplete backlog healthy.
+    *******************************************************************************/
+   private static HealthCheckOutcome backlog(QuickSearchStatus status)
+   {
+      HealthCheckOutcome outcome;
+      if(status.state() == QuickSearchStatus.State.DISABLED)
+      {
+         outcome = HealthCheckOutcome.unknown("Quick Search is disabled");
+      }
+      else if(status.pending() == null || status.exhausted() == null || status.indexingPaused() == null)
+      {
+         outcome = HealthCheckOutcome.degraded("Indexing backlog measurements unavailable");
+      }
+      else if(status.indexingPaused() || status.exhausted() > 0 || status.pending() > 100 || Boolean.TRUE.equals(status.backlogTruncated()))
+      {
+         outcome = HealthCheckOutcome.degraded("Indexing is paused or backlog needs attention");
+      }
+      else
+      {
+         outcome = HealthCheckOutcome.up("Indexing backlog is within limits");
+      }
+      detail(outcome, "pending", status.pending());
+      detail(outcome, "exhausted", status.exhausted());
+      detail(outcome, "indexingPaused", status.indexingPaused());
+      detail(outcome, "backlogTruncated", status.backlogTruncated());
+      return (outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Missing rows (including those outside the bounded snapshot) stay unknown.
+    ** Persisted error text is represented only by a boolean.
+    *******************************************************************************/
+   private static HealthCheckOutcome index(QuickSearchStatus status, String tableName)
+   {
+      IndexRow row = status.indexes().stream().filter(candidate -> tableName.equals(candidate.tableName())).findFirst().orElse(null);
+      if(row == null)
+      {
+         return (HealthCheckOutcome.unknown("Index measurements unavailable"));
+      }
+      boolean hasError = row.lastError() != null && !row.lastError().isBlank();
+      boolean mismatch = row.documentCount() != null && row.sourceCount() != null && !row.documentCount().equals(row.sourceCount());
+      HealthCheckOutcome outcome;
+      if(Boolean.FALSE.equals(row.enabled()))
+      {
+         outcome = HealthCheckOutcome.unknown("Index is disabled");
+      }
+      else if(!"ACTIVE".equals(row.status()) || hasError || mismatch || (row.realTimeErrorCount() != null && row.realTimeErrorCount() > 0))
+      {
+         outcome = HealthCheckOutcome.degraded("Index needs attention");
+      }
+      else
+      {
+         outcome = HealthCheckOutcome.up("Index is active");
+      }
+      outcome.withDetails(new HealthDetails().withValue("hasError", hasError).withValue("countMismatch", mismatch));
+      detail(outcome, "enabled", row.enabled());
+      detail(outcome, "documentCount", row.documentCount());
+      detail(outcome, "sourceCount", row.sourceCount());
+      detail(outcome, "countsCheckedAt", row.countsCheckedAt());
+      detail(outcome, "lastSuccessfulSync", row.lastSuccessfulSync());
+      detail(outcome, "realTimeErrorCount", row.realTimeErrorCount());
+      return (outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Unknown measurements are omitted rather than converted to zero.
+    *******************************************************************************/
+   private static void detail(HealthCheckOutcome outcome, String name, Object value)
+   {
+      if(value != null)
+      {
+         outcome.getDetails().withValue(name, value);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java

@@ -0,0 +1,605 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.listeners;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Real-time indexing hook. Registered once per QInstance; applies to the
+ ** tables this QBit indexes.
+ **
+ ** Events are built while the write's transaction is open, but published only
+ ** after that transaction commits, via
+ ** QBackendTransaction.addAfterCommitCallback. A write without a transaction
+ ** is already committed, so it publishes immediately. A rolled-back write never
+ ** reaches the index.
+ **
+ ** Inserted and updated records are re-read inside that transaction, with
+ ** display values and possible-value labels, so a real-time document matches
+ ** the one basepull, reconcile and full reindex build. The event's own records
+ ** are not enough: for an update they are the sparse input, and for an insert
+ ** they carry only raw values.
+ **
+ ** This listener never fails the host's write. When the re-read of inserted or
+ ** updated records or publishing fails (including a LinkageError from a missing
+ ** optional library while the client is built), each affected (table, record,
+ ** action) is written to quick_search_failed_event for the scheduled basepull to
+ ** replay. A failed re-read is recorded after the commit too, so a rolled-back
+ ** write leaves no row.
+ **
+ ** While a full reindex rebuilds into a new physical index, the alias still
+ ** points at the old one, so a published delete never reaches the new index.
+ ** Such deletes are also written to quick_search_failed_event with status
+ ** AWAITING_REINDEX; the reindex applies them after the alias swap.
+ *******************************************************************************/
+public class QuickSearchRecordChangeListener implements RecordChangeListenerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordChangeListener.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean appliesTo(String tableName, RecordChangeType type)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime != null
+         && Boolean.TRUE.equals(runtime.getConfig().getEnableRealTimeIndexing())
+         && !QuickSearchFieldRules.isQuickSearchTable(runtime.getConfig(), tableName)
+         && runtime.getTableConfig(tableName) != null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void onRecordsChanged(RecordChangeEvent event) throws QException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || event == null || CollectionUtils.nullSafeIsEmpty(event.getRecords()))
+      {
+         return;
+      }
+
+      String                     tableName   = event.getTableName();
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null || !runtime.isTableEnabled(tableName))
+      {
+         return;
+      }
+
+      String primaryKeyField = tableConfig.getPrimaryKeyField() != null ? tableConfig.getPrimaryKeyField() : "id";
+      if(QuickSearchFieldRules.lookupTable(tableName) == null || QuickSearchFieldRules.isProtected(QuickSearchFieldRules.lookupTable(tableName), primaryKeyField))
+      {
+         QuickSearchFieldRules.warnExcludedFields(QuickSearchFieldRules.lookupTable(tableName));
+         return;
+      }
+
+      ///////////////////////////////////////////////////////////////////////////
+      // paused: index nothing, but keep the written keys (DEFERRED), which     //
+      // resuming hands to the basepull; search keeps serving the old documents //
+      ///////////////////////////////////////////////////////////////////////////
+      if(runtime.isIndexingPaused())
+      {
+         List<IndexEvent> deferred = event.getType() == RecordChangeType.DELETE
+            ? toDeleteEvents(tableName, primaryKeyField, event.getRecords())
+            : toIndexEvents(tableName, primaryKeyField, event.getRecords());
+         runAfterCommit(event, () -> recordDeferred(runtime, deferred));
+         return;
+      }
+
+      List<IndexEvent> indexEvents  = new ArrayList<>();
+      List<IndexEvent> deleteEvents = new ArrayList<>();
+
+      switch(event.getType())
+      {
+         case INSERT, UPDATE ->
+         {
+            try
+            {
+               indexEvents.addAll(toIndexEvents(tableName, primaryKeyField, fetchCurrentRecords(event, primaryKeyField)));
+            }
+            catch(Exception | LinkageError e)
+            {
+               ///////////////////////////////////////////////////////////////////////
+               // log now: with a transaction, nothing else is logged until commit, //
+               // and a rollback (a transaction the failed re-read aborted, too)    //
+               // would leave no trace. Then record the written keys after commit,  //
+               // so basepull re-reads and indexes them.                            //
+               ///////////////////////////////////////////////////////////////////////
+               LOG.warn("Could not re-read " + (event.getType() == RecordChangeType.INSERT ? "inserted" : "updated")
+                     + " records for indexing; recording them for replay once the write commits", e,
+                  logPair("tableName", tableName), logPair("recordCount", event.getRecords().size()));
+               List<IndexEvent> failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords());
+               runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e));
+               return;
+            }
+         }
+         case DELETE -> deleteEvents.addAll(toDeleteEvents(tableName, primaryKeyField, event.getRecords()));
+      }
+
+      if(indexEvents.isEmpty() && deleteEvents.isEmpty())
+      {
+         return;
+      }
+
+      runAfterCommit(event, () -> publish(runtime, indexEvents, deleteEvents));
+   }
+
+
+
+   /*******************************************************************************
+    ** Run work once the event's transaction commits, or now when the write had
+    ** no transaction (it is already committed).
+    *******************************************************************************/
+   private static void runAfterCommit(RecordChangeEvent event, Runnable work)
+   {
+      if(event.getTransaction() != null)
+      {
+         event.getTransaction().addAfterCommitCallback(work);
+      }
+      else
+      {
+         work.run();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Publish, recording any failure durably, and capture the deletes if a full
+    ** reindex is rebuilding the table. Never throws.
+    **
+    ** The status is read before the deletes are published, never after: a
+    ** reindex can swap the alias and clear REBUILDING while the publish is in
+    ** flight, and a read after that would miss a delete that reached only the
+    ** old index. A read before the publish that sees no REBUILDING means either
+    ** the reindex has not started reading the table (the record, deleted and
+    ** committed, cannot be read into the new index) or the alias already points
+    ** at the new index (the publish reaches it).
+    *******************************************************************************/
+   static void publish(QuickSearchRuntime runtime, List<IndexEvent> indexEvents, List<IndexEvent> deleteEvents)
+   {
+      boolean rebuilding = isRebuildingBeforePublish(runtime, deleteEvents);
+      publishOrRecordFailures(runtime, indexEvents, deleteEvents);
+      if(rebuilding)
+      {
+         captureDeletesDuringRebuild(runtime, deleteEvents);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A LinkageError is caught along with Exception: building the lazy client,
+    ** or a host publisher, can hit an optional library that is not on the
+    ** classpath (the AWS SDK for AWS_SIGV4), which a DEGRADED start leaves for
+    ** the first write to find.
+    *******************************************************************************/
+   private static void publishOrRecordFailures(QuickSearchRuntime runtime, List<IndexEvent> indexEvents, List<IndexEvent> deleteEvents)
+   {
+      IndexEventPublisher publisher;
+      try
+      {
+         publisher = runtime.getPublisher();
+      }
+      catch(Exception | LinkageError e)
+      {
+         recordFailures(runtime, indexEvents, e);
+         recordFailures(runtime, deleteEvents, e);
+         return;
+      }
+
+      if(!indexEvents.isEmpty())
+      {
+         try
+         {
+            publisher.publishIndexEvents(indexEvents);
+         }
+         catch(Exception | LinkageError e)
+         {
+            recordFailures(runtime, indexEvents, e);
+         }
+      }
+
+      if(!deleteEvents.isEmpty())
+      {
+         try
+         {
+            publisher.publishDeleteEvents(deleteEvents);
+         }
+         catch(Exception | LinkageError e)
+         {
+            recordFailures(runtime, deleteEvents, e);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Write one quick_search_failed_event row per event. Never throws.
+    *******************************************************************************/
+   static void recordFailures(QuickSearchRuntime runtime, List<IndexEvent> events, Throwable cause)
+   {
+      if(events.isEmpty())
+      {
+         return;
+      }
+
+      String message = cause.getMessage() == null ? cause.getClass().getSimpleName() : cause.getMessage();
+      LOG.warn("Real-time indexing failed; recording failed events for replay", cause,
+         logPair("tableName", events.get(0).getTableName()), logPair("eventCount", events.size()));
+
+      try
+      {
+         if(!hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)))
+         {
+            LOG.error("quick_search_failed_event table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
+            return;
+         }
+
+         insertFailedEventRows(runtime, events, message, QuickSearchFailedEvent.STATUS_PENDING);
+         incrementRealTimeErrorCounts(runtime, events);
+      }
+      catch(Exception e)
+      {
+         LOG.error("Could not record failed index events", e, logPair("eventCount", events.size()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** QQQ UpdateInput has no arithmetic update. Serialize this read-modify-write
+    ** across listeners in this JVM, after the source transaction has committed.
+    ** Counter failures must not prevent replay or fail the source write.
+    *******************************************************************************/
+   private static synchronized void incrementRealTimeErrorCounts(QuickSearchRuntime runtime, List<IndexEvent> events)
+   {
+      Map<String, Integer> counts = new HashMap<>();
+      for(IndexEvent event : events)
+      {
+         counts.merge(event.getTableName(), 1, Integer::sum);
+      }
+
+      for(Map.Entry<String, Integer> entry : counts.entrySet())
+      {
+         try
+         {
+            String     indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(indexTable);
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, entry.getKey())).withLimit(1));
+            List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+            if(CollectionUtils.nullSafeIsEmpty(rows))
+            {
+               continue;
+            }
+
+            QRecord row      = rows.get(0);
+            Integer previous = row.getValueInteger("realTimeErrorCount");
+            UpdateInput updateInput = new UpdateInput();
+            updateInput.setTableName(indexTable);
+            updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id"))
+               .withValue("realTimeErrorCount", Math.addExact(previous == null ? 0 : previous, entry.getValue()))));
+            // A runtime counter, not a configuration change: not record-audited.
+            updateInput.setOmitDmlAudit(true);
+            List<QRecord> updated = QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)).getRecords();
+            if(CollectionUtils.nullSafeIsEmpty(updated) || CollectionUtils.nullSafeHasContents(updated.get(0).getErrors()))
+            {
+               throw (new QException("Counter update returned errors"));
+            }
+         }
+         catch(Exception | LinkageError e)
+         {
+            LOG.warn("Could not increment real-time indexing error count", e, logPair("tableName", entry.getKey()));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Keep changes made while indexing is paused, for the basepull to replay
+    ** when it resumes. Never throws.
+    *******************************************************************************/
+   static void recordDeferred(QuickSearchRuntime runtime, List<IndexEvent> events)
+   {
+      if(events.isEmpty())
+      {
+         return;
+      }
+      try
+      {
+         if(hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)))
+         {
+            insertFailedEventRows(runtime, events, "deferred: indexing was paused", QuickSearchFailedEvent.STATUS_DEFERRED);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not keep changes made while indexing is paused; the next reconcile covers them", e, logPair("eventCount", events.size()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a full reindex is rebuilding the deletes' table. The status is
+    ** read from the database (not cached) so a reindex on another node is seen.
+    ** Runs after the delete committed. Never throws.
+    *******************************************************************************/
+   static boolean isRebuildingBeforePublish(QuickSearchRuntime runtime, List<IndexEvent> deleteEvents)
+   {
+      if(deleteEvents.isEmpty())
+      {
+         return (false);
+      }
+
+      String tableName = deleteEvents.get(0).getTableName();
+      try
+      {
+         return (hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
+      }
+      catch(Exception e)
+      {
+         LOG.error("Could not read whether a full reindex is running; deletes made during one are not captured", e, logPair("tableName", tableName));
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Record deletes made while a full reindex is rebuilding the table, so the
+    ** reindex can apply them to the new physical index after the alias swap.
+    ** basepull does not replay AWAITING_REINDEX rows, so it cannot consume one
+    ** against the old index before the swap. Never throws.
+    *******************************************************************************/
+   static void captureDeletesDuringRebuild(QuickSearchRuntime runtime, List<IndexEvent> deleteEvents)
+   {
+      String tableName = deleteEvents.get(0).getTableName();
+      try
+      {
+         List<QRecord> inserted = insertFailedEventRows(runtime, deleteEvents, "deleted during a full reindex", QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
+         handOverIfRebuildFinished(runtime, tableName, inserted);
+      }
+      catch(Exception e)
+      {
+         LOG.error("Could not capture deletes made during a full reindex; the next full reindex or reconcile removes them", e,
+            logPair("tableName", tableName), logPair("eventCount", deleteEvents.size()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The reindex may have finished, and drained its captured deletes, between
+    ** the status read and the insert. Read the status again: if the table is no
+    ** longer REBUILDING, make the rows PENDING so basepull replays them against
+    ** the current index (deletes are idempotent). If it still is, the reindex
+    ** has not drained yet and will see these rows.
+    *******************************************************************************/
+   static void handOverIfRebuildFinished(QuickSearchRuntime runtime, String tableName, List<QRecord> inserted) throws QException
+   {
+      if(inserted.isEmpty() || isRebuilding(runtime, tableName))
+      {
+         return;
+      }
+
+      List<QRecord> updates = new ArrayList<>();
+      for(QRecord row : inserted)
+      {
+         updates.add(new QRecord().withValue("id", row.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING));
+      }
+
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
+      updateInput.setRecords(updates);
+      new UpdateAction().execute(updateInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the table's quick_search_index row is REBUILDING.
+    *******************************************************************************/
+   private static boolean isRebuilding(QuickSearchRuntime runtime, String tableName) throws QException
+   {
+      String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+      if(!hasTable(indexTable))
+      {
+         return (false);
+      }
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(indexTable);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)).withLimit(1));
+      List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeHasContents(rows) && AbstractIndexingStep.STATUS_REBUILDING.equals(rows.get(0).getValueString("status")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one quick_search_failed_event row per event; returns the inserted rows.
+    *******************************************************************************/
+   static List<QRecord> insertFailedEventRows(QuickSearchRuntime runtime, List<IndexEvent> events, String message, String status) throws QException
+   {
+      List<QRecord> rows = new ArrayList<>();
+      for(IndexEvent event : events)
+      {
+         rows.add(new QRecord()
+            .withValue("tableName", event.getTableName())
+            .withValue("recordId", event.getRecordId())
+            .withValue("action", event.getAction() == null ? null : event.getAction().name())
+            .withValue("errorMessage", truncate(message, 4000))
+            .withValue("attempts", 0)
+            .withValue("status", status));
+      }
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
+      insertInput.setRecords(rows);
+      return (CollectionUtils.nonNullList(new InsertAction().execute(insertInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean hasTable(String tableName)
+   {
+      return (QContext.getQInstance() != null && QContext.getQInstance().getTable(tableName) != null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<IndexEvent> toIndexEvents(String tableName, String primaryKeyField, List<QRecord> records)
+   {
+      List<IndexEvent> events = new ArrayList<>();
+      for(QRecord record : CollectionUtils.nonNullList(records))
+      {
+         String recordId = record.getValueString(primaryKeyField);
+         if(recordId == null || CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            continue;
+         }
+         QRecord safeRecord = new QRecord();
+         for(var field : QuickSearchFieldRules.indexableFields(QuickSearchFieldRules.lookupTable(tableName)))
+         {
+            safeRecord.setValue(field.getName(), record.getValue(field.getName()));
+            safeRecord.setDisplayValue(field.getName(), record.getDisplayValue(field.getName()));
+         }
+         if(!QuickSearchFieldRules.requiresFieldFiltering(tableName))
+         {
+            safeRecord.setRecordLabel(record.getRecordLabel());
+         }
+         events.add(new IndexEvent().withAction(IndexEventAction.INDEX).withTableName(tableName).withRecordId(recordId).withRecord(safeRecord));
+      }
+      return (events);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<IndexEvent> toDeleteEvents(String tableName, String primaryKeyField, List<QRecord> records)
+   {
+      List<IndexEvent> events = new ArrayList<>();
+      for(QRecord record : CollectionUtils.nonNullList(records))
+      {
+         String recordId = record.getValueString(primaryKeyField);
+         if(recordId == null || CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            continue;
+         }
+         events.add(new IndexEvent().withAction(IndexEventAction.DELETE).withTableName(tableName).withRecordId(recordId));
+      }
+      return (events);
+   }
+
+
+
+   /*******************************************************************************
+    ** The event's records are "as the backend returned them": for an update on
+    ** RDBMS that is the sparse input, and for an insert the raw values with no
+    ** display values. Re-read the full rows inside the same transaction, with
+    ** display values so possible-value labels are indexed (which can run host
+    ** possible-value providers).
+    *******************************************************************************/
+   private static List<QRecord> fetchCurrentRecords(RecordChangeEvent event, String primaryKeyField) throws QException
+   {
+      List<Serializable> primaryKeys = new ArrayList<>();
+      for(QRecord record : event.getRecords())
+      {
+         Serializable primaryKey = record.getValue(primaryKeyField);
+         if(primaryKey != null && CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+         {
+            primaryKeys.add(primaryKey);
+         }
+      }
+
+      if(primaryKeys.isEmpty())
+      {
+         return (List.of());
+      }
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(event.getTableName());
+      queryInput.setTransaction(event.getTransaction());
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, primaryKeys)));
+
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String truncate(String s, int max)
+   {
+      return (s == null || s.length() <= max ? s : s.substring(0, max));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchAdminAppMetaDataProducer.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchAdmin app from the tables the other component
+ ** producers added to the production output. Processes are not app children
+ ** (never in the nav): they run from their tables' Actions. Runs last.
+ *******************************************************************************/
+public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataProducer<QAppMetaData, QuickSearchQBitConfig>
+{
+   public static final String APP_NAME = "quickSearchAdmin";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QAppMetaData produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig config = getQBitConfig();
+
+      QAppMetaData app = new QAppMetaData()
+         .withName(config.applyPrefix(APP_NAME))
+         .withLabel("Quick Search Admin")
+         .withIcon(new QIcon("search"))
+         .withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+
+      MetaDataProducerMultiOutput produced = QBitProductionContext.peekMetaDataProducerMultiOutput();
+      if(produced == null)
+      {
+         return (app);
+      }
+
+      QWidgetMetaData diagnosticsWidget = produced.get(QWidgetMetaData.class, config.applyPrefix(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME));
+      if(diagnosticsWidget != null)
+      {
+         app.withWidgets(new ArrayList<>(List.of(diagnosticsWidget.getName())));
+      }
+
+      for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexField.TABLE_NAME, QuickSearchSettings.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME,
+         QuickSearchFailedEvent.TABLE_NAME, QuickSearchValidationRun.TABLE_NAME, QuickSearchDiagnosticEvent.TABLE_NAME, QuickSearchConfigTombstone.TABLE_NAME })
+      {
+         QTableMetaData table = produced.get(QTableMetaData.class, config.applyTablePrefix(tableName));
+         if(table != null)
+         {
+            app.withChild(table);
+         }
+      }
+
+      ////////////////////////////////////////////////////////////////////////
+      // processes are never in the nav: Basepull, Full Reindex, Reconcile   //
+      // and the config processes run from their tables' Actions             //
+      ////////////////////////////////////////////////////////////////////////
+      return (app);
+   }
+
+
+
+   /*******************************************************************************
+    ** last, so the tables and processes exist in the output
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (900);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchBasepullProcessMetaDataProducer.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the basepull process when enableBasepullProcess is true.
+ *******************************************************************************/
+public class QuickSearchBasepullProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchProcessMetaDataHelper.buildBasepullProcess(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableBasepullProcess()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigMetaDataHelper.java

@@ -0,0 +1,384 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchIndexFieldTableCustomizer;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchSettingsTableCustomizer;
+import com.kingsrook.qbits.quicksearch.config.QuickSearchTablePossibleValueProviders;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDocumentsChoice;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+
+
+/*******************************************************************************
+ ** Builds the data-driven config metadata (1.1.0-RC.3): the field, settings
+ ** and tombstone tables, the index-to-fields join and child-record widget,
+ ** and the possible-value sources the config screens use.
+ *******************************************************************************/
+public class QuickSearchConfigMetaDataHelper
+{
+   public static final String ELIGIBLE_TABLE_PVS   = "quick_search_eligible_table";
+   public static final String INDEXABLE_FIELD_PVS  = "quick_search_indexable_field";
+   public static final String TIMESTAMP_FIELD_PVS  = "quick_search_timestamp_field";
+   public static final String FIELD_ROLE_PVS       = "quick_search_field_role";
+   public static final String FIELD_ANALYZER_PVS   = "quick_search_field_analyzer";
+   public static final String DOCUMENTS_CHOICE_PVS = "quick_search_documents_choice";
+   public static final String CONFIG_SOURCE_PVS    = "quick_search_config_source";
+   public static final String INDEX_FIELDS_JOIN    = "quick_search_index_join_quick_search_index_field";
+   public static final String INDEX_FIELDS_WIDGET  = "quickSearchIndexFields";
+   public static final String FIELDS_ASSOCIATION   = "fields";
+   public static final String INDEX_RUNS_JOIN      = "quick_search_index_join_quick_search_index_run";
+   public static final String INDEX_RUNS_WIDGET    = "quickSearchIndexRuns";
+   public static final String RUN_TYPE_PVS         = "quick_search_run_type";
+   public static final String RUN_STATUS_PVS       = "quick_search_run_status";
+   public static final String EVENT_STATUS_PVS     = "quick_search_failed_event_status";
+   public static final String EVENT_ACTION_PVS     = "quick_search_failed_event_action";
+
+
+
+   /*******************************************************************************
+    ** quick_search_index_field: the searchable fields of each index row.
+    *******************************************************************************/
+   public static QTableMetaData buildIndexFieldTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
+         .withLabel("Quick Search Index Field")
+         .withIcon(new QIcon("text_fields"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("fieldName")
+         .withUniqueKey(new UniqueKey("quickSearchIndexId", "fieldName"))
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withIsRequired(true)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)))
+         .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withLabel("Field").withIsRequired(true)
+            .withPossibleValueSourceName(config.applyTablePrefix(INDEXABLE_FIELD_PVS)))
+         .withField(new QFieldMetaData("weight", QFieldType.INTEGER).withDefaultValue(1))
+         .withField(new QFieldMetaData("role", QFieldType.STRING).withDefaultValue(QuickSearchFieldRole.BODY.name())
+            .withPossibleValueSourceName(config.applyTablePrefix(FIELD_ROLE_PVS)))
+         .withField(new QFieldMetaData("analyzer", QFieldType.STRING).withLabel("Analyzer (blank: by role)")
+            .withPossibleValueSourceName(config.applyTablePrefix(FIELD_ANALYZER_PVS)))
+         .withField(new QFieldMetaData("includeInSnippet", QFieldType.BOOLEAN).withLabel("In Snippet (blank: by role)"))
+         .withField(new QFieldMetaData("includeLabel", QFieldType.BOOLEAN).withLabel("Index With Field Label").withDefaultValue(false))
+         .withField(new QFieldMetaData("sortOrder", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false).withPossibleValueSourceName(config.applyTablePrefix(CONFIG_SOURCE_PVS)))
+         .withField(new QFieldMetaData("ownerEdited", QFieldType.BOOLEAN).withLabel("Edited by Owner").withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Field", new QIcon("text_fields"), Tier.T1, List.of("id", "quickSearchIndexId", "fieldName", "weight", "role")))
+         .withSection(new QFieldSection("matching", "Matching", new QIcon("tune"), Tier.T2, List.of("analyzer", "includeInSnippet", "includeLabel", "sortOrder")))
+         .withSection(new QFieldSection("origin", "Origin", new QIcon("history"), Tier.T3, List.of("source", "ownerEdited", "createDate", "modifyDate"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_settings: the one row of system settings (the global pause
+    ** and the last priming report). Rows are never inserted or deleted by
+    ** users; the QBit creates the row when it primes.
+    *******************************************************************************/
+   public static QTableMetaData buildSettingsTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchSettings.TABLE_NAME))
+         .withLabel("Quick Search Settings")
+         .withIcon(new QIcon("settings"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
+         .withRecordLabelFormat("Quick Search Settings")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
+         .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchSettingsTableCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchSettingsTableCustomizer.class))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("indexingPaused", QFieldType.BOOLEAN).withLabel("Indexing Paused"))
+         .withField(new QFieldMetaData("pauseReason", QFieldType.STRING).withLabel("Pause Reason"))
+         .withField(new QFieldMetaData("pausedAt", QFieldType.DATE_TIME).withLabel("Paused Since").withIsEditable(false))
+         .withField(new QFieldMetaData("pausedBy", QFieldType.STRING).withLabel("Paused By").withIsEditable(false))
+         .withField(new QFieldMetaData("lastPrimedAt", QFieldType.DATE_TIME).withLabel("Last Primed").withIsEditable(false))
+         .withField(new QFieldMetaData("lastPrimingSummary", QFieldType.TEXT).withLabel("Last Priming").withIsEditable(false))
+         .withField(new QFieldMetaData("lastPrimingNewTables", QFieldType.INTEGER).withLabel("Tables Added by Last Priming").withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("pause", "Indexing", new QIcon("pause_circle"), Tier.T1, List.of("id", "indexingPaused", "pauseReason", "pausedAt", "pausedBy")))
+         .withSection(new QFieldSection("priming", "Self-priming", new QIcon("auto_mode"), Tier.T2, List.of("lastPrimedAt", "lastPrimingSummary", "lastPrimingNewTables")))
+         .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_config_tombstone: what an admin removed, so priming does
+    ** not add it back. Written by the QBit; deleting a row lets code defaults
+    ** seed that table or field again.
+    *******************************************************************************/
+   public static QTableMetaData buildTombstoneTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME))
+         .withLabel("Quick Search Removed Config")
+         .withIcon(new QIcon("delete_sweep"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("tableName", "fieldName")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
+         .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withLabel("Field (blank: whole table)").withIsEditable(false))
+         .withField(new QFieldMetaData("removedBy", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("reason", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Removed").withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_index to its field rows.
+    *******************************************************************************/
+   public static QJoinMetaData buildIndexFieldsJoin(QuickSearchQBitConfig config)
+   {
+      return (new QJoinMetaData()
+         .withName(config.applyTablePrefix(INDEX_FIELDS_JOIN))
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
+         .withRightTable(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
+         .withJoinOn(new JoinOn("id", "quickSearchIndexId"))
+         .withOrderBy(new QFilterOrderBy("sortOrder"))
+         .withOrderBy(new QFilterOrderBy("id")));
+   }
+
+
+
+   /*******************************************************************************
+    ** The field rows on the index record view, where fields are added,
+    ** edited and removed.
+    *******************************************************************************/
+   public static QWidgetMetaData buildIndexFieldsWidget(QuickSearchQBitConfig config)
+   {
+      QWidgetMetaData widget = ChildRecordListRenderer.widgetMetaDataBuilder(buildIndexFieldsJoin(config))
+         .withName(config.applyPrefix(INDEX_FIELDS_WIDGET))
+         .withLabel("Searchable Fields")
+         .withCanAddChildRecord(true)
+         .withDisabledFieldsForNewChildRecords(Set.of("quickSearchIndexId"))
+         .withDefaultValuesForNewChildRecordsFromParentFields(Map.of("quickSearchIndexId", "id"))
+         .withOnlyIncludeFieldNames(List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel", "source"))
+         .withMaxRows(100)
+         .getWidgetMetaData();
+      return (widget);
+   }
+
+
+
+   /*******************************************************************************
+    ** quick_search_index to its run history, newest first.
+    *******************************************************************************/
+   public static QJoinMetaData buildIndexRunsJoin(QuickSearchQBitConfig config)
+   {
+      return (new QJoinMetaData()
+         .withName(config.applyTablePrefix(INDEX_RUNS_JOIN))
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
+         .withRightTable(config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME))
+         .withJoinOn(new JoinOn("id", "quickSearchIndexId"))
+         .withOrderBy(new QFilterOrderBy("id", false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The index record's recent runs (read only).
+    *******************************************************************************/
+   public static QWidgetMetaData buildIndexRunsWidget(QuickSearchQBitConfig config)
+   {
+      return (ChildRecordListRenderer.widgetMetaDataBuilder(buildIndexRunsJoin(config))
+         .withName(config.applyPrefix(INDEX_RUNS_WIDGET))
+         .withLabel("Recent Runs")
+         .withCanAddChildRecord(false)
+         .withOnlyIncludeFieldNames(List.of("id", "runType", "status", "startTime", "endTime", "recordsProcessed", "recordsIndexed", "errorCount"))
+         .withMaxRows(10)
+         .getWidgetMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    ** The custom possible-value sources over the instance's tables and fields.
+    *******************************************************************************/
+   public static List<QPossibleValueSource> buildCustomPossibleValueSources(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         custom(config.applyTablePrefix(ELIGIBLE_TABLE_PVS), QuickSearchTablePossibleValueProviders.EligibleTables.class),
+         custom(config.applyTablePrefix(INDEXABLE_FIELD_PVS), QuickSearchTablePossibleValueProviders.IndexableFields.class),
+         custom(config.applyTablePrefix(TIMESTAMP_FIELD_PVS), QuickSearchTablePossibleValueProviders.TimestampFields.class)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The enum possible-value sources (role, analyzer, documents choice,
+    ** config source).
+    *******************************************************************************/
+   public static List<QPossibleValueSource> buildEnumPossibleValueSources(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         QPossibleValueSource.newForEnum(config.applyTablePrefix(FIELD_ROLE_PVS), QuickSearchFieldRole.values()),
+         QPossibleValueSource.newForEnum(config.applyTablePrefix(FIELD_ANALYZER_PVS), QuickSearchFieldAnalyzer.values()),
+         QPossibleValueSource.newForEnum(config.applyTablePrefix(DOCUMENTS_CHOICE_PVS), QuickSearchDocumentsChoice.values()),
+         stringEnum(config.applyTablePrefix(RUN_TYPE_PVS), "BASEPULL", "Update (basepull)", "RECONCILE", "Reconcile", "FULL_REINDEX", "Full reindex / build"),
+         stringEnum(config.applyTablePrefix(RUN_STATUS_PVS), "RUNNING", "Running", "COMPLETED", "Completed", "FAILED", "Failed", "CANCELLED", "Cancelled"),
+         stringEnum(config.applyTablePrefix(EVENT_STATUS_PVS), "PENDING", "Pending (retried by the next update)", "EXHAUSTED", "Exhausted (gave up)",
+            "AWAITING_REINDEX", "Awaiting full reindex", "DEFERRED", "Deferred (indexing paused)"),
+         stringEnum(config.applyTablePrefix(EVENT_ACTION_PVS), "INDEX", "Index", "DELETE", "Delete"),
+         new QPossibleValueSource()
+            .withName(config.applyTablePrefix(CONFIG_SOURCE_PVS))
+            .withType(QPossibleValueSourceType.ENUM)
+            .withEnumValues(List.of(
+               new QPossibleValue<>(QuickSearchIndex.SOURCE_CODE, "Code default"),
+               new QPossibleValue<>(QuickSearchIndex.SOURCE_OWNER, "Added by owner")))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Add what quick_search_index refers to (its field, settings and tombstone
+    ** tables, join, field-list widget and possible-value sources) to an
+    ** instance whose tables are built with the helpers instead of produced by
+    ** the QBit (tests, hand-assembled instances). Anything already there is
+    ** kept.
+    *******************************************************************************/
+   public static void addCompanionMetaData(QInstance qInstance, QuickSearchQBitConfig config)
+   {
+      for(QTableMetaData table : List.of(buildIndexFieldTable(config), buildSettingsTable(config), buildTombstoneTable(config)))
+      {
+         if(qInstance.getTable(table.getName()) == null)
+         {
+            qInstance.addTable(table);
+         }
+      }
+      QJoinMetaData join = buildIndexFieldsJoin(config);
+      if(qInstance.getJoin(join.getName()) == null)
+      {
+         qInstance.addJoin(join);
+      }
+      for(QJoinMetaData extraJoin : List.of(buildIndexRunsJoin(config)))
+      {
+         if(qInstance.getJoin(extraJoin.getName()) == null)
+         {
+            qInstance.addJoin(extraJoin);
+         }
+      }
+      for(QWidgetMetaData widget : List.of(buildIndexFieldsWidget(config), buildIndexRunsWidget(config)))
+      {
+         if(qInstance.getWidget(widget.getName()) == null)
+         {
+            qInstance.addWidget(widget);
+         }
+      }
+
+      List<QPossibleValueSource> sources = new ArrayList<>(buildCustomPossibleValueSources(config));
+      sources.addAll(buildEnumPossibleValueSources(config));
+      sources.add(new QPossibleValueSource()
+         .withName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
+         .withType(QPossibleValueSourceType.CUSTOM)
+         .withIdType(QFieldType.STRING)
+         .withCustomCodeReference(new QCodeReference(QuickSearchTablePossibleValueProviders.SearchableTables.class)));
+      for(QPossibleValueSource source : sources)
+      {
+         if(qInstance.getPossibleValueSource(source.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(source);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An enum source of string ids and labels (id, label, id, label...).
+    *******************************************************************************/
+   private static QPossibleValueSource stringEnum(String name, String... idsAndLabels)
+   {
+      List<QPossibleValue<?>> values = new ArrayList<>();
+      for(int i = 0; i + 1 < idsAndLabels.length; i += 2)
+      {
+         values.add(new QPossibleValue<>(idsAndLabels[i], idsAndLabels[i + 1]));
+      }
+      return (new QPossibleValueSource().withName(name).withType(QPossibleValueSourceType.ENUM).withEnumValues(values));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QPossibleValueSource custom(String name, Class<?> provider)
+   {
+      return (new QPossibleValueSource()
+         .withName(name)
+         .withType(QPossibleValueSourceType.CUSTOM)
+         .withIdType(QFieldType.STRING)
+         .withCustomCodeReference(new QCodeReference(provider)));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigMetaDataProducer.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the data-driven config's join, field-list widget and
+ ** possible-value sources (see QuickSearchConfigMetaDataHelper).
+ *******************************************************************************/
+public class QuickSearchConfigMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig       config = getQBitConfig();
+      MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexFieldsJoin(config));
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexFieldsWidget(config));
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexRunsJoin(config));
+      output.add(QuickSearchConfigMetaDataHelper.buildIndexRunsWidget(config));
+      for(QPossibleValueSource possibleValueSource : QuickSearchConfigMetaDataHelper.buildCustomPossibleValueSources(config))
+      {
+         output.add(possibleValueSource);
+      }
+      for(QPossibleValueSource possibleValueSource : QuickSearchConfigMetaDataHelper.buildEnumPossibleValueSources(config))
+      {
+         output.add(possibleValueSource);
+      }
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (150);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigProcessMetaDataHelper.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDocumentsChoice;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
+import com.kingsrook.qbits.quicksearch.processes.config.AddSearchIndexSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.ApplyIndexChangesSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.CancelRunSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.SetIndexingPausedSteps;
+import com.kingsrook.qbits.quicksearch.processes.config.StopIndexingSteps;
+
+
+/*******************************************************************************
+ ** The config processes. Each is a standard streamed-ETL process with a
+ ** review step (the transform's summary: what will happen, counts, problems)
+ ** and a result step, run from its table's Actions (never from the nav):
+ ** - Add Search Index, on quick_search_index (no selection needed);
+ ** - Disable / Remove Search Index and Apply Search Index Changes, on the
+ **   selected quick_search_index rows;
+ ** - Cancel Index Run, on the selected quick_search_index_run rows;
+ ** - Pause / Resume Indexing, on the quick_search_settings row.
+ ** All need the admin permission (quick_search_admin by default).
+ *******************************************************************************/
+public class QuickSearchConfigProcessMetaDataHelper
+{
+   public static final String ADD_INDEX_PROCESS_NAME     = "quickSearchAddIndex";
+   public static final String DISABLE_INDEX_PROCESS_NAME = "quickSearchDisableIndex";
+   public static final String REMOVE_INDEX_PROCESS_NAME  = "quickSearchRemoveIndex";
+   public static final String APPLY_CHANGES_PROCESS_NAME = "quickSearchApplyIndexChanges";
+   public static final String CANCEL_RUN_PROCESS_NAME    = "quickSearchCancelRun";
+   public static final String PAUSE_PROCESS_NAME         = "quickSearchPauseIndexing";
+   public static final String RESUME_PROCESS_NAME        = "quickSearchResumeIndexing";
+
+
+
+   /*******************************************************************************
+    ** Every config process.
+    *******************************************************************************/
+   public static List<QProcessMetaData> buildProcesses(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         buildAddIndexProcess(config),
+         buildStopIndexingProcess(config, DISABLE_INDEX_PROCESS_NAME, "Disable Search Index", "toggle_off", StopIndexingSteps.MODE_DISABLE, QuickSearchDocumentsChoice.KEEP),
+         buildStopIndexingProcess(config, REMOVE_INDEX_PROCESS_NAME, "Remove Search Index", "delete", StopIndexingSteps.MODE_REMOVE, QuickSearchDocumentsChoice.DELETE),
+         buildApplyChangesProcess(config),
+         buildCancelRunProcess(config),
+         buildSetPausedProcess(config, PAUSE_PROCESS_NAME, "Pause Indexing", "pause_circle", true),
+         buildSetPausedProcess(config, RESUME_PROCESS_NAME, "Resume Indexing", "play_circle", false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Add Search Index: the valid way to add a table.
+    *******************************************************************************/
+   public static QProcessMetaData buildAddIndexProcess(QuickSearchQBitConfig config)
+   {
+      String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
+      QProcessMetaData process = base(config, ADD_INDEX_PROCESS_NAME, "Add Search Index", "add_circle", indexTable,
+         AddSearchIndexSteps.Extract.class, AddSearchIndexSteps.Transform.class, AddSearchIndexSteps.Load.class, 0, List.of(
+            new QFieldMetaData(AddSearchIndexSteps.RECORD_TABLE, QFieldType.STRING).withLabel("Table")
+               .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)),
+            new QFieldMetaData("estimatedRecords", QFieldType.INTEGER).withLabel("Records to Index"),
+            new QFieldMetaData("fieldCount", QFieldType.INTEGER).withLabel("Fields"),
+            new QFieldMetaData(AddSearchIndexSteps.FIELD_INTERVAL, QFieldType.INTEGER).withLabel("Update Interval (Minutes)"),
+            new QFieldMetaData(AddSearchIndexSteps.FIELD_TIMESTAMP, QFieldType.STRING).withLabel("Timestamp Field"),
+            new QFieldMetaData(AddSearchIndexSteps.FIELD_ENABLED, QFieldType.BOOLEAN)));
+
+      process.withStep(0, new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Choose a table")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text",
+            "Only tables you can read, that are not indexed yet and are not internal or secret are offered. The review step shows the fields the index starts with "
+               + "and how many records its first build indexes; change the fields afterwards under Searchable Fields on the index."))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_TABLE_NAME, QFieldType.STRING).withLabel("Table").withIsRequired(true)
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.ELIGIBLE_TABLE_PVS)))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_INTERVAL, QFieldType.INTEGER).withLabel("Update Interval (Minutes)")
+            .withDefaultValue(config.getDefaultBasepullIntervalMinutes()))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_TIMESTAMP, QFieldType.STRING).withLabel("Timestamp Field (blank: detect)")
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.TIMESTAMP_FIELD_PVS)))
+         .withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_ENABLED, QFieldType.BOOLEAN).withLabel("Enabled").withDefaultValue(true)));
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Disable or Remove Search Index, with the documents choice.
+    *******************************************************************************/
+   static QProcessMetaData buildStopIndexingProcess(QuickSearchQBitConfig config, String name, String label, String icon, String mode, QuickSearchDocumentsChoice defaultChoice)
+   {
+      String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(config.applyPrefix(name))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(indexTable)
+         .withSourceTable(indexTable)
+         .withDestinationTable(indexTable)
+         .withExtractStepClass(ExtractViaQueryStep.class)
+         .withTransformStepClass(StopIndexingSteps.Transform.class)
+         .withLoadStepClass(StopIndexingSteps.Load.class)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withTransactionLevelAutoCommit()
+         .withMinInputRecords(1)
+         .withFields(List.of(new QFieldMetaData(StopIndexingSteps.FIELD_MODE, QFieldType.STRING).withDefaultValue(mode)))
+         .withReviewStepRecordFields(indexReviewFields(config))
+         .getProcessMetaData();
+      process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+
+      process.withStep(0, new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Search documents")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text",
+            StopIndexingSteps.MODE_REMOVE.equals(mode)
+               ? "Removing stops indexing the table and deletes its index and field rows. Code defaults never add it back (see Quick Search Removed Config)."
+               : "Disabling stops indexing the table and hides it from search; enabling it again rebuilds it."))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withFormField(new QFieldMetaData(StopIndexingSteps.FIELD_DOCUMENTS, QFieldType.STRING).withLabel("Search Documents").withIsRequired(true)
+            .withDefaultValue(defaultChoice.name())
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.DOCUMENTS_CHOICE_PVS))));
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Apply Search Index Changes: preview field changes, then rebuild.
+    *******************************************************************************/
+   static QProcessMetaData buildApplyChangesProcess(QuickSearchQBitConfig config)
+   {
+      String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
+      return (base(config, APPLY_CHANGES_PROCESS_NAME, "Apply Search Index Changes", "published_with_changes", indexTable,
+         ExtractViaQueryStep.class, ApplyIndexChangesSteps.Transform.class, ApplyIndexChangesSteps.Load.class, 1, indexReviewFields(config)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Cancel Index Run: the cooperative cancel flag.
+    *******************************************************************************/
+   static QProcessMetaData buildCancelRunProcess(QuickSearchQBitConfig config)
+   {
+      String runTable = config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME);
+      return (base(config, CANCEL_RUN_PROCESS_NAME, "Cancel Index Run", "cancel", runTable,
+         ExtractViaQueryStep.class, CancelRunSteps.Transform.class, CancelRunSteps.Load.class, 1, List.of(
+            new QFieldMetaData("id", QFieldType.INTEGER),
+            new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)),
+            new QFieldMetaData("runType", QFieldType.STRING),
+            new QFieldMetaData("status", QFieldType.STRING),
+            new QFieldMetaData("recordsProcessed", QFieldType.INTEGER))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Pause or Resume Indexing.
+    *******************************************************************************/
+   static QProcessMetaData buildSetPausedProcess(QuickSearchQBitConfig config, String name, String label, String icon, boolean pause)
+   {
+      String settingsTable = config.applyTablePrefix(QuickSearchSettings.TABLE_NAME);
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(config.applyPrefix(name))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(settingsTable)
+         .withSourceTable(settingsTable)
+         .withDestinationTable(settingsTable)
+         .withExtractStepClass(ExtractViaQueryStep.class)
+         .withTransformStepClass(SetIndexingPausedSteps.Transform.class)
+         .withLoadStepClass(SetIndexingPausedSteps.Load.class)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withTransactionLevelAutoCommit()
+         .withMinInputRecords(1)
+         .withFields(List.of(new QFieldMetaData(SetIndexingPausedSteps.FIELD_PAUSE, QFieldType.BOOLEAN).withDefaultValue(pause)))
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("indexingPaused", QFieldType.BOOLEAN).withLabel("Indexing Paused"),
+            new QFieldMetaData("pausedAt", QFieldType.DATE_TIME).withLabel("Paused Since"),
+            new QFieldMetaData("pausedBy", QFieldType.STRING).withLabel("Paused By")))
+         .getProcessMetaData();
+      process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+
+      if(pause)
+      {
+         process.withStep(0, new QFrontendStepMetaData()
+            .withName("input")
+            .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+            .withLabel("Pause indexing")
+            .withFormField(new QFieldMetaData(SetIndexingPausedSteps.FIELD_REASON, QFieldType.STRING).withLabel("Reason (optional)")));
+      }
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** The index fields shown on the review step.
+    *******************************************************************************/
+   private static List<QFieldMetaData> indexReviewFields(QuickSearchQBitConfig config)
+   {
+      return (List.of(
+         new QFieldMetaData("id", QFieldType.INTEGER),
+         new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table")
+            .withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)),
+         new QFieldMetaData("enabled", QFieldType.BOOLEAN),
+         new QFieldMetaData("status", QFieldType.STRING),
+         new QFieldMetaData("documentCount", QFieldType.INTEGER)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A streamed-ETL process with a validated review step, gated by the admin
+    ** permission.
+    *******************************************************************************/
+   private static QProcessMetaData base(QuickSearchQBitConfig config, String name, String label, String icon, String tableName, Class<? extends AbstractExtractStep> extract,
+      Class<? extends AbstractTransformStep> transform, Class<? extends AbstractLoadStep> load, Integer minInputRecords, List<QFieldMetaData> reviewFields)
+   {
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(config.applyPrefix(name))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(tableName)
+         .withSourceTable(tableName)
+         .withDestinationTable(tableName)
+         .withExtractStepClass(extract)
+         .withTransformStepClass(transform)
+         .withLoadStepClass(load)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withTransactionLevelAutoCommit()
+         .withMinInputRecords(minInputRecords)
+         .withReviewStepRecordFields(reviewFields)
+         .getProcessMetaData();
+      process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+      return (process);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchConfigProcessesMetaDataProducer.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the config processes (see QuickSearchConfigProcessMetaDataHelper).
+ *******************************************************************************/
+public class QuickSearchConfigProcessesMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+      for(QProcessMetaData process : QuickSearchConfigProcessMetaDataHelper.buildProcesses(getQBitConfig()))
+      {
+         output.add(process);
+      }
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}