Merge branch 'develop' into feature/developer-panel

chore: align with the QRun qbit repository standard

.circleci/config.yml

@@ -1,14 +1,71 @@
 version: 2.1
-setup: true
+
 orbs:
-  qqq-orb: kingsrook/qqq-orb@0.8
+   qqq-orb: kingsrook/qqq-orb@0.6.8
+
+commands:
+   configure-package-install:
+      steps:
+         - run:
+              name: Keep package installation unattended
+              command: |
+                 # These build machines are discarded after the job.
+                 sudo install -d /etc/needrestart/conf.d
+                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
+
 workflows:
-  setup:
-    jobs:
-    - qqq-orb/generate_pipeline:
-        orb_version: '0.8'
-        filters:
-          branches:
-            ignore: automation/qbit-rc-pending
-          tags:
-            only: /v.*/
+   test_only:
+      jobs:
+         - qqq-orb/mvn_test_only:
+              pre-steps:
+                 - configure-package-install
+              context: [qqq-maven-registry-credentials]
+              filters:
+                 branches:
+                    ignore: /(develop|release\/.*|hotfix\/.*)/
+
+   publish_snapshot:
+      jobs:
+         - qqq-orb/mvn_publish:
+              pre-steps:
+                 - configure-package-install
+              context: [qqq-maven-registry-credentials]
+              branch_type: snapshot
+              filters:
+                 branches:
+                    only: [develop]
+
+   publish_release_candidate:
+      jobs:
+         - qqq-orb/mvn_publish:
+              pre-steps:
+                 - configure-package-install
+              context: [qqq-maven-registry-credentials]
+              branch_type: release_candidate
+              filters:
+                 branches:
+                    only: [/release\/.*/]
+
+   publish_release:
+      jobs:
+         - qqq-orb/mvn_publish:
+              pre-steps:
+                 - configure-package-install
+              context: [qqq-maven-registry-credentials]
+              branch_type: release
+              filters:
+                 branches:
+                    ignore: /.*/
+                 tags:
+                    only: [/v.*/]
+
+   publish_hotfix_release:
+      jobs:
+         - qqq-orb/mvn_publish:
+              pre-steps:
+                 - configure-package-install
+              context: [qqq-maven-registry-credentials]
+              branch_type: hotfix
+              filters:
+                 branches:
+                    only: [/hotfix\/.*/]

.claude/settings.local.json

@@ -1,63 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(find:*)",
-      "Bash(grep:*)",
-      "Bash(mvn compile:*)",
-      "Bash(jar -tf:*)",
-      "Bash(jar -xf:*)",
-      "Bash(cat:*)",
-      "Bash(mvn test:*)",
-      "Bash(done)",
-      "Bash(mvn verify:*)",
-      "Bash(git add:*)",
-      "Bash(git commit:*)",
-      "Bash(git push:*)",
-      "mcp__circleci-mcp-server__get_latest_pipeline_status",
-      "mcp__circleci-mcp-server__list_followed_projects",
-      "mcp__circleci-mcp-server__run_pipeline",
-      "mcp__circleci-mcp-server__get_build_failure_logs",
-      "mcp__circleci-mcp-server__config_helper",
-      "mcp__github__get_file_contents",
-      "Bash(mvn dependency:tree:*)",
-      "Bash(mvn help:effective-pom:*)",
-      "Bash(git checkout:*)",
-      "Bash(ls /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/docs/*.md)",
-      "Bash(git:*)",
-      "Bash(ls:*)",
-      "Bash(open:*)",
-      "Bash(WT=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/.claude/worktrees/agent-a9dc6e8e)",
-      "Bash(DEST=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search)",
-      "Bash(cp \"$WT/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java\" \"$DEST/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/\")",
-      "Bash(mvn clean:*)",
-      "Bash(jar tf:*)",
-      "Bash(unzip -p /Users/james.maes/.m2/repository/com/kingsrook/qqq/qqq-backend-core/0.40.0-SNAPSHOT/qqq-backend-core-0.40.0-SNAPSHOT.jar com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java)",
-      "Bash(sed -i '' 's/postInsertRecord/postInsert/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java)",
-      "Bash(sed -i '' 's/postUpdateRecord/postUpdate/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java)",
-      "Bash(sed -i '' 's/postDeleteRecord/postDelete/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java)",
-      "Bash(docker info:*)",
-      "Bash(docker run:*)",
-      "Read(//var/run/**)",
-      "Read(//Users/james.maes/**)",
-      "Bash(echo \"DOCKER_HOST=$DOCKER_HOST\")",
-      "Bash(echo \"TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=$TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE\")",
-      "Bash(DOCKER_HOST=unix:///var/run/docker.sock TESTCONTAINERS_RYUK_DISABLED=true mvn verify -Dit.test=OpenSearchIntegrationTest)",
-      "Bash(docker version:*)",
-      "Bash(DOCKER_HOST=unix:///var/run/docker.sock mvn verify -Dit.test=OpenSearchIntegrationTest)",
-      "Bash(curl -s --unix-socket /var/run/docker.sock http://localhost/v1.24/info)",
-      "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)}  OS: {d.get\\(OperatingSystem,?\\)}''''\\)\")",
-      "Bash(curl -s --unix-socket /Users/james.maes/Library/Containers/com.docker.docker/Data/docker-cli.sock http://localhost/v1.47/info)",
-      "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)}  OS: {d.get\\(OperatingSystem,?\\)}  Containers: {d.get\\(Containers,?\\)}''''\\)\")",
-      "Bash(curl -s --unix-socket /Users/james.maes/.docker/run/docker.sock http://localhost/v1.47/info)",
-      "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)}  NCPU: {d.get\\(NCPU,?\\)}  OSType: {d.get\\(OSType,?\\)}''''\\)\")",
-      "Bash(DOCKER_HOST=unix:///Users/james.maes/.docker/run/docker.sock DOCKER_API_VERSION=1.47 mvn verify -Dit.test=OpenSearchIntegrationTest -pl .)",
-      "Bash(cp /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/LICENSE /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/LICENSE)",
-      "Bash(cp -r /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/checkstyle/* /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/checkstyle/)",
-      "Bash(gh pr:*)",
-      "Bash(gh repo:*)",
-      "Bash(gh api:*)",
-      "Bash(python3 -c \":*)",
-      "mcp__circleci-mcp-server__get_job_test_results"
-    ]
-  }
-}

.editorconfig

@@ -0,0 +1,20 @@
+# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 3
+tab_width = 3
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{cmd,bat}]
+end_of_line = crlf

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto eol=lf
+*.cmd text eol=crlf
+mvnw text eol=lf

.gitea/workflows/qrun-release.yaml

@@ -1,25 +0,0 @@
-# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
-name: qrun-release
-on:
-  push: {}
-  workflow_dispatch: {}
-permissions:
-  contents: read
-  actions: read
-jobs:
-  release:
-    uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
-    with:
-      runner: linux-small
-      docker-runner: container-build
-      browsers: true
-      browser-runner: browser-build
-  postpublish:
-    if: ${{ needs.release.outputs.capture == 'true' }}
-    needs: release
-    permissions:
-      contents: write
-      actions: write
-    uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
-    with:
-      runner: linux-small

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @KofTwentyTwo

.github/dependabot.yml

@@ -0,0 +1,25 @@
+# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
+# major updates stay separate so each gets its own review. At most three version-update PRs per
+# ecosystem are open at once. Security updates are enabled in the repository settings and are
+# not limited by this schedule.
+version: 2
+updates:
+  - package-ecosystem: maven
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      maven-minor-and-patch:
+        update-types: [minor, patch]
+
+  - package-ecosystem: github-actions
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      github-actions:
+        patterns: ['*']

.github/workflows/codeql.yml

@@ -1,23 +0,0 @@
-name: CodeQL
-'on':
-  push:
-    branches:
-    - main
-    - develop
-    - release/**
-    - hotfix/**
-  pull_request: {}
-  schedule:
-  - cron: 41 16 * * 0
-  workflow_dispatch: {}
-permissions:
-  contents: read
-  actions: read
-  security-events: write
-  packages: read
-concurrency:
-  group: codeql-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-jobs:
-  analyze:
-    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.gitignore

@@ -18,3 +18,4 @@ hs_err_pid*
 dependency-reduced-pom.xml
 /.env.local
 /.cache/
+.claude/settings.local.json

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,3 @@
+wrapperVersion=3.3.4
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip

.qqq-ci.yml

@@ -1,16 +0,0 @@
-profile: qbit
-orb_version: '0.8'
-default_branch: develop
-java_version: '21'
-browsers: true
-api_checks: false
-docs_only: false
-release_on_main: true
-release_tag_policy: v-prefix
-test_branch_policy: non-release-no-main-hotfix
-publish:
-  snapshot: true
-  release_candidate: true
-  release: true
-  hotfix: true
-  asciidoc: false

AGENTS.md

@@ -0,0 +1,68 @@
+# AGENTS.md
+
+This file provides guidance to Codex when working with code in this repository.
+
+## Project Overview
+
+**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications.
+
+## Build Commands
+
+```bash
+./mvnw compile          # Compile (always use the wrapper: Maven 3.10 cannot read the parent POM)
+./mvnw test             # Run unit tests
+./mvnw test -Dtest=ClassName  # Run single test class
+./mvnw package          # Build JAR
+CI=true ./mvnw verify   # Full verify: integration tests (Docker) and the JaCoCo gate
+./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0   # other server line
+./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1   # other qqq 4.1 build (profile is active by default)
+```
+
+Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
+Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
+Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped without Docker locally, failed when `CI=true`.
+
+## Architecture
+
+**Entry point:** `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`. QQQ's default `produce()` registers `QBitMetaData` (`com.kingsrook.qbits:quick-search`), validates the config, runs the component producers in `metadata/` (tables, processes, PVS, app), then `postProduceActions` discovers searchable tables, builds the `QuickSearchRuntime`, registers `QuickSearchRecordChangeListener` and `QuickSearchRuntimeService`, and in `FAIL_FAST` mode prepares the index. `produce(QInstance)` adds everything to the instance (0.x call style still works).
+
+**Key Classes:**
+- `QuickSearchQBitConfig` - Host-facing config: connection (`opensearchUrl`), `authMode`, `tls`, timeouts, startup mode, scheduling, permission rules, search bounds, tables. List-based, null-safe `validate`.
+- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
+- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
+- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
+- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
+- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
+- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
+- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
+- `metadata/*` - Component producers and the table/process helpers.
+
+**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
+
+**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
+
+## Code Style
+
+- 3-space indentation, opening braces on next line
+- Javadoc flower box comments (80-char width with `**` borders)
+- Fluent-style setters (`withX()`)
+- Use wrapper types (`Integer`, `Boolean`) not primitives
+
+## Dependencies
+
+- Java 21
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI
+- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
+- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
+- JUnit 5 + AssertJ + Mockito for testing
+- Testcontainers 1.21.4 for integration tests (requires Docker)
+
+## Knowledge base
+
+Second-brain vault notes covering this repo and the QQQ platform it plugs into:
+
+- Platform hub: `$SECOND_BRAIN_VAULT/knowledge/qqq/qqq-hub.md` (start here; read `knowledge/qqq/architecture/metadata-model.md` for QBit mechanics)
+- This repo's dossier: `$SECOND_BRAIN_VAULT/knowledge/qqq/repos/qbit-quick-search.md` (reviewed at develop @ `8d9ec2711fde`, 2026-07-04)
+- Production-readiness audit + P0-P3 roadmap: `$SECOND_BRAIN_VAULT/projects/qbit-quick-search.md`
+
+Note: the dossier was reviewed before the qqq 4.0 re-pin; trust this file for build and dependency facts.

CHANGELOG.md

@@ -0,0 +1,94 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
+
+## [Unreleased]
+
+Requires QQQ 4.1.0-RC.4 or later (the developer panel contract, QRun-IO/qqq#1076).
+
+### Added
+- Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`.
+
+## [1.1.0-RC.1] - 2026-10-11
+
+First release candidate of 1.1, published to Maven Central from `release/1.1`. It contains every change listed under 1.1.0 and 1.0.0 below (1.0.0 has not been released as GA). Requires QQQ 4.1.0-RC.2 or later. Upgrade notes: `docs/MIGRATION-1.1.md`.
+
+## [1.1.0] - Unreleased
+
+Requires QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI; built against 4.1.0-RC.2. See `docs/MIGRATION-1.1.md`.
+
+### Added
+- `QuickSearchRecordSearchProvider`: serves QQQ core record search (`RecordSearchAction`, `POST /qqq/v1/search`, the Next UI global search) from OpenSearch for the tables Quick Search indexes. One `_msearch` request with a per-table top-N search (same query, AND semantics and field weights as `QuickSearchAction`); returns record ids only, which core re-reads through `QueryAction` (permissions, record locks, labels). Audit finding F-11, decision D1.
+- `serveCoreRecordSearch` (default `true`): registers the provider and gives indexed tables core `searchFields` (visible string and integer searchable fields) when they have none. Host `searchFields` and a host-registered provider are never replaced.
+- `recordSearchTimeoutMillis` (default 2500): wall-clock limit on the OpenSearch request; on timeout or any failure core falls back to its own search, and the provider claims no tables for 30 seconds.
+- `QuickSearchOpenSearchClient.searchRecordIdsPerTable`.
+- Diagnostic tools (#35), on by default (`enableDiagnosticTools`): Search Playground, Validate Search Index (schedulable with `validationCronExpression`; reports kept in the new `quickSearchValidationRun` table), Test Search Analyzer, and on every indexed table and on `quickSearchFailedEvent` rows the record actions Inspect Search Document (database vs index, and why a record does or does not match a term) and Reindex in Search. All are table or record actions, gated by the new `debugPermissionRules` (`quickSearchDebug.hasAccess`; Reindex in Search uses `adminPermissionRules`), rate-limited per user (`diagnosticRunsPerMinute`), and recorded in the new `quickSearchDiagnosticEvent` table. Settings: `enableRecordDiagnosticActions`, `validationCronExpression`, `validationCronTimeZoneId`, `validationSampleSize`.
+- Search failure diagnostics: a failed or timed-out OpenSearch request behind core record search or `QuickSearchAction` is recorded as a `SEARCH_FAILED` diagnostic event (tables, duration, error, server; never the term); the runtime counts failures and backoff fallbacks. The Search Diagnostics widget (`quickSearchDiagnostics`, on the admin app home) shows the provider's state, recent failures and the latest validation.
+- `QuickSearchDiagnosticsOperations`: read-only probe search (named clauses, field highlights, request and response JSON), explain, document reads, `_analyze`, document sampling, and raw cluster, mapping and settings reads.
+
+### Changed
+- Run history purging also purges `quickSearchDiagnosticEvent` and `quickSearchValidationRun` rows older than `runHistoryRetentionDays`.
+
+### Security
+- Fields of type `PASSWORD` are no longer indexed (as hidden fields already were). Reconcile or fully reindex once to remove values a 1.0 index may hold.
+
+## [1.0.0-RC.2] - 2026-10-06
+
+Second release candidate, published to Maven Central from `release/1.0`. Fixes the five pre-GA findings from the PR #10 review (#13 to #17); the changes are listed under 1.0.0 below. Requires QQQ 4.1. No schema change, but two new status values: `REBUILDING` on `quickSearchIndex.status` and `AWAITING_REINDEX` on `quickSearchFailedEvent.status`; a host that sized its own `quickSearchFailedEvent.status` column needs at least 16 characters. Known gaps before GA are tracked in #12.
+
+## [1.0.0-RC.1] - 2026-10-05
+
+First release candidate of 1.0, published to Maven Central from `release/1.0`. It contains every change listed under 1.0.0 below. Requires QQQ 4.1 (built against 4.1.0-RC.1). Known gaps before GA are tracked in #12.
+
+## [1.0.0] - Unreleased
+
+Requires QQQ 4.1. Closes every blocker of the 2026-10-04 production-readiness audit (`docs/audits/2026-10-04/README.md`).
+
+### Added
+- Standard QQQ 4.1 QBit contract: `QuickSearchQBitProducer implements QBitMetaDataProducer`, self-registered `QBitMetaData` (`com.kingsrook.qbits:quick-search`, real artifact version), `sourceQBitName` on produced metadata, component producers, `QuickSearchRuntimeService`.
+- Real-time indexing through `RecordChangeListenerInterface`, published after commit via `QBackendTransaction.addAfterCommitCallback`.
+- `quickSearchFailedEvent` table: failed real-time events are recorded and replayed by basepull.
+- Scheduling: `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`.
+- Full reindex of all tables builds a fresh physical index and swaps the alias; the configured index name is now an alias.
+- Mapping version 2: separate search analyzer, dynamic template mapping `fieldValues.*` as text, `_meta.quickSearchMappingVersion`; outdated indexes are detected and rebuilt by a full reindex.
+- Permission-aware search: table READ permission filter, `QueryAction` post-filter for record security locks, `tableNames`, `limitPerTable`, `tableLabel`, `totalHitsIsLowerBound`, term and page bounds.
+- Connection security: `opensearchUrl`, `authMode` (`NONE`, `BASIC`, `AWS_SIGV4`), `tls` block (CA, trust store, mTLS, hostname verification, guarded `insecureSkipVerify`), timeouts, pool size, `User-Agent`, `OpenSearchTransportCustomizer` SPI, `${env.X}` secret references, redacting `toString`, `@JsonIgnore` on secret getters.
+- `startupMode` (`FAIL_FAST` default, `DEGRADED`), `enableBasepullProcess` and `enableMaintenanceProcesses`, `basepullOverlapSeconds`, `runHistoryRetentionDays`, `maxFieldLength`, `maxSearchLimit`, `maxBulkRequestBytes`, `applyRecordSecurityLocks`, `adminPermissionRules`, `tablePermissionRules`, `tableMetaDataCustomizer`.
+- Admin table fields: `lastReconcileTime`, `lastRunStatus`, `lastErrorMessage`, `documentCount`, `realTimeErrorCount`; `quickSearchIndexRun.quickSearchIndexId` shows the table name; icons and permission rules on the app, tables and processes; unique key on `quickSearchIndex.tableName`.
+- Display values and possible-value labels are indexed; record labels fall back to QQQ's record label format; hidden fields are excluded; values are stored as strings and truncated to `maxFieldLength`.
+- External versioning (`external_gte`) from the basepull timestamp field so stale writes never overwrite newer documents.
+- Reconcile removes documents of tables no longer configured; drift between configured fields and the index row sets `status = NEEDS_REINDEX`.
+- Maven Wrapper (3.9.11), JaCoCo coverage gate actually enforced, parameterized Testcontainers image (`-Dopensearch.test.image`).
+
+### Changed
+- Build targets qqq 4.1 (4.1.0-RC.1 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 is released), opensearch-java 3.10.0, JUnit from the parent BOM.
+- Basepull and full reindex page by primary key; the basepull watermark is the run start (minus overlap) with greater-than-or-equal semantics.
+- Basepull fails at the end when any table failed (one table's failure does not stop the others).
+- Query terms use the AND operator and are no longer edge-ngrammed at search time.
+- `QuickSearchOpenSearchClient.search` takes a collection of allowed tables; `deleteDocumentsForTable` returns the deleted count; all OpenSearch errors surface as `QException`.
+- `IndexingUtils.normalizeSearchText` no longer lower-cases (analyzers do).
+- `enableScheduledProcesses` is deprecated in favour of `enableBasepullProcess` and `enableMaintenanceProcesses`.
+- `QuickSearchQBitContext` is deprecated; state lives on `QuickSearchRuntime`, resolved from the `QInstance`.
+- Search `offset` counts results the session can read. With record security locks on, hits are scanned from the start in batches of at most 1,000, filtered for access and deduplicated; one call reads at most 10,000 hits over 32 requests, and in per-table mode each of T tables gets max(10,000 / T, 2 x `limitPerTable`) hits. When that share runs out `hasMore` is false and `totalHitsIsLowerBound` is true. With locks on, `totalHits` counts hits confirmed readable. Per-table mode now honours `offset` (#16).
+- A full reindex of all tables marks each table `REBUILDING` while it runs; drift detection is suspended for those tables. After a killed run, run the full reindex again (#14).
+
+### Removed
+- The three table customizers (`QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`), replaced by `QuickSearchRecordChangeListener`.
+- Support for QQQ 4.0.
+
+### Security
+- `opensearchUrl` with userinfo (`https://user:pass@host`) fails validation; an `${env.X}` or `${prop.X}` credential or TLS path that resolves to an empty or blank value fails transport creation and names the field, never the value; `tls.hostnameVerification=false` logs a warning (#15).
+
+### Fixed
+- Rolled-back writes could be indexed; real-time failures were silent; basepull lost rows modified during a run and rows at the boundary second; unordered paging skipped or duplicated rows; full reindex blanked search; search leaked labels and snippets across tables regardless of permissions; mixed field types across tables broke indexing; non-text searchable fields failed every search; record labels were null for annotation tables; duplicate `quickSearchIndex` rows on concurrent first runs; unbounded `limit`/`offset`; unchecked `OpenSearchException`; unreachable cluster at startup left the host healthy with search broken.
+- Deletes made during a full reindex of all tables were lost from the new index; they are now captured and applied after the alias swap (#14).
+- An update was dropped without a `quickSearchFailedEvent` row when the listener's re-read failed or the optional AWS SDK was missing (`LinkageError`); both are now recorded for basepull, and a `DEGRADED` start without the AWS SDK no longer aborts host startup (#13).
+- Search pages overlapped under record locks, per-table mode ignored `offset`, and a large `maxSearchLimit` could exceed the 10,000 result window (#16).
+- A failing document count was swallowed and cleared `documentCount`; it is now logged at warn and the previous count is kept (#17).
+- A real-time insert was indexed with raw values (a foreign key as its id, the record label as "2 - email") until the next basepull. Inserts are now re-read in the write's transaction with display values and possible-value labels, as updates are; a failed re-read is recorded in `quickSearchFailedEvent` for basepull and never fails the insert (#28).
+
+## [0.2.0] - 2026-03-30
+- Config-driven table indexing (`SearchableTableConfig`).
+
+## [0.1.0] - 2026-03-29
+- Initial release.

CLAUDE.md

@@ -4,59 +4,42 @@ This file provides guidance to Claude Code when working with code in this reposi
 
 ## Project Overview
 
-**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications. Current version: 0.1.0.
+**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications.
 
 ## Build Commands
 
 ```bash
-mvn compile          # Compile
-mvn test             # Run all tests (174 tests)
-mvn test -Dtest=ClassName  # Run single test class
-mvn package          # Build JAR
-mvn verify           # Full verify with coverage check
+./mvnw compile          # Compile (always use the wrapper: Maven 3.10 cannot read the parent POM)
+./mvnw test             # Run unit tests
+./mvnw test -Dtest=ClassName  # Run single test class
+./mvnw package          # Build JAR
+CI=true ./mvnw verify   # Full verify: integration tests (Docker) and the JaCoCo gate
+./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0   # other server line
+./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1   # other qqq 4.1 build (profile is active by default)
 ```
 
 Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
 Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
+Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped without Docker locally, failed when `CI=true`.
 
 ## Architecture
 
-**Entry point:** `QuickSearchQBitProducer.produce(QInstance)` validates config, discovers annotated entity classes, initializes the singleton `QuickSearchOpenSearchClient`, creates an `IndexEventPublisher`, registers all metadata into the `QInstance`, and attaches customizers to source tables.
+**Entry point:** `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`. QQQ's default `produce()` registers `QBitMetaData` (`com.kingsrook.qbits:quick-search`), validates the config, runs the component producers in `metadata/` (tables, processes, PVS, app), then `postProduceActions` discovers searchable tables, builds the `QuickSearchRuntime`, registers `QuickSearchRecordChangeListener` and `QuickSearchRuntimeService`, and in `FAIL_FAST` mode prepares the index. `produce(QInstance)` adds everything to the instance (0.x call style still works).
 
 **Key Classes:**
-- `QuickSearchQBitProducer` - Orchestrator: produces tables, processes, app, and registers customizers
-- `QuickSearchQBitConfig` - Configuration (backend name, OpenSearch connection, feature flags)
-- `QuickSearchQBitContext` - Static singleton holding the live config, client, publisher, and discovered table configs
-- `QuickSearchOpenSearchClient` - All OpenSearch operations (ensure index, bulk index, delete, search)
-- `QuickSearchableTableConfig` - Runtime config for a single indexed table (fields, weights, basepull settings)
+- `QuickSearchQBitConfig` - Host-facing config: connection (`opensearchUrl`), `authMode`, `tls`, timeouts, startup mode, scheduling, permission rules, search bounds, tables. List-based, null-safe `validate`.
+- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
+- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
+- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
+- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
+- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
+- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
+- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
+- `metadata/*` - Component producers and the table/process helpers.
 
-**Customizers** (`customizers/` package, all three registered when `enableRealTimeIndexing = true`):
-- `QuickSearchPostInsertCustomizer` - Fires after insert, publishes index events
-- `QuickSearchPostUpdateCustomizer` - Fires after update, publishes index events
-- `QuickSearchPostDeleteCustomizer` - Fires after delete, publishes delete events
+**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
 
-**Processes** (`processes/` package):
-- `AbstractIndexingStep` - Base class with shared indexing logic (query source table, build documents, publish events, update run record)
-- `BasepullIndexStep` - Queries records modified since last run timestamp
-- `FullReindexStep` - Wipes and rebuilds the index for one or all tables
-- `IndexingUtils` - Converts `QRecord` values to indexed text using field configs
-
-**Publisher** (`publisher/` package):
-- `IndexEventPublisher` - Strategy interface with `publishIndexEvents()`, `publishDeleteEvents()`, `close()`
-- `SynchronousIndexEventPublisher` - Default implementation; processes events immediately via the OpenSearch client
-- `IndexEvent` / `IndexEventAction` - Event model
-
-**Actions** (`actions/` package):
-- `QuickSearchAction` - Executes a paginated multi-match search across all indexed fields with per-field boosting
-- `QuickSearchInput` / `QuickSearchOutput` / `QuickSearchResult` - Search API model
-
-**Tables Produced:**
-- `quickSearchIndex` - One row per indexed table; tracks enabled state, basepull interval, last run times
-- `quickSearchIndexRun` - Run history with status, record counts, and error messages
-
-**Annotation Discovery:**
-- `@QuickSearchable(tableName = "...")` - Mark entity classes; defines basepull settings
-- `@QuickSearchField(weight, includeLabel)` - Mark fields to include in indexed text; takes precedence over `fields[]` array
+**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
 
 ## Code Style
 
@@ -68,9 +51,18 @@ Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
 ## Dependencies
 
 - Java 21
-- QQQ Backend Core 0.40.0-SNAPSHOT (via `qqq-bom-pom`)
-- OpenSearch Java Client 2.10.0
-- Apache HttpClient5 5.3
-- jackson-datatype-jsr310 2.21.0 (Instant serialization for OpenSearch client)
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.4`: the record-search provider SPI arrived in RC.2, the developer panel contract (`DeveloperPanelProviderInterface`) in RC.4
+- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
+- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
 - JUnit 5 + AssertJ + Mockito for testing
-- Testcontainers for integration tests (requires Docker)
+- Testcontainers 1.21.4 for integration tests (requires Docker)
+
+## Knowledge base
+
+Second-brain vault notes covering this repo and the QQQ platform it plugs into:
+
+- Platform hub: `$SECOND_BRAIN_VAULT/knowledge/qqq/qqq-hub.md` (start here; read `knowledge/qqq/architecture/metadata-model.md` for QBit mechanics)
+- This repo's dossier: `$SECOND_BRAIN_VAULT/knowledge/qqq/repos/qbit-quick-search.md` (reviewed at develop @ `8d9ec2711fde`, 2026-07-04)
+- Production-readiness audit + P0-P3 roadmap: `$SECOND_BRAIN_VAULT/projects/qbit-quick-search.md`
+
+Note: the dossier was reviewed before the qqq 4.0 re-pin; trust this file for build and dependency facts.

README.md

@@ -1,86 +1,61 @@
 # QBit: Quick Search
 
-[![Version](https://img.shields.io/badge/version-0.2.0-blue.svg)](https://github.com/QRun-IO/qbit-quick-search)
+[![Version](https://img.shields.io/badge/version-1.1.0--RC.1-blue.svg)](https://github.com/QRun-IO/qbit-quick-search)
 [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://www.apache.org/licenses/LICENSE-2.0)
 [![Java](https://img.shields.io/badge/java-21+-blue.svg)](https://adoptium.net/)
 
-> **OpenSearch-Powered Full-Text Search for QQQ Applications**
+> OpenSearch-backed full-text search across the tables of a QQQ application.
 
-This QBit provides global search across QQQ application tables via OpenSearch. Annotate entity classes or declare tables via config, configure a connection, produce the QBit, and gain real-time indexing, scheduled basepull catch-up, per-field relevance boosting, and a pagination-aware search API.
+Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
 
-## Core Capabilities
+Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the 1.1 line builds against 4.1.0-RC.4 from the next release candidate on.
 
-- **Real-Time Indexing**: Post-insert, post-update, and post-delete customizers index records immediately on write
-- **Basepull Safety Net**: Scheduled process catches records missed during downtime or bulk imports
-- **Field-Level Boosting**: `@QuickSearchField(weight = N)` controls relevance ranking per field
-- **Edge-Ngram Typeahead**: Custom analyzer enables prefix matching ("ord" finds "order")
-- **Paginated Search**: `QuickSearchAction` returns results with scores, highlights, total hits, and hasMore flag
-- **Config-Driven Indexing**: `SearchableTableConfig` API indexes tables from third-party QBit jars without modifying their source
-- **Extensible Publisher**: `IndexEventPublisher` interface allows swapping in RabbitMQ, Kafka, or any async transport
-- **Admin Observability**: Operational tables track index status and run history
+## What it does
 
-## Open Source & Full Control
+| Capability | How |
+|---|---|
+| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quickSearchFailedEvent` and replayed by the scheduled basepull. |
+| Basepull catch-up | Scheduled process (`schedulerName` plus `basepullRepeatSeconds`) that re-indexes rows changed since the previous run's start, with a configurable overlap, paging by primary key. |
+| Reconcile | Scheduled or on-demand process that re-indexes everything in place, then removes documents whose source row is gone, including documents of tables no longer configured. No search blackout. |
+| Full rebuild | Indexes every table into a fresh physical index and atomically swaps the alias; the old index keeps serving until the new one is complete. Also upgrades an index created by an older release. |
+| Core search | `QuickSearchRecordSearchProvider` serves QQQ's `RecordSearchAction` (`POST /qqq/v1/search`, the Next UI global search box) from OpenSearch for the indexed tables, with a timeout and an automatic fallback to core's own search. |
+| Search | `QuickSearchAction`: AND semantics, edge-ngram typeahead with a plain search analyzer, per-field weights, highlights, `tableNames` and `limitPerTable`, bounded paging, `tableLabel` on results. Only tables the session may read are searched; hits are re-read through `QueryAction` so record security locks apply. |
+| Connections | HTTP or HTTPS, basic auth, AWS IAM SigV4, custom CA or trust store, mTLS, timeouts, pool size, a transport customizer SPI for anything else. Secrets by `${env.X}` reference. |
+| Operations | `quickSearchIndex` (per-table status, counts, last run, last error), `quickSearchIndexRun` (history, purged after `runHistoryRetentionDays`), `quickSearchFailedEvent`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
 
-QBit Quick Search is 100% open source under Apache 2.0. All data stays in your OpenSearch cluster.
+## Supported platforms
 
-## Architecture
+Tested in CI against the Docker images named in `pom.xml` (`opensearch.test.image`).
 
-### Design Principles
+| Target | Status |
+|---|---|
+| Self-managed OpenSearch 2.19.x and 3.x | Supported, security plugin disabled or HTTP basic authentication, HTTP or HTTPS with a trusted, configured or (development only) unverified certificate |
+| Amazon OpenSearch Service managed domains, OpenSearch 2.x or 3.x | Supported with fine-grained access control: either an internal-database master user with `authMode BASIC`, or IAM with `authMode AWS_SIGV4` (`awsRegion`, `awsServiceName es`, optional `awsAssumeRoleArn`). Port 443 over HTTPS. Keep `bulkBatchSize` and `maxBulkRequestBytes` under the instance type's 10 MiB payload limit on small instances. |
+| Amazon OpenSearch Serverless | Not supported in 1.0 (`_delete_by_query` and `_refresh`, used by reconcile and full rebuild, are not offered there) |
+| Elasticsearch, OpenSearch 1.x | Not supported |
 
-1. **Real-time first, basepull as safety net**: Table customizers index records synchronously on every insert, update, and delete. The basepull process runs on a schedule as a catch-up mechanism for bulk imports, OpenSearch downtime, or any records the real-time path missed.
+Least-privilege role for the identity the QBit uses, on the alias and its physical indexes (`{indexName}*`): `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*`, `indices:admin/aliases`, `indices:admin/delete` (for the alias swap) and `indices:admin/mapping/get`; cluster level `cluster_composite_ops` for `_bulk`. For AWS fine-grained access control map the same to the basic-auth user or the IAM role. Treat the index as sensitive data: it holds copies of every indexed field value.
 
-2. **Annotation and config-driven discovery**: Entity classes marked with `@QuickSearchable` and `@QuickSearchField` are discovered at produce time. Tables from third-party jars can be declared via `SearchableTableConfig` in code. Both paths merge during `produce()`.
+For `AWS_SIGV4` add these optional dependencies to your host (the QBit declares them `optional`):
 
-3. **Extensible transport**: The `IndexEventPublisher` strategy interface decouples the write path from the indexing path. The default `SynchronousIndexEventPublisher` writes directly to OpenSearch. Consumers can provide a queue-backed implementation for async indexing.
-
-### Technology Stack
-
-- **Java 21** with QQQ backend modules
-- **OpenSearch 2.x** for full-text search with custom edge-ngram analyzer
-- **QQQ Framework**: Entities, processes, customizers, permissions, API layer
-
-### Module Organization
-
-```
-qbit-quick-search/
-  src/main/java/com/kingsrook/qbits/quicksearch/
-    QuickSearchQBitConfig.java        -- Configuration and validation
-    QuickSearchQBitProducer.java      -- Produces tables, processes, customizers
-    QuickSearchQBitContext.java       -- Static runtime context
-    QuickSearchableTableConfig.java   -- Per-table index configuration
-    SearchableFieldConfig.java        -- Config-driven field definition
-    SearchableTableConfig.java        -- Config-driven table definition
-    annotations/                      -- @QuickSearchable, @QuickSearchField
-    model/                            -- QuickSearchIndex, QuickSearchIndexRun entities
-    opensearch/                       -- OpenSearch client, document model, bulk results
-    actions/                          -- QuickSearchAction, input/output DTOs
-    processes/                        -- AbstractIndexingStep, BasepullIndexStep, FullReindexStep
-    customizers/                      -- Post-insert, post-update, post-delete customizers
-    publisher/                        -- IndexEventPublisher interface, SynchronousIndexEventPublisher
+```xml
+<dependency><groupId>software.amazon.awssdk</groupId><artifactId>apache-client</artifactId></dependency>
+<dependency><groupId>software.amazon.awssdk</groupId><artifactId>sts</artifactId></dependency> <!-- only for awsAssumeRoleArn -->
 ```
 
-## Getting Started
-
-### Prerequisites
-
-- **Java 21+**
-- **Maven 3.8+**
-- **OpenSearch 2.x** instance
-- **QQQ Application** (this is a QBit, not a standalone application)
-
-### Usage
-
-#### Maven dependency
+## Getting started
 
 ```xml
 <dependency>
     <groupId>com.kingsrook.qbits</groupId>
     <artifactId>qbit-quick-search</artifactId>
-    <version>0.2.0-SNAPSHOT</version>
+    <version>1.1.0-RC.1</version>
 </dependency>
 ```
 
-#### Annotate entity classes
+`1.1.0-RC.1` is the current release candidate (the 1.1 line, published from `release/1.1`; it needs QQQ 4.1.0-RC.2 or later, see [Upgrading from 1.0](#upgrading-from-10)). Hosts staying on the 1.0 line use `1.0.0-RC.2`; `1.0.0` follows once host applications have adopted it (#12).
+
+Annotate entity classes:
 
 ```java
 @QuickSearchable(tableName = "customer")
@@ -97,265 +72,204 @@ public class Customer
 }
 ```
 
-#### Minimal setup
+Produce the QBit (after your backends, tables and scheduler are in the instance):
 
 ```java
 QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("yourBackendName")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
+   .withBackendName("rdbms")
+   .withOpensearchUrl("https://search.example.com:443")
+   .withAuthMode(QuickSearchAuthMode.BASIC)
+   .withOpensearchUsername("${env.OPENSEARCH_USER}")
+   .withOpensearchPassword("${env.OPENSEARCH_PASSWORD}")
    .withOpensearchIndexName("my-app-search")
+   .withSchedulerName("myScheduler")
    .withSearchableEntityClasses(List.of(Customer.class, Order.class));
 
-new QuickSearchQBitProducer()
-   .withConfig(config)
-   .produce(qInstance);
+new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
 ```
 
-#### With authentication and SSL
+The producer implements QQQ's `QBitMetaDataProducer`, registers its own `QBitMetaData` (`com.kingsrook.qbits:quick-search`), and adds its tables, processes and app to the instance. Do not add a `QBitMetaData` for it yourself.
+
+Tables from other QBits or jars, without annotations:
 
 ```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("yourBackendName")
-   .withOpensearchHost("search.example.com")
-   .withOpensearchPort(443)
-   .withUseSsl(true)
-   .withOpensearchUsername("admin")
-   .withOpensearchPassword("secretPassword")
-   .withOpensearchIndexName("my-app-search")
-   .withSearchableEntityClasses(List.of(Customer.class));
-```
-
-#### With custom publisher (async indexing)
-
-```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("yourBackendName")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
-   .withOpensearchIndexName("my-app-search")
-   .withSearchableEntityClasses(List.of(Customer.class))
-   .withIndexEventPublisher(new RabbitMqIndexEventPublisher(connectionFactory));
-```
-
-#### Config-driven table indexing
-
-When tables come from third-party QBit jars (like qbit-crm or qbit-wms), you cannot add annotations to their entity classes. Use `SearchableTableConfig` and `SearchableFieldConfig` to declare those tables as searchable in your host application config.
-
-```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("rdbms")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
-   .withOpensearchIndexName("voyage-search")
-   .withSearchableTable("crmContact", List.of(
+config.withSearchableTable(new SearchableTableConfig("crmContact", List.of(
       new SearchableFieldConfig("firstName").withWeight(2),
       new SearchableFieldConfig("lastName").withWeight(2),
-      new SearchableFieldConfig("email").withWeight(3),
-      new SearchableFieldConfig("phone").withWeight(1)))
-   .withSearchableTable(new SearchableTableConfig("crmFormSubmission", List.of(
-      new SearchableFieldConfig("email").withWeight(3),
-      new SearchableFieldConfig("firstName").withWeight(1),
-      new SearchableFieldConfig("message").withWeight(1)))
-      .withRecordLabelFormat("%s - %s", "formType", "email"));
+      new SearchableFieldConfig("email").withWeight(3)))
+   .withRecordLabelFormat("%s %s", "firstName", "lastName"));
 ```
 
-`withSearchableTable()` accepts either a `(String tableName, List<SearchableFieldConfig> fields)` shorthand or a full `SearchableTableConfig` for advanced options.
+Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden and password fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull, and the QBit logs a warning for it at startup. Set `basepullTimestampField` to null on a config-driven table to opt out without the warning.
 
-##### SearchableFieldConfig
-
-| Method | Description |
-|--------|-------------|
-| `new SearchableFieldConfig(fieldName)` | Constructor; QQQ field name is required |
-| `.withWeight(int)` | Boost factor for relevance ranking (default 1) |
-| `.withIncludeLabel(boolean)` | Prefix indexed text with the field label (default false) |
-
-##### SearchableTableConfig
-
-| Method | Description |
-|--------|-------------|
-| `new SearchableTableConfig(tableName, fields)` | Constructor; table name and at least one field required |
-| `.withBasepullIntervalMinutes(int)` | Override basepull schedule (defaults to config-level setting) |
-| `.withBasepullTimestampField(String)` | Change detection field (default `"modifyDate"`) |
-| `.withEnabledByDefault(boolean)` | Whether indexing starts active (default `true`) |
-| `.withRecordLabelFormat(String format, String... fields)` | `String.format` pattern and field names for search result display label |
-
-##### Record label override
-
-By default, search results use QQQ's record label for display. Config-driven tables can override this with `withRecordLabelFormat()`. The format string is passed to `String.format()` with field values pulled from the record in the order specified.
+### Searching
 
 ```java
-new SearchableTableConfig("crmFormSubmission", fields)
-   .withRecordLabelFormat("%s - %s", "formType", "email");
-// produces labels like "Contact Form - user@example.com"
-```
-
-#### Mixing annotations and config
-
-Annotations and config-driven declarations can be used together. Both paths merge during `produce()`. Each table must appear in only one source; duplicate table names across annotation and config sources will produce a validation error.
-
-```java
-QuickSearchQBitConfig config = new QuickSearchQBitConfig()
-   .withBackendName("rdbms")
-   .withOpensearchHost("localhost")
-   .withOpensearchPort(9200)
-   .withOpensearchIndexName("my-app-search")
-   .withSearchableEntityClasses(List.of(Customer.class, Order.class))
-   .withSearchableTable("crmContact", List.of(
-      new SearchableFieldConfig("firstName").withWeight(2),
-      new SearchableFieldConfig("lastName").withWeight(2),
-      new SearchableFieldConfig("email").withWeight(3)));
-```
-
-## Searching
-
-Once the QBit is produced, use `QuickSearchAction` to execute searches:
-
-```java
-QuickSearchOutput output = new QuickSearchAction().execute(
-   new QuickSearchInput()
-      .withSearchTerm("widget")
-      .withLimit(25)
-      .withOffset(0));
-
-// Check results
-System.out.println("Total hits: " + output.getTotalHits());
-System.out.println("Has more: " + output.getHasMore());
+QuickSearchOutput output = new QuickSearchAction().execute(new QuickSearchInput()
+   .withSearchTerm("widget")
+   .withTableNames(List.of("customer", "order"))   // optional
+   .withLimit(25)
+   .withOffset(0));
 
 for(QuickSearchResult result : output.getResults())
 {
-   System.out.println(result.getTableName() + ":" + result.getRecordId()
-      + " - " + result.getRecordLabel()
-      + " (score: " + result.getScore() + ")");
+   result.getTableName(); result.getTableLabel(); result.getRecordId();
+   result.getRecordLabel(); result.getScore(); result.getHighlightSnippet();
 }
+output.getTotalHits(); output.getTotalHitsIsLowerBound(); output.getHasMore();
 ```
 
-To filter results to a single table:
+`limitPerTable` returns up to that many hits from each table instead of one ranked list. Terms shorter than 2 characters return nothing; longer than 100 characters are rejected. `limit` is capped at `maxSearchLimit` and `offset + limit` at 10,000.
+
+`offset` skips N accessible results: with record security locks on, hits a lock hides are not counted, so consecutive pages never overlap or skip a visible hit, and with `limitPerTable` the offset applies within each table. With locks on, `totalHits` counts hits confirmed accessible so far; `totalHitsIsLowerBound` is true when the scan stopped before reading every candidate (page full, result window, or scan budget). A single-list search reads at most 10,000 raw hits in batches of at most 1,000. In per-table mode the T tables share that budget, each getting max(10,000 / T, 2 x `limitPerTable`) hits, so one call reads at most max(10,000, T x 2 x `limitPerTable`) raw hits. Paging within a table stops past that share of raw hits, as it does past 10,000 in a single list: `hasMore` turns false and `totalHitsIsLowerBound` stays true to signal unreachable matches.
+
+Search runs as the current `QContext` session: tables the session cannot read are skipped, and with `applyRecordSecurityLocks` (default) hits are re-read through `QueryAction` so record security locks apply.
+
+### Core search (`POST /qqq/v1/search`)
+
+QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v1/search` and used by the Next UI's global search box): a case-insensitive contains-query over each table's `searchFields`. With `serveCoreRecordSearch` (default `true`) the producer registers `QuickSearchRecordSearchProvider` on the instance, and core asks it to search every table Quick Search indexes. The API and the UI do not change.
+
+- **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quickSearchIndex`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
+- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights); each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order. Core re-reads them by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, and records deleted since they were indexed drop out.
+- **Failure.** The OpenSearch request is cancelled after `recordSearchTimeoutMillis` (default 2500). Any failure (timeout, cluster down, index missing) makes core search those tables with its own contains-query instead, so search degrades but never fails. After a failure, Quick Search claims no tables for 30 seconds, so a down cluster does not add a timeout to every search.
+- **Opting out.** `withServeCoreRecordSearch(false)` leaves core search and `searchFields` untouched. If the host already registered another record-search provider, the QBit keeps it and logs a warning (an instance has one provider).
+
+`QuickSearchAction` remains the richer API (scores, highlights, pagination, totals).
+
+### Processes
+
+| Process | Trigger | What it does |
+|---|---|---|
+| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Recovers a full reindex that stopped without finishing, replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
+| Quick Search Reconcile Index | Cron via `reconcileCronExpression` when `schedulerName` is set; also on demand (optional `tableName` input) | Re-indexes everything in place, removes documents without a source row and documents of unconfigured tables |
+| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place. Fails to start while another full reindex is running |
+
+Without `schedulerName` the processes exist but nothing runs them; the QBit logs a warning at startup.
+
+### Scheduling
+
+The schedules run only when the host starts through QQQ's `QApplicationLauncher` (`com.kingsrook.qqq.middleware.javalin.QApplicationLauncher`, in `qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` (because the QBit's processes are scheduled), then the instance's runtime services. A host that starts only a server, or builds the `QInstance` itself, gets the processes but nothing runs them on a schedule, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
+
+Basepull's fixed interval runs on any QQQ scheduler. `reconcileCronExpression` needs a scheduler that supports cron, which in QQQ 4.1 means a `QuartzSchedulerMetaData`; a `SimpleSchedulerMetaData` runs fixed intervals only. The expression uses Quartz syntax, seconds first (`0 0 3 * * ?` is 03:00 every day), and needs `reconcileCronTimeZoneId`, a Java time zone ID such as `UTC` or `America/Chicago`; QQQ has no default zone. `produce()` rejects a cron without a zone, an invalid expression or zone, and a cron on a scheduler in the instance that does not support cron.
 
 ```java
-QuickSearchOutput output = new QuickSearchAction().execute(
-   new QuickSearchInput()
-      .withSearchTerm("blue")
-      .withTableName("customer")
-      .withLimit(10));
+qInstance.addScheduler(new QuartzSchedulerMetaData().withName("quartz").withProperties(quartzProperties));
+
+config.withSchedulerName("quartz")
+   .withReconcileCronExpression("0 0 3 * * ?")
+   .withReconcileCronTimeZoneId("America/Chicago");
 ```
 
-### Triggering a Full Reindex
+### Diagnostic tools
 
-The full reindex process can be triggered programmatically:
+With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate the search and debug it from the admin UI. They are table and record actions (never left-nav entries), gated by permissions, recorded in `quickSearchDiagnosticEvent`, limited to `diagnosticRunsPerMinute` runs per user and tool, and read-only except Reindex in Search.
 
-```java
-RunBackendStepInput input = new RunBackendStepInput();
-input.addValue("tableName", "customer"); // optional: omit to reindex all tables
-RunBackendStepOutput output = new RunBackendStepOutput();
-new FullReindexStep().run(input, output);
-```
+| Tool | Where | What it does |
+|---|---|---|
+| Search Playground | Quick Search Index, Actions (select rows to pre-fill their tables) | Runs a term the way the global search does, as you: the path each table takes (OpenSearch provider, core database search, or not searched) and why; the provider's own request, timed; the global search results you would see; per indexed table, the production query with scores, highlights, matched fields, which hits you can read and which the global search shows, the query DSL and the raw response (hits you cannot read are redacted); the term's tokens; and `explain` for a hit (`table:id`). The form stays open, so you can change the term and run again. |
+| Validate Search Index | Quick Search Index, Actions; also scheduled by `validationCronExpression` | Cluster and index health, the alias, leftover physical indexes from interrupted rebuilds, the live mapping (version, field types, analyzers, the `fieldValues` template) and analysis settings, then per table: database rows vs documents, a sample of records checked present and current in the index, a random sample of documents checked to still have a record, the index row's status and last run, and waiting failed events. Each finding is OK, INFO, WARN or FAIL with what to do. The report is saved as a `quickSearchValidationRun` row (HTML for the record view, JSON for tools). Database reads run as the system user so counts compare like with like; the report holds counts, ids and field names, never record values. One validation runs at a time per server. |
+| Test Search Analyzer | Quick Search Index, Actions | `_analyze` on the live index: the tokens of a text as indexed (edge n-grams of 2 to 20 characters) and as a query, and, given a term, whether every term token is among the text's indexed tokens. |
+| Inspect Search Document | Every indexed table's records (Actions), and Quick Search Failed Event rows | The record's document in the index next to the document indexing would write from the database now, field by field (current, stale or missing), with versions and the raw document. Enter a term to see why the record does or does not match it: term length, table enabled, read permission, document present and current, each term token found or not (and why: one character, longer than 20, not in the text, or only in the database's newer values), `explain`, and the record's rank against the number of ids the global search takes. |
+| Reindex in Search | Every indexed table's records, and Quick Search Failed Event rows | Rebuild the selected records' documents from the database now (versioned writes, so a newer document is never overwritten); a record that no longer exists has its document removed; a record you cannot read is skipped. Asks to confirm. Needs the admin permission. |
+| Search Diagnostics widget | The Quick Search Admin app home (hosts with their own home add `quickSearchDiagnostics`) | Whether the global search is served by OpenSearch on this server or backing off (and why), failure and fallback counts, the latest search failures from every server, and the latest validation run. |
 
-Or through the QQQ admin UI via the "Full Reindex" process.
+Search failures are recorded too: when the provider's OpenSearch request fails or times out (and core falls back to its own search), or `QuickSearchAction`'s search fails, a `SEARCH_FAILED` event keeps the tables, duration, error class and message, and the server. The user's term is never stored, only its length. At most 120 events are written per server per minute.
 
-## Data Model
+Permissions: the read-only tools and the widget use `debugPermissionRules` (default `quickSearchDebug.hasAccess`); Reindex in Search uses `adminPermissionRules` (`quickSearchAdmin.hasAccess`); the two history tables use `tablePermissionRules` (grant `.read` only: rows are written by the QBit). Impersonation ("search as user X") is deliberately not offered: a fabricated session would not carry the real user's security key values, so its results would mislead.
 
-### Tables (2)
+### Record developer panel
 
-| Table | Description |
-|-------|-------------|
-| `quickSearchIndex` | Per-table index configuration, status, and basepull tracking |
-| `quickSearchIndexRun` | Individual indexing run history with record counts and error tracking |
+With QQQ 4.1.0-RC.4 or later the QBit registers a record developer panel provider (`QuickSearchDeveloperPanelProvider`, id `quickSearch`, in QQQ's `DeveloperPanelsMetaData`). Frontends that render developer panels (the Next UI, `/app/{table}/{id}/dev`) show a "Quick Search" panel on every record of an indexed table, to users who may read Quick Search Index. Every part is permission-checked:
 
-### Processes (2)
+| Part | Shown when |
+|---|---|
+| Index: enabled, status, record count, last basepull, last full reindex, searchable fields, link to the index row | always (read on Quick Search Index) |
+| Latest index run: type, status, start and end, processed, indexed, errors | the user may read Quick Search Index Run |
+| Search document: verdict (current, stale, missing) and a Field / Database / Index / Match table | `enableDiagnosticTools` and the debug permission (`debugPermissionRules`); otherwise a note says why it is missing |
+| Failed index events of the record | the user may read Quick Search Failed Event and the record has any |
+| Actions: Inspect search document, Reindex this record | the record actions exist and the user may run them (debug and admin permission) |
+| Link to Quick Search Admin | the user may open the app |
 
-| Process | Description |
-|---------|-------------|
-| Basepull Index | Scheduled: queries each source table for records modified since last run, indexes in batches |
-| Full Reindex | On-demand: deletes and re-indexes all records for one or all tables |
+The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned off in Quick Search Index), or OpenSearch unavailable. When OpenSearch cannot be reached the panel still shows the database-side state, with a warning in place of the document comparison. Hosts without a developer-panel frontend are unaffected.
 
-## Configuration Reference
+## Configuration reference
 
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `backendName` | `String` | required | QQQ backend for operational tables |
-| `opensearchHost` | `String` | required | OpenSearch hostname |
-| `opensearchPort` | `Integer` | required | OpenSearch port |
-| `opensearchIndexName` | `String` | required | OpenSearch index name |
-| `searchableEntityClasses` | `List<Class<?>>` | none | Entity classes with `@QuickSearchable` |
-| `searchableTables` | `List<SearchableTableConfig>` | none | Config-driven table definitions |
-| `opensearchUsername` | `String` | none | HTTP basic auth username |
-| `opensearchPassword` | `String` | none | HTTP basic auth password |
-| `useSsl` | `Boolean` | `false` | Use HTTPS |
-| `tableNamePrefix` | `String` | none | Prefix for produced table names |
-| `enableRealTimeIndexing` | `Boolean` | `true` | Register post-insert/update/delete customizers |
-| `enableScheduledProcesses` | `Boolean` | `true` | Register scheduled basepull process |
-| `defaultBasepullIntervalMinutes` | `Integer` | `5` | Default basepull schedule |
-| `bulkBatchSize` | `Integer` | `500` | Documents per OpenSearch bulk request |
-| `sourceBatchSize` | `Integer` | `1000` | Records per source table query batch |
-| `indexEventPublisher` | `IndexEventPublisher` | sync | Custom publisher for async transport |
+| Field | Default | Description |
+|---|---|---|
+| `backendName` | required | QQQ backend for the operational tables |
+| `opensearchUrl` | | `https://host[:port]`; preferred over `opensearchHost`, `opensearchPort`, `useSsl`; credentials in the URL are rejected |
+| `opensearchHost`, `opensearchPort`, `useSsl` | `useSsl=false` | Legacy connection fields |
+| `opensearchIndexName` | required | Alias name owned by this QBit; lowercase, OpenSearch naming rules |
+| `authMode` | inferred | `NONE`, `BASIC` (inferred when credentials are set), `AWS_SIGV4` |
+| `opensearchUsername`, `opensearchPassword` | | BASIC credentials; `${env.X}` references recommended (an empty value counts as missing); refused over plain HTTP unless `allowPlaintextCredentials` |
+| `awsRegion`, `awsServiceName`, `awsAssumeRoleArn` | region from `AWS_REGION`; `es` | AWS_SIGV4 settings |
+| `tls` | JVM defaults | `caCertificatePath` or `trustStorePath`/`trustStorePassword`/`trustStoreType`, `keyStorePath`/`keyStorePassword`/`keyStoreType` (mTLS), `hostnameVerification` (`false` logs a warning), `insecureSkipVerify` (loopback only unless `allowInsecureInProduction`) |
+| `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections` | 5000, 60000, 30 | Transport settings |
+| `transportCustomizer` | | `QCodeReference` to an `OpenSearchTransportCustomizer` (bearer tokens, API keys, interceptors) |
+| `startupMode` | `FAIL_FAST` | `FAIL_FAST` fails `produce()` when the cluster is unreachable; `DEGRADED` boots and retries on first use |
+| `enableRealTimeIndexing` | `true` | Register the record-change listener |
+| `enableBasepullProcess`, `enableMaintenanceProcesses` | `true` | Register basepull; register full reindex and reconcile. `enableScheduledProcesses` is a deprecated alias for both |
+| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none, none | Scheduling; a cron needs the time zone ID and a Quartz scheduler, and nothing runs without `QApplicationLauncher` (see [Scheduling](#scheduling)) |
+| `basepullOverlapSeconds` | 300 | Re-read window before the previous run start |
+| `defaultBasepullIntervalMinutes` | 5 | Per-table due interval for config-driven tables |
+| `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
+| `fullReindexStaleMinutes` | 30 | A running full reindex that shows no sign of life for this long is treated as stopped: it no longer blocks a new one, and basepull recovers its tables. Keep it well above the time one page of `sourceBatchSize` records takes; null means the default |
+| `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
+| `maxFieldLength` | 10000 | Truncate each indexed value |
+| `maxSearchLimit` | 100 | Largest page size |
+| `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
+| `serveCoreRecordSearch` | `true` | Register `QuickSearchRecordSearchProvider` so core `POST /qqq/v1/search` uses OpenSearch for indexed tables, and give those tables `searchFields` when they have none |
+| `recordSearchTimeoutMillis` | 2500 | Wall-clock limit on the OpenSearch request behind a core search; past it, core falls back to its own search |
+| `enableDiagnosticTools` | `true` | Add the diagnostic tools, the `quickSearchDiagnosticEvent` and `quickSearchValidationRun` tables and the diagnostics widget |
+| `enableRecordDiagnosticActions` | `true` | With the tools on, add Inspect Search Document and Reindex in Search to every indexed table |
+| `validationCronExpression`, `validationCronTimeZoneId` | none | Schedule Validate Search Index; same rules as the reconcile schedule (a cron-capable scheduler such as Quartz, a valid Quartz expression, a time zone ID) |
+| `validationSampleSize` | 50 | Records and documents sampled per table by validation (at most 1000) |
+| `diagnosticRunsPerMinute` | 30 | Per-user, per-tool limit on diagnostic runs |
+| `debugPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchDebug` | Rules for the read-only diagnostic tools and the widget |
+| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchAdmin` | Rules for the app, the maintenance processes and Reindex in Search |
+| `tablePermissionRules` | `READ_WRITE_PERMISSIONS` | Rules for the operational tables |
+| `tableMetaDataCustomizer` | | Applied to every produced table (backend details, for example) |
+| `searchableEntityClasses`, `searchableTables` | at least one | What to index |
+| `indexEventPublisher` | synchronous | Replace the publisher (experimental SPI; see below) |
 
-At least one of `searchableEntityClasses` or `searchableTables` must be provided. Both can be used together.
+`@QuickSearchable(tableName, fields, basepullIntervalMinutes, basepullTimestampField, enabledByDefault)` and `@QuickSearchField(weight, includeLabel)` keep their 0.x meaning.
 
-## Annotations
+## Operational tables
 
-### `@QuickSearchable`
+- `quickSearchIndex`: one row per table with `enabled` (honoured by indexing and search, cached for one minute), `basepullIntervalMinutes`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`, `NEEDS_REINDEX` when the configured fields changed since the row was created, or `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
+- `quickSearchIndexRun`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status`, counts, `errorMessage`.
+- `quickSearchDiagnosticEvent` (with `enableDiagnosticTools`): `eventType` (`SEARCH_FAILED`, `TOOL_RUN`, `REINDEX`, `RATE_LIMITED`), `source`, `outcome`, `userName`, `tableNames`, `searchTerm` (tool runs only), `durationMillis`, `errorClass`, `errorMessage` (text), `detail` (text), `hostName`, `createDate`. Purged after `runHistoryRetentionDays`.
+- `quickSearchValidationRun` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
+- `quickSearchFailedEvent`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
 
-| Attribute | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `tableName` | `String` | required | QQQ table name |
-| `fields` | `String[]` | `{}` | Fallback field list if no `@QuickSearchField` annotations |
-| `basepullIntervalMinutes` | `int` | `5` | Basepull schedule for this table |
-| `basepullTimestampField` | `String` | `"modifyDate"` | Field for change detection |
-| `enabledByDefault` | `boolean` | `true` | Active on first produce |
+## Extending
 
-### `@QuickSearchField`
+`IndexEventPublisher` lets a host replace the synchronous publisher with a queue. In 1.0 the SPI is experimental: `IndexEvent` carries the table, record id, action and the record snapshot, but no sequence number, so an asynchronous consumer must order INDEX and DELETE for the same record itself. `OpenSearchTransportCustomizer` customizes the HTTP transport.
 
-| Attribute | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `weight` | `int` | `1` | Boost factor for search relevance |
-| `includeLabel` | `boolean` | `false` | Prefix value with field label in indexed text |
-
-## Companion QBit Integration
-
-### qbit-crm
-
-Index `crmContact`, `crmCompany`, and `crmDeal` for full-text global search across CRM records.
-
-### qbit-wms
-
-Index `wmsItem` and `wmsLocation` for warehouse item and location search.
-
-## Testing
+## Building
 
 ```bash
-mvn test                    # Run all 174 unit tests
-mvn verify                  # Run unit tests + integration tests (requires Docker)
-mvn test -Dtest=ClassName   # Run single test class
+./mvnw test                                 # unit tests (memory backend, mocked client)
+CI=true ./mvnw verify                       # plus Testcontainers integration tests and the coverage gate (Docker required)
+./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
 ```
 
-### Coverage
+Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.4 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
 
-- **70%+ instruction coverage** (JaCoCo enforced)
-- **90%+ class coverage** (JaCoCo enforced)
-- All unit tests run in-memory via MemoryRecordStore (no OpenSearch required)
-- Integration tests use Testcontainers with real OpenSearch
+## Upgrading from 1.0
 
-## Documentation
+See [docs/MIGRATION-1.1.md](docs/MIGRATION-1.1.md). In short: 1.1 needs the QQQ release with the record-search provider SPI; core search starts using OpenSearch for indexed tables by default, and indexed tables gain `searchFields` (set `serveCoreRecordSearch(false)` to keep 1.0 behaviour).
 
-- **[QQQ Wiki](https://github.com/Kingsrook/qqq/wiki)** - Framework documentation
-- **[QBit Development Guide](https://github.com/Kingsrook/qqq/wiki/QBit-Development)** - How QBits work
+## Upgrading from 0.x
 
-## Contributing
+See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, grant `quickSearchAdmin.hasAccess` and the table permissions, add the new operational table columns and `quickSearchFailedEvent`, and run a full reindex once so the index gets the new mapping.
 
-QBit Quick Search is open source and welcomes contributions.
+## Adopting in another app
 
-- **[Report Issues](https://github.com/QRun-IO/qbit-quick-search/issues)** - Bug reports and feature requests
-- **[QQQ Contribution Guide](https://github.com/Kingsrook/qqq/wiki/Contribution-Guidelines)** - How to contribute
+Follow [Getting started](#getting-started), the [Configuration reference](#configuration-reference) and [Operational tables](#operational-tables) (the host application creates the three tables; this QBit ships no DDL), then run a full reindex. Hosts on 1.0 or 0.x should read the upgrade sections above.
 
-## About Kingsrook
-
-QBit Quick Search is built by **[Kingsrook](https://qrun.io)** - making engineers more productive through intelligent automation and developer tools.
-
-- **Website**: [https://qrun.io](https://qrun.io)
-- **Contact**: [contact@kingsrook.com](mailto:contact@kingsrook.com)
-- **GitHub**: [https://github.com/QRun-IO](https://github.com/QRun-IO)
+See also the [QQQ adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) for the full per-feature list (QQQ upgrade, Next dashboard, QBits, migrations, permissions).
 
 ## License
 
-This project is licensed under the **Apache License, Version 2.0** - see the [LICENSE](LICENSE) file for details.
+Apache License, Version 2.0. See [LICENSE](LICENSE).

checkstyle/config.xml

@@ -6,18 +6,17 @@
   ~ contact@kingsrook.com
   ~ https://github.com/Kingsrook/
   ~
-  ~ This program is free software: you can redistribute it and/or modify
-  ~ it under the terms of the GNU Affero General Public License as
-  ~ published by the Free Software Foundation, either version 3 of the
-  ~ License, or (at your option) any later version.
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
   ~
-  ~ This program is distributed in the hope that it will be useful,
-  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
-  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  ~ GNU Affero General Public License for more details.
+  ~     https://www.apache.org/licenses/LICENSE-2.0
   ~
-  ~ You should have received a copy of the GNU Affero General Public License
-  ~ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
   -->
 
 <!DOCTYPE module PUBLIC

checkstyle/license.txt

@@ -5,16 +5,15 @@
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
  *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */

com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -1,288 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.customizers;
-
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Common interface used by all (core) TableCustomizer types (e.g., post-query,
- ** and {pre,post}-{insert,update,delete}.
- **
- ** Note that the abstract-base classes for each action still exist, though have
- ** been back-ported to be implementors of this interface.  The action classes
- ** will now expect this type, and call this type's methods.
- **
- *******************************************************************************/
-public interface TableCustomizerInterface
-{
-   QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
-
-   /*******************************************************************************
-    ** custom actions to run after a query (or get!) takes place.
-    **
-    *******************************************************************************/
-   default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postQuery is running...  Probably not expected!", logPair("tableName", queryInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions before an insert takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview field, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (the inputs to
-    ** the insert action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly manipulating values (`setValue`)
-    ** - possibly throwing an exception - if you really don't want the insert operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
-    *******************************************************************************/
-   default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      try
-      {
-         return (preInsertOrUpdate(insertInput, records, isPreview, Optional.empty()));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
-   {
-      return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions after an insert takes place.
-    **
-    ** General implementation would be, to iterate over the records (the outputs of
-    ** the insert action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly throwing an exception - though doing so won't stop the update, and instead
-    **   will just set a warning on all of the updated records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back to the caller.
-    *******************************************************************************/
-   default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
-   {
-      try
-      {
-         return (postInsertOrUpdate(insertInput, records, Optional.empty()));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    ** custom actions before an update takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview field, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (the inputs to
-    ** the update action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly manipulating values (`setValue`)
-    ** - possibly throwing an exception - if you really don't want the update operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
-    **
-    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
-    ** available (if the backend supports it)
-    *******************************************************************************/
-   default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      try
-      {
-         return (preInsertOrUpdate(updateInput, records, isPreview, oldRecordList));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    ** custom actions after an update takes place.
-    **
-    ** General implementation would be, to iterate over the records (the outputs of
-    ** the update action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
-    ** - possibly throwing an exception - though doing so won't stop the update, and instead
-    **   will just set a warning on all of the updated records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back to the caller.
-    **
-    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
-    ** available (if the backend supports it).
-    *******************************************************************************/
-   default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      try
-      {
-         return (postInsertOrUpdate(updateInput, records, oldRecordList));
-      }
-      catch(NotImplementedHereException e)
-      {
-         LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-         return (records);
-      }
-   }
-
-
-   /*******************************************************************************
-    ** Custom actions before a delete takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview param, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (which the DeleteAction
-    ** would look up based on the inputs to the delete action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly throwing an exception - if you really don't want the delete operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) - this is how errors
-    **   and warnings are propagated to the DeleteAction.  Note that any records with
-    **   an error will NOT proceed to the backend's delete interface - but those with
-    **   warnings will.
-    *******************************************************************************/
-   default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      LOG.info("A default implementation of preDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** Custom actions after a delete takes place.
-    **
-    ** General implementation would be, to iterate over the records (ones which didn't
-    ** have a delete error), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
-    ** - possibly throwing an exception - though doing so won't stop the delete, and instead
-    **   will just set a warning on all of the deleted records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back
-    **   to the caller - this is how errors and warnings are propagated .
-    *******************************************************************************/
-   default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
-      return (records);
-   }
-
-
-   /***************************************************************************
-    ** Optional method to override in a customizer that does the same thing for
-    ** both preInsert & preUpdate.
-    ***************************************************************************/
-   default List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      throw NotImplementedHereException.instance;
-   }
-
-
-   /***************************************************************************
-    ** Optional method to override in a customizer that does the same thing for
-    ** both postInsert & postUpdate.
-    ***************************************************************************/
-   default List<QRecord> postInsertOrUpdate(AbstractActionInput input, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      throw NotImplementedHereException.instance;
-   }
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   default Optional<Map<Serializable, QRecord>> oldRecordListToMap(String primaryKeyField, Optional<List<QRecord>> oldRecordList)
-   {
-      if(oldRecordList.isPresent())
-      {
-         return (Optional.of(CollectionUtils.listToMap(oldRecordList.get(), r -> r.getValue(primaryKeyField))));
-      }
-      else
-      {
-         return (Optional.empty());
-      }
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   class NotImplementedHereException extends QException
-   {
-      private static NotImplementedHereException instance = new NotImplementedHereException();
-
-
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      private NotImplementedHereException()
-      {
-         super("Not implemented here");
-      }
-   }
-}

com/kingsrook/qqq/backend/core/model/metadata/SupplementalCustomizerType.java

@@ -1,35 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.model.metadata;
-
-
-/*******************************************************************************
- ** Interface to be implemented by enums of the possible supplemental types
- **
- *******************************************************************************/
-public interface SupplementalCustomizerType
-{
-   /*******************************************************************************
-    ** allows implementing enums to return the expected class being used
-    *******************************************************************************/
-   public Class<?> getCodeReferenceType();
-}

docs/MIGRATION-1.0.md

@@ -0,0 +1,96 @@
+# Migrating from 0.x to 1.0
+
+1.0 is a trust and platform release. Hosts that used the 0.2 fluent API keep compiling; the steps below are what changes at runtime.
+
+## Prerequisites
+
+- QQQ 4.1 (the QBit uses record-change listeners, after-commit callbacks and runtime services, none of which exist in 4.0). Until `qbit-build-parent` 2.1.0 is released, the QBit builds against 4.1.0-RC.1.
+- OpenSearch 2.19.x or 3.x, or an Amazon OpenSearch Service managed domain. Serverless is not supported.
+
+## Required changes
+
+1. **Remove any manual `QBitMetaData` registration.** The producer now registers `com.kingsrook.qbits:quick-search` itself; a second `addQBit` with that name throws at startup. (qqq-all's `DemoQBits` did this in 0.x.)
+2. **Decide the startup mode.** `startupMode` defaults to `FAIL_FAST`: `produce()` fails when OpenSearch is unreachable. Demo profiles that may run without a cluster should set `withStartupMode(QuickSearchStartupMode.DEGRADED)`.
+3. **Run a full reindex once after upgrading.** The index mapping changed (version 2: search analyzer, text-typed `fieldValues`). The QBit logs a warning when it finds an index with the old mapping; "Quick Search Full Reindex" with no table name builds a new physical index and swaps the configured name to an alias. Search keeps working during the rebuild. The identity the QBit uses needs index create, delete and alias privileges for this (see the README role).
+4. **Set `schedulerName` and start the host through `QApplicationLauncher`** (and optionally `basepullRepeatSeconds`, or `reconcileCronExpression` with `reconcileCronTimeZoneId` on a Quartz scheduler) so basepull and reconcile actually run. In 0.x they were never scheduled. See [Scheduling](#scheduling).
+5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `withAllowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
+6. **Grant the permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant the names listed under [Permissions](#permissions) to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
+7. **Update the operational tables before starting 1.0.** `quickSearchIndex` gains five columns, `searchableFieldsJson` changes from `STRING` to `TEXT`, and `tableName` becomes required with a unique key. `quickSearchIndexRun.errorMessage` changes from `STRING` to `TEXT`. There is a new `quickSearchFailedEvent` table. A host that checks its columns at startup rejects the 0.x tables. See [Schema changes](#schema-changes).
+
+## Permissions
+
+QQQ names a permission `{baseName}.{type}`. The admin rules use the base name `quickSearchAdmin`; the table rules have no base name, so each table's name is its base name. With the default rules, grant these to operators:
+
+| Permission | Covers |
+|---|---|
+| `quickSearchAdmin.hasAccess` | The `quickSearchAdmin` app and the three processes (`quickSearchBasepullIndex`, `quickSearchReconcileIndex`, `quickSearchFullReindex`) |
+| `quickSearchIndex.read`, `quickSearchIndex.write` | The `quickSearchIndex` table |
+| `quickSearchIndexRun.read`, `quickSearchIndexRun.write` | The `quickSearchIndexRun` table |
+| `quickSearchFailedEvent.read`, `quickSearchFailedEvent.write` | The `quickSearchFailedEvent` table |
+
+The tables use `READ_WRITE_PERMISSIONS`: `.read` covers query, get and count, and `.write` covers insert, edit and delete. If you set `tablePermissionRules` to `READ_INSERT_EDIT_DELETE_PERMISSIONS`, grant `.read`, `.insert`, `.edit` and `.delete` on each table instead. With `tableNamePrefix` set, the table permissions carry the prefix (`{prefix}quickSearchIndex.read`); the admin permission stays `quickSearchAdmin.hasAccess`.
+
+Users who only search need none of these. Search checks the read permission of each searched table as that table's own rules name it (for example `customer.read`). The QBit's own reads and writes of its tables go through QQQ's table actions, which do not check permissions, so real-time indexing does not depend on these grants.
+
+For a worked example, qqq-all's RC.2 adoption (QRun-IO/qqq-all#30) adds `infra/postgres/06-quick-search-permissions.sql`, which grants the seven names above to its admin role.
+
+## Schema changes
+
+The QBit declares its tables in metadata and ships no DDL. A backend that creates tables from metadata gets the 1.0 shape on a new database. A host that manages its own schema, or checks columns at startup, must change the 0.x tables before starting 1.0. Names below are QQQ field names and types; your backend's naming decides the physical table and column names.
+
+`quickSearchIndex`:
+
+| Field | Type | Change from 0.x |
+|---|---|---|
+| `lastReconcileTime` | `DATE_TIME` | New |
+| `lastRunStatus` | `STRING` | New. Run type and outcome, for example `FULL_REINDEX COMPLETED` (at most 22 characters) |
+| `lastErrorMessage` | `TEXT` | New. Up to 4,000 characters |
+| `documentCount` | `INTEGER` | New |
+| `realTimeErrorCount` | `INTEGER` | New |
+| `searchableFieldsJson` | `TEXT` | Was `STRING` |
+| `tableName` | `STRING` | Now required, with a unique key |
+
+`status` holds `ACTIVE`, `NEEDS_REINDEX` (new in 1.0) or `REBUILDING` (new in RC.2), so it needs at least 13 characters. QQQ checks the `tableName` unique key on insert; a unique index in the database also stops two concurrent first runs from both inserting. 0.x could leave duplicate rows, so keep one row per table before adding the index.
+
+`quickSearchIndexRun` has no new columns. `errorMessage` changed from `STRING` to `TEXT` and holds up to 4,000 characters.
+
+`quickSearchFailedEvent` is new:
+
+| Field | Type | Notes |
+|---|---|---|
+| `id` | `INTEGER` | Primary key, generated by the backend like the other two tables |
+| `tableName` | `STRING` | Source table |
+| `recordId` | `STRING` | Source record's primary key as text |
+| `action` | `STRING` | `INDEX` or `DELETE` |
+| `errorMessage` | `TEXT` | Up to 4,000 characters |
+| `attempts` | `INTEGER` | Replay attempts |
+| `status` | `STRING` | `PENDING`, `EXHAUSTED` or `AWAITING_REINDEX` (new in RC.2); size it for at least 16 characters |
+| `createDate`, `modifyDate` | `DATE_TIME` | |
+
+The QBit sets no maximum length on its `STRING` fields. Map `STRING` to a character column sized for the values above, `TEXT` to a type that holds at least 4,000 characters, and `DATE_TIME` to a timestamp, the same way your other QQQ tables do.
+
+If your backend creates tables from metadata and you do not need the 0.x run history, you can instead drop the 0.x `quickSearchIndex` and `quickSearchIndexRun` tables and let startup recreate them, as qqq-all#30 documents. The next basepull, reconcile or full reindex recreates each `quickSearchIndex` row, with `enabled` and `basepullIntervalMinutes` back at their configured defaults.
+
+## Behaviour changes to be aware of
+
+- Real-time indexing happens after commit. Code that searched for a record inside the same transaction that inserted it will not find it until commit.
+- Real-time failures no longer vanish: they land in `quickSearchFailedEvent` and are replayed by the next basepull. Review that table after an OpenSearch outage.
+- Inserted and updated records are re-read inside the write's transaction, with display values and possible-value labels, before they are indexed. That is one extra query per write batch on an indexed table, and it runs the host's possible-value providers. If the re-read fails, the write still succeeds and the records are replayed by basepull.
+- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile. The QBit logs a warning for each such table at startup, for annotated and config-driven tables alike; a config-driven table with `basepullTimestampField` set to null has opted out and gets no warning.
+- `limit` is capped (default 100), `offset + limit` at 10,000, and search terms must be 2 to 100 characters.
+- Query semantics changed from OR over n-grams to AND over whole terms with prefix matching: "blue widget" now requires both words.
+- `enableScheduledProcesses` still works and sets both `enableBasepullProcess` and `enableMaintenanceProcesses`.
+- `QuickSearchQBitContext` is deprecated and removed in 1.1; use `QuickSearchRuntime.get()`.
+
+## Scheduling
+
+Setting `schedulerName` puts schedules on the basepull and reconcile processes; QQQ runs them only once something starts its `QScheduleManager`.
+
+- **Start through `QApplicationLauncher`.** In QQQ 4.1 that is `com.kingsrook.qqq.middleware.javalin.QApplicationLauncher` (`qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` when any process is scheduled, then the instance's runtime services, including this QBit's. A host that starts only a `QApplicationJavalinServer`, or builds its `QInstance` without the launcher, gets the processes but no scheduled runs, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
+- **A reconcile cron needs a Quartz scheduler.** Basepull's fixed interval runs on any QQQ scheduler, but only `QuartzSchedulerMetaData` supports cron; QQQ's `SimpleSchedulerMetaData` runs fixed intervals only, and a cron schedule on it fails when the schedule manager starts.
+- **A reconcile cron needs a time zone.** Set `reconcileCronTimeZoneId` to a Java time zone ID (`UTC`, `America/Chicago`). QQQ has no default zone: its instance validation rejects a cron schedule without one at startup. `produce()` now catches this first, along with an invalid Quartz expression (seconds first, for example `0 0 3 * * ?`), an unknown zone ID, and a cron on a scheduler in the instance that does not support cron.
+
+## Removed
+
+- `QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`. Hosts that referenced them (for example to compose with their own customizers) should remove those references; the record-change listener coexists with host customizers.
+- `QuickSearchOpenSearchClient.search(String term, String tableName, ...)`; use the collection overload.

docs/MIGRATION-1.1.md

@@ -0,0 +1,39 @@
+# Migrating from 1.0 to 1.1
+
+1.1 lets QQQ core's record search (`POST /qqq/v1/search`, the Next UI global search box) use the Quick Search index. Hosts that do nothing get it by default.
+
+## Prerequisites
+
+- The QQQ release that carries the record-search provider SPI (`RecordSearchProviderInterface`, `QInstance.withRecordSearchProvider`), first released in QQQ `4.1.0-RC.2`. 1.1 builds against `4.1.0-RC.2` through the `qqq-snapshot` profile.
+
+## What changes at startup
+
+1. **The QBit registers a record-search provider.** `QuickSearchRecordSearchProvider` becomes the instance's `recordSearchProvider`. If the host already registered a different provider, the QBit keeps it and logs a warning; core search then does not use OpenSearch.
+2. **Indexed tables get `searchFields`.** Core only searches tables with `searchFields`. Each indexed table that has none gets its visible string and integer searchable fields. Tables that already have `searchFields` keep them. A table added to the instance after the QBit is produced gets none, so add tables first (as for field validation) or set `searchFields` yourself.
+3. **Password fields are no longer indexed.** Fields of type `PASSWORD` are now dropped from the searchable fields like hidden fields. If a 1.0 index holds such values, run "Quick Search Reconcile Index" (or a full reindex) to rewrite the documents without them.
+
+## Behaviour to be aware of
+
+- Next UI search results for indexed tables now come from OpenSearch: all words must match (in any searchable field), with prefix matching and field weights, instead of a contains-match on one field. Tables not indexed by Quick Search keep core's search.
+- Results still pass core's security: core re-reads every hit through `QueryAction`, so table permissions and record security locks apply, and records deleted since they were indexed do not appear.
+- Records are only as fresh as the index: a record updated in a way that is not yet indexed (for example while real-time indexing is off) is found by its indexed text.
+- When OpenSearch is slow or down, core waits at most `recordSearchTimeoutMillis` (default 2500) and then searches with its own contains-query; for the next 30 seconds it skips OpenSearch entirely.
+
+## Opting out
+
+```java
+new QuickSearchQBitConfig()
+   // ...
+   .withServeCoreRecordSearch(false);
+```
+
+restores 1.0 behaviour: no provider, no `searchFields` changes.
+
+## Diagnostic tools (1.1.0-RC.2)
+
+The QBit now adds admin tools to validate and debug search (see the README, "Diagnostic tools"). They are on by default.
+
+- **Two new operational tables**, on `backendName` like the others: `quickSearchDiagnosticEvent` and `quickSearchValidationRun` (columns in the README, "Operational tables"). Create them in the host's schema, with the naming the host applies to the other operational tables (`tableMetaDataCustomizer` applies to them too). Until they exist, writing events and saving reports fails quietly (logged at warn) and the tools still run.
+- **Permissions.** Grant `quickSearchDebug.hasAccess` to the people who may use the read-only tools and see the widget, `quickSearchAdmin.hasAccess` for Reindex in Search, and `.read` on the two new tables (the QBit writes them; users never do).
+- **New processes** have a `tableName` (Quick Search Index, Quick Search Failed Event, and each indexed table), so they appear as table and record actions. They are not added to the admin app's children; a host that strips processes from its nav needs no change.
+- **Opting out.** `withEnableDiagnosticTools(false)` adds none of it; `withEnableRecordDiagnosticActions(false)` keeps the tools on Quick Search Index but adds nothing to the indexed tables.

docs/PLAN-1.0-release.md

@@ -0,0 +1,50 @@
+# PLAN: Quick Search 1.0 release readiness
+
+> Executed natively in-session, task by task, test-first, one commit per task on `feature/GH-1-0-release-readiness`. Spec: `docs/initiatives/quick-search-1.0.md` (PRD) and `docs/audits/2026-10-04/README.md` (blockers B1 to B25). Decisions D1 to D11 are resolved with the PRD's recommendations unless noted under "Decisions applied".
+
+## Goal
+
+Close every audit blocker and ship a QBit that is a standard QQQ 4.1 citizen, keeps a trustworthy index, searches safely, connects to real OpenSearch deployments, and documents all of it.
+
+## Decisions applied
+
+| ID | Resolution used here |
+|---|---|
+| D1 | API-only in 1.0; `QuickSearchResult` gains `tableLabel`, input gains `tableNames` and `limitPerTable`; the core SPI is proposed separately (TODO) |
+| D2 | 1.0.0-SNAPSHOT on this branch; 0.3.0 compat release remains the qqq#921 chain's job |
+| D3 | Table-level READ filter plus `QueryAction` post-filter by primary key (`applyRecordSecurityLocks`, default true) |
+| D4 | `schedulerName` null leaves processes unscheduled with a startup WARN (not an error), so qqq-all keeps booting; set it to schedule |
+| D5 | Full reindex of all tables builds a fresh physical index and swaps the alias; a single-table full reindex uses the reconcile algorithm (index in place, then delete stale) |
+| D6 | `authMode` `AWS_SIGV4` included with optional AWS SDK v2 dependencies |
+| D7 | `RecordChangeListenerInterface` plus `addAfterCommitCallback` |
+| D8 | Listener never fails the write; failures go to `quickSearchFailedEvent`, replayed by the scheduled basepull |
+| D9 | Supported platforms statement as in the audit |
+| D10 | 4.1-only: the build uses qqq 4.1.0-RC.1 through the (now default-active) `qqq-snapshot` profile until parent 2.1.0 ships; customizers are deleted |
+| D11 | `startupMode` default `FAIL_FAST`; `DEGRADED` available |
+
+## Architecture
+
+- `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`; component producers in the package build the two tables, three processes, failed-event table, table PVS and app; `postProduceActions` registers the record-change listener and the runtime service and resolves discovered tables.
+- `QuickSearchRuntime` (instance-scoped, held `transient` on the config, resolved through `QContext.getQInstance().getQBits()`) owns discovered tables, the lazily built client, the publisher, the enabled-table cache and lifecycle. `QuickSearchQBitContext` stays as a deprecated static facade delegating to it (tests keep working).
+- `QuickSearchRecordChangeListener` replaces the three customizers; publishes after commit when a transaction is present; records failures in `quickSearchFailedEvent`.
+- `QuickSearchOpenSearchClient` gains a transport factory (`NONE`, `BASIC`, `AWS_SIGV4`, TLS block, timeouts), mapping v2 (search analyzer, dynamic templates, `_meta` version), alias management, health, and wraps `OpenSearchException`.
+- Indexing steps page by primary key, anchor the basepull watermark at run start, honour `enabled`, sweep orphans, detect drift, purge run history, replay failed events, and use `external_gte` versioning from the timestamp field.
+- `QuickSearchAction` filters tables by permission and enabled state, bounds inputs, supports `tableNames` and `limitPerTable`, post-filters through `QueryAction`, returns `tableLabel`.
+
+## Tasks (each ends with green `mvn test`, then a commit)
+
+1. Build: default-active `qqq-snapshot` profile on `4.1.0-RC.1`, `revision` 1.0.0-SNAPSHOT, opensearch-java 3.10.0 with the `Refresh.True` fix, JaCoCo `prepare-agent` with explicit version, drop JUnit 5.10.0 pins (fallback: keep if JUnit 6 breaks a dependency). Files: `pom.xml`, `QuickSearchOpenSearchClient.java:427`.
+2. Config surface and runtime holder: new fields (`startupMode`, `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`, `enableBasepullProcess`, `enableMaintenanceProcesses`, `adminPermissionRules`, `tablePermissionRules`, `applyRecordSecurityLocks`, `maxSearchLimit`, `basepullOverlapSeconds`, `runHistoryRetentionDays`, `maxFieldLength`, `opensearchUrl`, `authMode`, `awsRegion`, `awsServiceName`, `awsAssumeRoleArn`, `allowPlaintextCredentials`, `tls*`, `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `transportCustomizer`), deprecated `enableScheduledProcesses` alias, null-safe validation with field-existence checks, `getDefaultBackendNameForTables`, settable `tableMetaDataCustomizer`, `@JsonIgnore` secret getters, redacting `toString`, `transient` publisher. `QuickSearchRuntime` class; `QuickSearchQBitContext` facade. Tests: `QuickSearchQBitConfigTest`, new `QuickSearchRuntimeTest`.
+3. Producer contract: `QBitMetaDataProducer`, `QBitMetaData` (artifactId `quick-search`, version from `quick-search.properties` filtered by Maven), component producers, unique key on `tableName`, permission rules, schedules, `QuickSearchRuntimeService`, `isEnabled` false without config, compat shims (`withConfig`, `getConfig`, `produce(QInstance)` adds and returns an empty output). Tests: `QuickSearchQBitProducerTest` updated plus new assertions (qbit registered, sourceQBitName, schedule, permission rules, unique key).
+4. Real-time path: `QuickSearchRecordChangeListener`, `quickSearchFailedEvent` table and entity, failed-event replay in basepull, delete customizers and their tests, new `QuickSearchRecordChangeListenerTest` (insert/update/delete, after-commit via a fake transaction, rollback discards, failure writes a failed-event row).
+5. Client: transport factory with auth modes, TLS, timeouts, user agent; secret interpretation via `QMetaDataVariableInterpreter`; mapping v2 with `_meta.mappingVersion`; `lenient` boosts and `operator=AND`; `OpenSearchException` wrapping; alias helpers (`createIndexForAlias`, `swapAlias`, `deleteIndex`, `resolvePhysicalIndex`); `health()`; `external_gte` versioning on bulk index; byte-capped bulk batches. Tests: `QuickSearchOpenSearchClientTest` (config to transport mapping, validation of modes), integration tests updated for mapping v2 and the "stella/stone" and mixed-type cases.
+6. Indexing steps: keyset paging in `AbstractIndexingStep`; basepull start-anchored watermark with overlap and `GREATER_THAN_OR_EQUALS`, `enabled` honoured, failures aggregated, run purge, failed-event replay; full reindex alias swap (all tables) or in-place (single table), error message on FAILED; reconcile orphan sweep and `enabled`; drift detection sets `status = NEEDS_REINDEX`; `IndexingUtils` display values, label fallback, hidden-field exclusion, truncation, safe label formatting. Tests: step tests updated plus new scenarios from audit 05.
+7. Search: `QuickSearchInput extends AbstractActionInput`, `tableNames`, `limitPerTable`, bounds, permission and enabled filtering, `QueryAction` post-filter, `tableLabel`, `totalHitsIsLowerBound`, `track_total_hits`. Tests: `QuickSearchActionTest` with a permission-denied session and bounds.
+8. Admin surface: process frontend steps with an optional `tableName` field backed by a discovered-tables PVS, admin table fields (`lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`, `realTimeErrorCount`), read-only tracking fields, run-to-index PVS, labels, icon, health process.
+9. Docs and CI: README rewrite, `CHANGELOG.md`, `docs/MIGRATION-1.0.md`, configuration reference, supported platforms and roles, parameterized Testcontainers image with a 2.19.6 and 3.9.0 matrix in CI, CLAUDE.md and AGENTS.md architecture updates.
+10. Close-out: full `mvn verify` with Docker, PRD and audit status columns, TODO and SESSION-STATE, second-brain save.
+
+## Open Questions
+
+- Parent 2.1.0 re-pin happens when qbit-bom publishes it; until then the default-active profile carries 4.1.0-RC.1 (recorded as a deviation from ADR-0007, to be reverted).
+- The qqq core `RecordSearchProviderInterface` proposal is filed as a follow-up, not implemented here.

docs/SESSION-STATE.md

@@ -0,0 +1,31 @@
+# Session State
+
+Updated: 2026-10-06
+
+## Context
+
+`1.0.0-RC.2` is released (2026-10-06). It fixes the five pre-GA findings from the PR #10 review, each on its own branch and PR into `develop`:
+
+| Issue | PR | Merge commit | Fix |
+|---|---|---|---|
+| #15 | #18 | `2398e62` | Transport: warn on `hostnameVerification=false`, blank `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected |
+| #17 | #21 | `e9ec54a` | `safeCount` logs at warn and keeps the previous `documentCount` |
+| #16 | #19 | `9941a70` | Search `offset` counts readable results under record locks; bounded scan; 10,000 window cap |
+| #14 | #20 | `dce0f26` | Deletes during a full reindex captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap |
+| #13 | #22 | `c5c41d4` | Listener re-read failure and missing AWS SDK (`LinkageError`) recorded as failed events |
+
+`develop` (`c5c41d4`) was merged into `release/1.0` as `b7d366c` (only difference from develop: `<revision>1.0.0-RC.1</revision>`). The orb's `publish_release_candidate` workflow succeeded on `b7d366c` (CircleCI build 82) and pushed `c11b2c2` ("Bump version to 1.0.0-RC.2 [skip ci]"). Maven Central serves the jar, pom, sources, javadoc and signatures for `com.kingsrook.qbits:qbit-quick-search:1.0.0-RC.2`.
+
+## Release mechanics (qqq-orb 0.6.8, confirmed in its source)
+
+- RCs publish artifacts only: no git tag and no GitHub release. The GitHub release comes when the release branch merges to `main` and a `vX.Y.Z` tag is pushed.
+- Never push a `vX.Y.Z-RC.n` tag: it matches the GA job's `v*` filter, maps back to `release/X.Y.Z`, and would publish a stray `RC.n+1`.
+- The next push to `release/1.0` publishes `1.0.0-RC.3`. Keep the release branch's `<revision>` on merge.
+- `develop` requires one approving review and the PR author cannot approve their own PR; James merges with `gh pr merge <n> --squash --admin`. The agent cannot run `gh pr merge`, and the auto-mode classifier also refused a background loop that polled PR merge state.
+- GitHub SSH times out from this machine; push over HTTPS with `git -c credential.helper='!gh auth git-credential' push https://github.com/QRun-IO/qbit-quick-search.git <ref>`.
+
+## Next
+
+1. qqq-all and Voyage pin `1.0.0-RC.2` and report back; their feedback decides RC.3 or GA.
+2. Remaining "Before 1.0.0 GA" items in `docs/TODO.md`: stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review), and the `qbit-build-parent` 2.1.0 re-pin.
+3. GA: merge `release/1.0` to `main` and tag `v1.0.0`.

docs/TODO.md

@@ -0,0 +1,45 @@
+# TODO
+
+## Before tagging 1.0.0
+
+- [ ] qqq-all and Voyage adopt `1.0.0-RC.2` (items below) and report back.
+
+- [x] PR #10 (`feature/GH-1-0-release-readiness`) opened and reviewed 2026-10-05; `AGENTS.md` kept.
+- [ ] Re-pin to `qbit-build-parent` 2.1.0 when qbit-bom publishes it, and remove the `activeByDefault` on the `qqq-snapshot` profile (ADR-0007).
+- [ ] qqq-all follow-up PR: remove the manual `instance.addQBit(...)` for quick-search in `DemoQBits`, pass `withSchedulerName("demoScheduler")`, set `withStartupMode(QuickSearchStartupMode.DEGRADED)` for the full profile test, pin 1.0.0.
+- [ ] Voyage: pin 1.0.0, set `schedulerName`, switch to `opensearchUrl` and `${env.}` credentials.
+- [ ] CI: add a second test job with `-Dopensearch.test.image=opensearchproject/opensearch:3.9.0` (needs a qqq-orb parameter for extra Maven args, or a plain job).
+- [x] Core SPI proposed as QRun-IO/qqq#1042 (D1); the 1.1 adapter `QuickSearchRecordSearchProvider` is on `feature/record-search-provider`.
+- [x] Tracking issue #12 created and recorded in the PRD (2026-10-06).
+- [x] PR #8 merged 2026-10-05; Dependabot PR #1 closed as superseded.
+- [x] PR #10 merged and `1.0.0-RC.1` published to Maven Central from `release/1.0` (2026-10-05).
+- [ ] Upstream: tell qbit-bom that the published parent POM has `<version>${revision}</version>`, which Maven 3.10 rejects (this repo works around it with the wrapper on 3.9.11).
+
+## Before 1.0.0 GA (from the PR #10 review, 2026-10-05; filed as #13 to #17 under #12)
+
+- [x] #13 Listener: a failed in-transaction re-read or a missing AWS SDK (`LinkageError`) records `quickSearchFailedEvent` rows (PR #22).
+- [x] #14 Full reindex: deletes made during a rebuild are captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap (PR #20).
+- [x] #15 Transport: warn on `tls.hostnameVerification=false`, empty `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected (PR #18).
+- [x] #16 Search: `offset` counts readable results under record locks, per-table `offset`, bounded scan and 10,000 window cap (PR #19).
+- [x] #17 `safeCount` logs at warn and keeps the previous `documentCount` (PR #21).
+- [x] `1.0.0-RC.2` published to Maven Central from `release/1.0` (2026-10-06).
+- [ ] Stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review): only another all-tables reindex clears it, `AWAITING_REINDEX` rows accumulate and drift detection stays off for those tables. No data is lost. Fix needs a liveness check on the `FULL_REINDEX` run record before reconcile may reset the status, plus refusing a second concurrent full reindex.
+- [ ] Drift detection still writes `NEEDS_REINDEX` into `status`, so a few-millisecond window can overwrite `REBUILDING`; the durable fix is a separate drift field (schema change, so 1.1 at the earliest).
+
+## Before releasing 1.1.0
+
+- [x] Re-pointed the `qqq-snapshot` profile from `4.1.0-SNAPSHOT` to `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI (2026-10-11).
+- [ ] qqq-all / Website-Backend: confirm the Next UI search box is served from OpenSearch (provider registered, indexed tables have `searchFields`).
+
+## Later (1.x)
+
+- [ ] Health widget for the admin app (data exists: `isHealthy`, `documentCount`, `lastRunStatus`).
+- [ ] `IndexEvent` sequence number so an async publisher can order INDEX and DELETE; promote the publisher SPI from experimental.
+- [ ] Amazon OpenSearch Serverless (needs removal of `_delete_by_query` and `_refresh` from the design).
+- [ ] Jev table routing prototype; see `docs/reference/typesafe-jev-candidates.md`.
+
+## Done
+
+- [x] Status assessment, deep-dive audit (25 blockers), PRD (2026-10-04).
+- [x] 4.1.0-RC.1 compatibility verified (2026-10-04).
+- [x] All 25 audit blockers implemented and tested on the release branch (2026-10-04).

docs/audits/2026-10-04/01-qqq-4.1-integration.md

@@ -0,0 +1,93 @@
+# Audit 01: qbit-quick-search readiness for QQQ 4.1
+
+Date: 2026-10-04. Read-only audit of `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` (0.2.1-SNAPSHOT, parent `qbit-build-parent` 2.0.0, qqq 4.0.0) against the QQQ tag `v4.1.0-RC.1`, the sibling qbits on their `origin/develop` branches (local checkouts are stale, Dec 2025), `qqq-all` `origin/develop` (e35f1f6, 2026-10-01) and `qbit-bom` `v2.0.0`.
+
+Path abbreviations used below:
+
+- `QS/` = `src/main/java/com/kingsrook/qbits/quicksearch/` in the qbit
+- `CORE/` = `qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/` at `v4.1.0-RC.1`
+- `JAV/` = `qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/` at `v4.1.0-RC.1`
+- `WH/` = `qbit-webhooks` `origin/develop:src/main/java/com/kingsrook/qbits/webhooks/`
+- `ALL/` = `qqq-all` `origin/develop:qqq-all-app/src/`
+
+Taken as given: 233 unit tests and 12 Testcontainers integration tests pass on qqq 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1. Nothing below is a compile or test failure; everything is a contract, consistency or operability gap.
+
+## Findings table
+
+| ID | Severity | Finding | Evidence (qbit) | Evidence (qqq tag / siblings / consumers) | Recommended change | 1.0 blocker |
+|---|---|---|---|---|---|---|
+| F-01 | High | Producer is a plain class, not a `QBitMetaDataProducer`; it registers no `QBitMetaData`, so the host must stamp one by hand, and the version it stamps is a hand-copied string. | `QS/QuickSearchQBitProducer.java:60` (`public class QuickSearchQBitProducer`), `:112` (`produce(QInstance)`), `:200-203` (static context populated), `:214-215,230-232,260` (`addTable/addProcess/addApp` directly) | `CORE/model/metadata/qbits/QBitMetaDataProducer.java:43,50,56,108-189` (contract: `getQBitConfig`, `getQBitMetaData`, default `produce` that calls `qInstance.addQBit` at `:115`, pushes `QBitProductionContext` at `:117-118`, validates at `:122`, finds producers at `:124`, applies `getTableMetaDataCustomizer`/`getDefaultBackendNameForTables` at `:156-166`, stamps `sourceQBitName` at `:172-175`). Siblings: `WH/WebhooksQBitProducer.java:45,55-64` and `qbit-standard-process-trace` `StandardProcessTraceQBitProducer.java:42,90-94`, `qbit-workflows` `WorkflowsQBitProducer.java:65,77-81` all implement `QBitMetaDataProducer<C>`; only `qbit-user-role-permissions` still uses the gen-1 `QBitProducer` (`UserRolePermissionsQBitProducer.java:41`). Consumer workaround: `ALL/main/java/com/kingsrook/qqq/all/DemoQBits.java:161-163` calls `produce(instance)` then `instance.addQBit(new QBitMetaData()...withVersion("0.2.1-SNAPSHOT"))` itself; `ALL/test/java/com/kingsrook/qqq/all/FullQBitsTest.java:44-48,78-79` depends on the artifactId `quick-search` being present. | Implement `QBitMetaDataProducer<QuickSearchQBitConfig>`; return `QBitMetaData` with groupId `com.kingsrook.qbits`, artifactId `quick-search` (keep this exact string, qqq-all asserts it), version read from a build-time resource or constant kept in sync by the release step, `withNamespace(getNamespace())`, `withConfig(getQBitConfig())`. Keep `withConfig(...)`/`produce(QInstance)` as thin compatibility shims (see migration plan) so `DemoQBits.java:161` keeps compiling; the host's manual `addQBit` at `:162-163` then becomes a duplicate and should be removed in qqq-all once this ships. | Yes |
+| F-02 | High | Real-time indexing runs inside the caller's uncommitted transaction. Post-insert/update/delete customizers fire before any commit, so a rolled-back bulk/ETL write is still indexed; a committed write that OpenSearch rejected is logged and dropped. | `QS/customizers/QuickSearchPostInsertCustomizer.java:67-105` (publishes synchronously, swallows failure at `:99-102`); `QuickSearchPostUpdateCustomizer.java:82-123,149-154` (re-reads with `updateInput.getTransaction()`); `QuickSearchPostDeleteCustomizer.java:68-118` | `CORE/actions/tables/InsertAction.java:286` (`runPostInsertCustomizers` is the last thing before `return insertOutput` at `:292`); `UpdateAction.java:268`; `DeleteAction.java:370-390`. Commit happens later in the caller: `CORE/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java:204-207` commits after the actions ran, `:215-218` rolls back on exception. 4.1 adds the fix: `CORE/actions/QBackendTransaction.java:110-113` (`addAfterCommitCallback`, callbacks discarded on `rollback()` at `:89-96`), `qqq-backend-module-rdbms/.../RDBMSTransaction.java:128` (`runAfterCommitCallbacks` after JDBC commit). Reference use: `qqq-esb/src/main/java/com/kingsrook/qqq/esb/publish/EsbRecordChangeListener.java:88-97`. Not in 4.0.0 (0 hits for `addAfterCommitCallback` in `v4.0.0:CORE/actions/QBackendTransaction.java`). | When `input.getTransaction() != null`, build the events now (the record data is correct inside the transaction) but defer `publisher.publish*` via `transaction.addAfterCommitCallback(...)`; publish immediately only when there is no transaction (RDBMS without a transaction autocommits per statement, `RDBMSInsertAction.java:73-80`). Behind a `Capability`/reflection guard if 4.0 support must be kept. | Yes |
+| F-03 | High | The customizers are the wrong 4.1 hook. 4.1 adds an instance-level `RecordChangeListenerInterface` that is designed for exactly this (fires for INSERT/UPDATE/DELETE, cannot fail the write, carries old records for updates and the stored records for deletes, carries the transaction for after-commit work). The qbit's per-table `withCustomizer` approach also throws when the host already has a customizer in the slot (see F-04). | `QS/QuickSearchQBitProducer.java:571-578` | `CORE/actions/tables/listeners/RecordChangeListenerInterface.java:28-61` (contract, `appliesTo(tableName, type)` at `:56`, `onRecordsChanged` at `:61`); `RecordChangeEvent.java:30-54` (records/oldRecords/transaction semantics per type); `CORE/model/metadata/QInstance.java:1989-1996` (`withRecordChangeListener`); fire points `InsertAction.java:311-328`, `UpdateAction.java:259,488-537` (oldRecords paired by primary key), `DeleteAction.java:365,809-823`; `RecordChangeListenerHelper.java:54-70,78-102` (never fails the write). Introduced by qqq commit `8db90166a`; absent in 4.0.0 (`v4.0.0:CORE/model/metadata/QInstance.java` has 0 `RecordChangeListener` hits). | Replace the three customizers with one `QuickSearchRecordChangeListener implements RecordChangeListenerInterface`: `appliesTo` returns true only for discovered tables (so no extra old-record fetches elsewhere), `onRecordsChanged` builds INDEX events from `event.getRecords()` for INSERT/UPDATE and DELETE events from `event.getRecords()` for DELETE, and publishes after commit when `event.getTransaction() != null` (F-02). The update re-read in `QuickSearchPostUpdateCustomizer.java:132-155` stays necessary (UPDATE `records` are "as the backend returned them", which for RDBMS is the sparse input), so keep it, using `event.getTransaction()`. Register once in `postProduceActions` with `qInstance.withRecordChangeListener(new QCodeReference(...))`. Keep the customizer classes for one release as a 4.0 fallback if dual support is required. | Yes (choose F-03 or F-04; F-03 preferred) |
+| F-04 | High | `QTableMetaData.withCustomizer` throws `IllegalArgumentException` when the slot is occupied, so any host table that already has a POST_INSERT/UPDATE/DELETE customizer makes `produce` blow up (and not with a `QException`). | `QS/QuickSearchQBitProducer.java:571-578` | `CORE/model/metadata/tables/QTableMetaData.java:644-657` (`:651-654` throws `IllegalArgumentException("Attempt to add a second customizer with role ...")`); same at `v4.0.0`. Composition options: `CORE/actions/customizers/MultiCustomizer.java:66-70` (`MultiCustomizer.of(...)`) and `:79-83` (`addTableCustomizer(existing, codeRef)`); instance-level `CORE/model/metadata/QInstance.java:1927-1945` (`withTableCustomizer(role, codeRef)`), consumed in `InsertAction.java:356-362` and `UpdateAction.java:303-309` but NOT in `DeleteAction.java` (0 hits for `getTableCustomizers`), so instance-level customizers never fire for deletes. Sibling precedent: `WH/WebhooksQBitProducer.java:102-103` (instance-level, insert/update only), `WH/model/Webhook.java:154-157` (`MultiCustomizer.of`). | If customizers are kept (instead of F-03): read `table.getCustomizers().get(role)`; if absent add directly; if present and it is a `QCodeReferenceWithProperties` for `MultiCustomizer`, call `MultiCustomizer.addTableCustomizer`; otherwise replace the slot with `MultiCustomizer.of(existing, ours)` via `table.getCustomizers().put(role, ...)` (the map is mutable). Do not use `QInstance.withTableCustomizer` for this qbit: it is not per-table (it would have to filter by `QuickSearchQBitContext.getTableConfig`) and it skips deletes. | Yes (if F-03 not taken) |
+| F-05 | High | Processes are produced without `QScheduleMetaData` or `BasepullConfiguration`, so nothing ever runs the basepull or reconcile unless the host schedules it by hand; `enableScheduledProcesses` is misnamed (it only gates registration). The qbit hand-rolls last-run tracking on its own table instead of QQQ's basepull support, and the basepull window has a gap (query at T0, stamp `Instant.now()` at T1 after the batches finish; rows modified in (T0, T1] with the same second are skipped next run). | `QS/QuickSearchQBitProducer.java:509-549` (three `QProcessMetaData` with only a step); `:228-233`; `QS/QuickSearchQBitConfig.java:55` (`enableScheduledProcesses = true`); `QS/processes/BasepullIndexStep.java:202-209` (filter `GREATER_THAN lastBasepullTime`), `:268-278` (stamps `Instant.now()` after processing); `README.md:306-307` describes it as a schedule | `CORE/model/metadata/processes/QProcessMetaData.java:75,564` (`withSchedule(QScheduleMetaData)`), `:61,598` (`withBasepullConfiguration`); `CORE/model/metadata/scheduleing/QScheduleMetaData.java:40-49,89,284` (`repeatSeconds`, `cronExpression`, `schedulerName`); `CORE/scheduler/QScheduleManager.java:284-289,413-429` (processes with a schedule are set up against `getSchedule().getSchedulerName()`); `JAV/QApplicationLauncher.java:139-153,216-222` (launcher starts the scheduler only if some process/queue/automation has a schedule); `CORE/instances/QInstanceValidator.java:2255-2258,2290-2326` (a schedule must have repeat or cron, and a scheduler name that exists in the instance); `CORE/processes/implementations/basepull/BasepullConfiguration.java:33-43` (table/keyField/lastRunTimeFieldName/timestampField/seconds-to-subtract); `CORE/actions/processes/RunProcessAction.java:147-155,183-187,914` (framework reads the stored timestamp before the step, stores the new one after, keyed by process name). Consumers: `ALL/.../DemoQBits.java:63` adds `SimpleSchedulerMetaData "demoScheduler"`, `:106,171` pass `withSchedulerName("demoScheduler")` to webhooks and sftp. | Add `schedulerName` (nullable) and `basepullRepeatSeconds`/`reconcileCronExpression` (nullable) to `QuickSearchQBitConfig`; when `schedulerName` is set, attach `new QScheduleMetaData().withSchedulerName(...).withRepeatSeconds(...)` to the basepull process and a cron schedule to reconcile. Leave processes unscheduled when `schedulerName` is null (hosts that use the `scheduledJob` table can still schedule them). Validate `schedulerName` in `validate` only when `qInstance.getSchedulers()` is non-empty (see F-07). Basepull: either adopt `BasepullConfiguration` (one process per table would be needed because `BASEPULL_KEY_VALUE` is per run; or run one process that overrides `basepullKeyValue` per table) or keep the own table but stamp the run start time (captured before the first query, minus a small overlap) instead of `Instant.now()` after the loop. Rename `enableScheduledProcesses` to `enableIndexingProcesses` with a deprecated alias. | Yes |
+| F-06 | High | Search results carry no permission filtering. `QuickSearchAction` returns table/recordId/label from OpenSearch for any table the index holds, regardless of the session's table permissions or record security locks; produced tables/processes/app set no `QPermissionRules`, which is fine for the demo (instance default is `NOT_PROTECTED`) but means a host that sets a restrictive `defaultPermissionRules` silently locks the admin app, and a host that does not gets an unprotected admin app. | `QS/actions/QuickSearchAction.java:63-130` (no `PermissionsHelper` use; `:81-82` reads static context); `QS/QuickSearchQBitProducer.java:457-549` (no `withPermissionRules`) | `CORE/actions/tables/RecordSearchAction.java:103-113` (core search skips tables without `PermissionsHelper.hasTablePermission(input, table, READ)` and queries through `QueryAction` so record locks apply, `:63-67`); `CORE/instances/QInstanceEnricher.java:544-575` (null rules get `qInstance.getDefaultPermissionRules()`, else `QPermissionRules.defaultInstance()` = `NOT_PROTECTED`, `CORE/model/metadata/permissions/QPermissionRules.java:44-47`). Siblings: `qbit-workflows` processes set `withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))` explicitly (`GetWorkflowRevisionProcess.java:72` etc.); qqq-all protects webhook tables/processes itself (`ALL/test/.../FullQBitsTest.java:67-77`, `DemoUsers.protectWebhookResources`). | In `QuickSearchAction`: require a `QContext` session; filter `tableConfigs` to tables with READ permission before building the OpenSearch query (pass the filtered table list as a `terms` filter on `sourceTable`); for tables with record security locks, re-verify hits by primary key through `QueryAction` (or document that locked tables are not indexable). Add `permissionRules` to the config for the admin app/tables/processes, defaulting to `HAS_ACCESS_PERMISSION` for the processes and `READ_INSERT_EDIT_DELETE_PERMISSIONS` for the two operational tables, with `NOT_PROTECTED` opt-out, mirroring what qqq-all does for webhooks. | Yes |
+| F-07 | Medium | `validate` does a backend lookup, which works today only because the qbit is produced after backends exist. Under `QBitMetaDataProducer` the same timing holds (validate runs inside `produce` with the host's instance), but if a host lets `MetaDataProducerHelper` discover the producer by package scan, it is instantiated with a no-arg constructor and null config, and `QBitMetaDataProducer.produce` dereferences `getQBitConfig()` with no null check. | `QS/QuickSearchQBitConfig.java:73-81` (`qInstance.getBackend(backendName) == null`); `QS/QuickSearchQBitProducer.java:117-122` | `CORE/model/metadata/qbits/QBitMetaDataProducer.java:113-122` (`qBitConfig.validate(qInstance)` on the live instance; no null guard); `CORE/model/metadata/qbits/QBitConfig.java:58-75` (exceptions from `validate` are caught and only logged at `:66-69`, so a thrown NPE inside the list-based validate is swallowed, and only `errors` entries fail); `CORE/model/metadata/MetaDataProducerHelper.java:200-202,585-595` (any `MetaDataProducerInterface` in the scanned package with a no-arg constructor is instantiated and run). Known gap note: `WH/WebhooksQBitConfig.java:85-99` ("qbit validation is happening before the full instance is built"; backend and scheduler checks commented out). `QInstanceValidator` has no qbit-specific checks (0 hits for `sourceQBit`/`qbit` in `CORE/instances/QInstanceValidator.java`), so produced metadata passes as long as apps list every child in a section (`:2597-2605`; the enricher creates a default section) and schedules reference a known scheduler (`:2323-2326`). | Keep the list-based `validate`; guard null `qInstance` and null `getBackends()`; return `isEnabled() == false` from the producer when `getQBitConfig() == null` so package scanning skips it (`MetaDataProducerHelper.java:323,345` honors `isEnabled`). Add a `QInstanceValidatorPluginInterface` (registered via `QInstanceValidator.addValidatorPlugin`, `:319`) for late checks (backend exists, each searchable table exists, scheduler exists) so they run against the full instance. | No |
+| F-08 | Medium | Static `QuickSearchQBitContext` holds config/client/publisher/tables for the JVM. Two instances in one JVM (tests, namespace use, blue/green instance reload via `QInstance` hot swap) collide; customizers and steps read it instead of the instance. 4.1 gives two sanctioned places: `QBitMetaData.getConfig()` reachable from any `SourceQBitAware` table/process, and `QBitProductionContext` during production. | `QS/QuickSearchQBitContext.java:32-37` (static volatile fields), `:152-158` (`clear()` for tests); readers `QS/processes/AbstractIndexingStep.java:68-73,85-90,102-122`, `QS/customizers/*:74,89,75`, `QS/actions/QuickSearchAction.java:81-82` | `CORE/model/metadata/qbits/SourceQBitAware.java:54-75` (`getSourceQBit()` / `getSourceQBitConfig()` via `QContext.getQInstance().getQBits()`), implemented by `QTableMetaData` and `QProcessMetaData` (not `QAppMetaData`); `CORE/model/metadata/qbits/QBitProductionContext.java:48-66` (config stack during produce). Sibling pattern: `WH/WebhooksQBitConfig.java:146-157` (`getConfigValue` resolves config from a produced table's `getSourceQBitConfig()`, with a default fallback); `WH/model/WebhookEvent.java:121` (`QBitProductionContext.peekQBitConfig()` inside component producers). | Keep `QuickSearchQBitConfig` as the single source of truth and put the runtime objects (client, publisher, discovered tables) on it lazily (`transient`, built on first use, closed by a runtime service, F-10). Resolve the config in steps via `QContext.getQInstance().getProcess(input.getProcessName()).getSourceQBitConfig()`, in the listener via the discovered table's `getSourceQBitConfig()`, and in `QuickSearchAction` via an explicit `QuickSearchInput.withQBitName`/default lookup over `qInstance.getQBits()`. Keep the static accessors as deprecated delegates for one release. | No |
+| F-09 | Medium | Produce is "best effort" on OpenSearch: a missing or unreachable cluster yields a null client and publisher, the warnings are the only signal, every write then logs "publisher is null", and `QuickSearchAction` throws at query time. There is no fail-fast toggle and no health exposure. qqq-all depends on the degraded behaviour: `FullQBitsTest` builds the full instance with OpenSearch at `127.0.0.1:1`. | `QS/QuickSearchQBitProducer.java:159-167,172-182,187-195`; `QS/customizers/QuickSearchPostInsertCustomizer.java:74-79`; `QS/actions/QuickSearchAction.java:84-87`; `QS/opensearch/QuickSearchOpenSearchClient.java:94-150` (transport is built lazily; only `ensureIndexExists` at `:161` touches the network) | `JAV/QApplicationLauncher.java:118-175` (start order: server, scheduler, runtime services; a thrown `start` stops what started and fails `run`), `:192-207` (`failOnMetaDataProducerError` only reaches producer-based apps); `CORE/instances/QRuntimeServiceInterface.java:39-68` (`start(QInstance)`, `stop()`, `afterApplicationStop()`); `CORE/model/metadata/QInstance.java:2153` (`withRuntimeService`). Consumer: `ALL/test/.../FullProfileTest.java:123-124` (`OPENSEARCH_HOST=127.0.0.1`, `OPENSEARCH_PORT=1`) and `FullQBitsTest.java:41-42` (instance must define and validate with that); `ALL/main/.../QqqAllApplication.java:98` sets `withFailOnMetaDataProducerError(true)`, `:154` registers a runtime service; `qqq-all` plan: "disabled with a notice in core" (`origin/develop:docs/plan.md:80`). | Add `startupMode` to the config: `DEGRADED` (default, current behaviour, keeps qqq-all's test green) and `FAIL_FAST` (throw `QBitConfigValidationException`/`QException` from `produce` when the client or index cannot be created). Move the connect and `ensureIndexExists` out of `produce` into a `QuickSearchRuntimeService implements QRuntimeServiceInterface` registered with `qInstance.withRuntimeService(...)` in `postProduceActions`: `start` connects (fail-fast or warn per mode), `stop` closes the transport (today nothing calls `SynchronousIndexEventPublisher.close()` or `QuickSearchOpenSearchClient.close()`, `QS/publisher/SynchronousIndexEventPublisher.java:179-182`, `QS/opensearch/QuickSearchOpenSearchClient.java:595-606`). Expose a `quickSearchHealth` process or widget that reports connected/index exists/document count so the host's `/health` can show a notice. | No |
+| F-10 | Medium | Lifecycle: nothing closes the OpenSearch transport or publisher at shutdown; index creation happens during metadata production (a side effect on an external system inside `defineQInstance`). | `QS/QuickSearchQBitProducer.java:174-182`; `QS/publisher/SynchronousIndexEventPublisher.java:179-182` (only `close()` path, never invoked) | `JAV/QApplicationLauncher.java:155-168,243-273` (runtime services started after the instance is built, stopped in reverse order); `CORE/instances/QRuntimeServiceInterface.java:47-55` | Covered by F-09: runtime service owns connect/ensureIndex/close. | No |
+| F-11 | Medium | Core 4.1 ships `RecordSearchAction` and `POST /qqq/v1/search` with no provider SPI; Quick Search cannot back it, so a 4.1 host gets two incompatible "search" features (core: LIKE over `searchFields`; qbit: OpenSearch) and Next's search UI will call the core one. | `QS/actions/QuickSearchAction.java:63-130`, `QS/actions/QuickSearchInput.java`, `QuickSearchOutput`, `QuickSearchResult` (parallel model, no relation to core) | `CORE/actions/tables/RecordSearchAction.java:71-119` (`new RecordSearchAction().execute(input)` concrete class, iterates `qInstance.getTables()` with `getSearchFields()`, no code-reference indirection); `CORE/model/actions/tables/search/RecordSearchInput.java:35-40` (`searchTerm`, `tableNames`, `limitPerTable`, `inputSource`); `RecordSearchResult` (tableName, tableLabel, recordId, recordLabel); `JAV/executors/RecordSearchExecutor.java:51-57` (hard-wires `new RecordSearchAction()`); `JAV/specs/v1/RecordSearchSpecV1.java:67-68` (`POST /search`); qqq commit `7dd36c8ad`; `QTableMetaData.java:100,2016` (`searchFields`), validator `QInstanceValidator.java:1282-1298`. | Smallest core SPI (propose in qqq): `QInstance.withRecordSearchProvider(QCodeReference)` plus `interface RecordSearchProviderInterface { boolean appliesTo(QTableMetaData table); List<RecordSearchResult> search(RecordSearchInput input, QTableMetaData table, String term, int limit) throws QException; }`. `RecordSearchAction.searchTable` consults the provider when `appliesTo` is true (and the table has `searchFields` or the provider says it is searchable), still after the permission check at `:110`, so record-security filtering stays core's job. Until then: make `QuickSearchResult` shape-compatible with `RecordSearchResult` (add `tableLabel`), and set `searchFields` on indexed tables via the table customizer so Next's UI at least shows the search box; document the overlap. | No (SPI is a qqq change; the qbit-side adapter is 1.1) |
+| F-12 | Low | Namespace support is absent: produced names are prefixed with the qbit's own `tableNamePrefix`, not with the `QBitMetaData.namespace`; `getNamespace()` is not implemented. Core does not prefix names automatically either, so a namespace-aware qbit has to do it itself. | `QS/QuickSearchQBitConfig.java:169-195` (`applyPrefix`, `tableNamePrefix`); `QS/QuickSearchQBitProducer.java:208-209,220-222,246` | `CORE/model/metadata/qbits/QBitMetaData.java:60-68` (namespace is part of the qbit's unique name only); `QBitMetaDataProducer.java:62-65` (`getNamespace()` default null); qqq test qbit does no name prefixing (`qqq-backend-core/src/test/.../testqbit/metadata/SomeTableMetaDataProducer.java:59-61`); `ALL/.../DemoQBits.java:120,182` pass a namespace to gen-1 producers (`produce(instance, "shipping-geo")`). | Add `namespace` to the producer (`withNamespace`), use it as the default `tableNamePrefix` when the prefix is null, and feed it to `QBitMetaData.withNamespace`. Keep `tableNamePrefix` as an override. | No |
+| F-13 | Low | Build: the pom declares `jacoco-maven-plugin` with a `check` execution but no version and no `prepare-agent`, so the check is skipped ("missing execution data file") and the plugin version floats; the parent manages no jacoco plugin. JUnit is pinned to 5.10.0 (BOM import and direct dependency) while the parent manages `junit-bom` 6.0.1. | `pom.xml:156-181` (jacoco `check` only), `:69-77` (`junit-bom` 5.10.0 import), `:118-122,147-150` (junit-jupiter 5.10.0) | `qbit-bom` `v2.0.0:pom.xml:54-55` (`qqq-bom-pom` 4.0.0), `:62-63` (`junit-bom` 6.0.1), `:70-90` (pluginManagement contains only `maven-compiler-plugin`); no 2.1.0 tag exists yet (`git tag` ends at `v2.0.0`; `origin/main aaf52d2` "prepare build parent 2.0.0 for QQQ 4.0"); CI orb `kingsrook/qqq-orb@0.5.2` in `.circleci/config.yml:4` (parent uses 0.5.0). qqq's own BOM manages no JUnit (`v4.1.0-RC.1:qqq-bom/pom.xml` has no junit entries). | Either add `prepare-agent` (and a `prepare-agent-integration`/`report` pair for failsafe) with an explicit plugin version and make the 0.70/0.90 thresholds real, or delete the jacoco block (ADR). Drop the local `junit-bom` import and the 5.10.0 pins and inherit 6.0.1 from the parent (JUnit 6 requires Java 17+, fine on 21). When `qbit-build-parent` 2.1.0 (qqq 4.1) is released, re-pin and delete the `qqq-snapshot` profile's `qqq-bom-pom` import (ADR-0007). | No |
+| F-14 | Low | Source-table batch paging is unordered (`limit`/`skip` with no `orderBy`), so full reindex, reconcile and basepull can skip or duplicate rows on backends without a stable default order. | `QS/processes/AbstractIndexingStep.java:348-374` (no `withOrderBy`); `QS/processes/BasepullIndexStep.java:220-262` | `CORE/actions/tables/RecordSearchAction.java:165` (core orders by primary key before limiting, as a reference pattern) | Add `effectiveFilter.withOrderBy(new QFilterOrderBy(primaryKeyField))` and prefer keyset paging (`primaryKey > lastSeen`) over `skip`. | No (listed in the PRD's E2) |
+
+## Answers to the audit questions
+
+### 1. QBit contract
+
+What must change (all evidence above, F-01, F-08, F-12):
+
+1. `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>` with `getQBitConfig()`, `getQBitMetaData()` and `postProduceActions(...)` (`CORE/.../QBitMetaDataProducer.java:50,56,70`). The default `produce` scans the producer's own package for `MetaDataProducerInterface` implementations (`:124`, `getPackageNameForFindingMetaDataProducers` at `:80-101`), so the two operational tables, three processes and the app become small producers (`QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>` etc., `CORE/.../QBitComponentMetaDataProducer.java:34-69`) or `@QMetaDataProducingEntity` annotations on `QuickSearchIndex`/`QuickSearchIndexRun` (today they are plain `QRecordEntity` with `@QField`, `QS/model/QuickSearchIndex.java:32-36`). Customizer/listener registration and the OpenSearch bootstrap belong in `postProduceActions`, as webhooks does (`WH/WebhooksQBitProducer.java:89-104`).
+2. `QBitMetaData` with groupId `com.kingsrook.qbits`, artifactId `quick-search`, real version, namespace, config (`CORE/.../QBitMetaData.java:47-52`). The framework then stamps `sourceQBitName` on every produced table and process (`QBitMetaDataProducer.java:172-175`; `QTableMetaData` and `QProcessMetaData` implement `SourceQBitAware`), which is what makes F-08's static context removable.
+3. `QuickSearchQBitConfig` already implements `QBitConfig` (`QS/QuickSearchQBitConfig.java:45`); add `getDefaultBackendNameForTables()` returning `backendName` and `getTableMetaDataCustomizer()` (settable, like `WH/WebhooksQBitConfig.java:563,805`) so hosts can set RDBMS backend details the way `ALL/.../DemoQBits.java:80-88` does for the other qbits.
+
+Idiomatic choice: `QBitMetaDataProducer` (3 of 4 siblings; qqq's own interface doc calls it the "2nd generation ... evolution over QBitProducer", `:39-40`). `QBitProducer` (gen 1) is only kept by user-role-permissions and qqq's test qbit.
+
+qqq-all impact: `DemoQBits.java:161` uses `new QuickSearchQBitProducer().withConfig(searchConfig).produce(instance)`. `QBitMetaDataProducer.produce(QInstance)` returns `MetaDataProducerMultiOutput` instead of `void`, so that call still compiles (the return value is ignored) only if the metadata is added to the instance. The default `produce` does NOT add to the instance; it returns the multi-output and the host calls `.addSelfToInstance(instance)` (`DemoQBits.java:93,101,108,116`). To keep source compatibility: keep `withConfig(...)` as an alias of `withQBitConfig(...)`, and override `produce(QInstance)` to call the interface's default `produce`, then `addSelfToInstance(qInstance)` on the result before returning it (idempotent for hosts that also call `addSelfToInstance`, since `QInstance.addTable` etc. would throw on duplicates; so the override must either add and return an empty output, or be documented as "already added"; returning an empty `MetaDataProducerMultiOutput` after adding is the safe choice). The manual `instance.addQBit(...)` at `DemoQBits.java:162-163` will then duplicate the qbit's own `addQBit` (`QBitMetaDataProducer.java:115`); `QInstance.addQBit` behaviour on duplicate names should be checked when that qqq-all change is made (`CORE/model/metadata/QInstance.java:1800`).
+
+### 2. Validation timing
+
+- `QBitConfig.validate` runs at the start of `QBitMetaDataProducer.produce` (`:122`) against whatever is in the host's instance at that moment. It is not "before the full instance exists" in the sense of the host's backends: hosts add backends first (`ALL/.../FullProfileApplication.java:70-76`). The webhooks note (`WH/WebhooksQBitConfig.java:85-99`) refers to hosts that produce qbits before their own tables/schedulers; the backend lookup in this qbit (`QS/QuickSearchQBitConfig.java:79`) is safe under both the current flow and the `QBitMetaDataProducer` flow for the qqq-all ordering, but a host that produces the qbit before `addBackend` gets a false "Backend not found".
+- Under `MetaDataProducerHelper` scanning the producer would be instantiated with a null config (`MetaDataProducerHelper.java:585-595`) and `produce` would NPE at `QBitMetaDataProducer.java:113-122`; `isEnabled()` returning false when the config is null avoids that (`:145-149`).
+- `QBitConfig.validate(QInstance)` swallows exceptions from the list-based validate and only fails on collected errors (`QBitConfig.java:62-74`), so validation must add to `errors`, never throw.
+- `QInstanceValidator` 4.1 adds nothing qbit-specific. Relevant checks the produced metadata must satisfy: process steps have names and `BackendStep` code (`:2178-2197`), a schedule has repeat or cron plus an existing scheduler (`:2290-2326`), app children appear in sections (`:2597-2605`; the enricher builds a default section), table `searchFields` if set must be visible string/integer fields (`:1282-1298`). Nothing validates `sourceQBitName`.
+
+### 3. Scheduling
+
+- A process is scheduled by `QProcessMetaData.withSchedule(new QScheduleMetaData().withSchedulerName(name).withRepeatSeconds(n) | .withCronExpression(expr).withCronTimeZoneId(tz))` (`QProcessMetaData.java:564`, `QScheduleMetaData.java:40-49`). The host must own a `QSchedulerMetaData` (`SimpleSchedulerMetaData` or `QuartzSchedulerMetaData`, `CORE/model/metadata/scheduleing/{simple,quartz}/*.java:45-54`) and either start `QScheduleManager` itself or use `QApplicationLauncher`, which starts it only when something has a schedule (`JAV/QApplicationLauncher.java:139-153,216-222`). `QScheduleManager.start` sets up every process with a schedule (`:284-289`) under the system session supplier (`:163`), and `qqq.scheduleManager.enabled`/`onlyStartNamesMatching` gate it (`:153`, `SchedulerUtils.java:61-71`). Cron needs Quartz (`SimpleSchedulerMetaData.supportsCronSchedules`, `:56`).
+- Minimal correct way for this qbit: config fields `schedulerName`, `basepullRepeatSeconds` (default 300), `reconcileCronExpression`+`reconcileCronTimeZoneId` (default null, so no reconcile schedule unless the host asks); attach schedules only when `schedulerName` is set. Host provides the scheduler and the launcher (qqq-all already has `demoScheduler`, `DemoQBits.java:63`).
+- Basepull duplication: yes. `BasepullConfiguration` + `RunProcessAction` already persist a last-run timestamp per process key, pass it to the step as `RunBackendStepInput.getBasepullLastRunTime()` (`RunProcessAction.java:724-728`), and only store the new time when the step sets `BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD` (`:183-187`), with overlap seconds (`BasepullConfiguration.java:42-43`). `BasepullIndexStep` re-implements this per table on `quickSearchIndex.lastBasepullTime` (`:202-209,268-278`) and stamps `Instant.now()` after the loop (window gap). The qbit's per-table interval and per-table enable flag are not expressible with one `BasepullConfiguration` (its key is the process name, `RunProcessAction.java:880-885`), so the pragmatic fix is to keep the own table but (a) capture the run start before the first query and store that, minus an overlap, and (b) run the process on a `QScheduleMetaData`. Full adoption of `BasepullConfiguration` would mean one scheduled process per indexed table, which is also legitimate and gives per-table schedules for free.
+
+### 4. Customizer composition
+
+Confirmed: `QTableMetaData.withCustomizer` throws `IllegalArgumentException` on an occupied slot at the 4.1 tag (`QTableMetaData.java:651-654`), same at 4.0.0. Options: `MultiCustomizer.of(a, b)` / `MultiCustomizer.addTableCustomizer(existing, c)` (`MultiCustomizer.java:66-83`), or instance-level `QInstance.withTableCustomizer(role, codeRef)` (`QInstance.java:1927-1945`), which runs for every table on insert/update (`InsertAction.java:356-362`, `UpdateAction.java:303-309`) but not on delete (`DeleteAction.java` has no `getTableCustomizers` call). For per-table targeting, `MultiCustomizer` composition is the right customizer-based answer; but the better 4.1 answer is the `RecordChangeListenerInterface` (F-03), which is instance-level, filters by `appliesTo(tableName, type)`, covers delete, and never fails the write.
+
+### 5. Transactions and consistency
+
+- Customizers run after the backend write and before the action returns, inside the caller's transaction if one was passed (`InsertAction.java:286-292`, `UpdateAction.java:268`, `DeleteAction.java:370-390`); the `RecordChangeListenerInterface` javadoc states this explicitly (`:32-34`). ETL/bulk processes commit after the actions (`StreamedETLExecuteStep.java:204-207`) and roll back on exception (`:215-218`), so a rolled-back bulk insert is indexed today. The javalin single-record endpoints pass no transaction (`QJavalinImplementation.java:689,726,957` set up sessions only; no `openTransaction`), and the RDBMS module then uses a fresh autocommit connection (`RDBMSInsertAction.java:73-80`; only `RDBMSTransaction` sets autocommit false, `RDBMSTransaction.java:55`), so those are effectively committed before the customizer runs.
+- Fix available only in 4.1: `QBackendTransaction.addAfterCommitCallback` (`QBackendTransaction.java:110-113`), honoured by `RDBMSTransaction.commit` (`:128`), discarded on rollback (`:89-96`, `RDBMSTransaction.java:114`).
+- Update customizer receives `Optional<List<QRecord>> oldRecordList` (`UpdateAction.java:289`); records are the sparse updated records, hence the qbit's re-read (`QuickSearchPostUpdateCustomizer.java:132-155`). Delete customizer receives the old records minus those in error (`DeleteAction.java:376-390`), so primary keys are present. The listener API gives the same: UPDATE `oldRecords` paired by primary key (`UpdateAction.java:498-537`), DELETE `records` = stored records before delete (`RecordChangeEvent.java:35,41`).
+- `QContext`/`QSession` exist inside customizers and listeners: javalin calls `setupSession` then `QContext.init(qInstance, session, null, input)` (`QJavalinImplementation.java:495,573`); the scheduler runs under the system session (`QScheduleManager.java:163`). The re-read in the update path therefore runs as the writing user; for indexing that is correct (index what the user could write), but for a permission-aware re-read on behalf of other readers it is not a substitute for F-06.
+
+### 6. Permissions
+
+Siblings mostly leave rules null and let the enricher apply the instance default (`QInstanceEnricher.java:544-575` → `NOT_PROTECTED` unless the host sets `defaultPermissionRules`); workflows sets `NOT_PROTECTED` explicitly on its processes; qqq-all layers its own rules on the webhook tables/processes and asserts them (`FullQBitsTest.java:67-77`). `RecordSearchAction` filters by table READ permission and queries through `QueryAction` so record locks apply (`RecordSearchAction.java:103-113,168-177`). There is no provider/SPI: the action is a concrete class with no code-reference indirection, and the javalin executor instantiates it directly (`RecordSearchExecutor.java:51`). Smallest SPI sketched in F-11.
+
+### 7. Application launcher and qqq-all
+
+`QApplicationLauncher` (`JAV/QApplicationLauncher.java:106-184`) starts the server (which builds the instance), the scheduler if anything is scheduled, then runtime services; failures stop what started and throw. Two things this qbit should support: a `QRuntimeServiceInterface` for connect/ensure-index/close with a fail-fast or degraded mode (F-09, F-10), and health reporting the host can surface. qqq-all's core profile excludes the qbit entirely (`CoreQBitsTest.java:42`) and its README says "Quick search is disabled in core because it requires OpenSearch" (`ALL/main/.../README.md:14`); the full profile's unit test builds the instance with an unreachable OpenSearch (`FullProfileTest.java:123-124`), so the degraded mode must remain the default. qqq-all also runs with `withFailOnMetaDataProducerError(true)` (`QqqAllApplication.java:98`), which only affects `MetaDataProducerHelper`; the qbit's own `produce` must throw a `QException` with a clear message when fail-fast is requested.
+
+### 8. Build and release
+
+Parent `v2.0.0` pins `qqq-bom-pom` 4.0.0 and `junit-bom` 6.0.1 and manages only the compiler plugin (`qbit-bom v2.0.0:pom.xml:54-55,62-63,70-90`); no 2.1.0 tag exists yet. The qbit's jacoco `check` has no `prepare-agent` and no version (`pom.xml:156-181`), so coverage is never measured and the plugin version floats with the repository metadata; the local JUnit 5.10.0 pins (`pom.xml:69-77,118-122,147-150`) override the parent's 6.0.1. The `qqq-snapshot` profile is correct for RC checks and should be removed or re-pointed when parent 2.1.0 ships. The CI orb versions differ (0.5.2 in the qbit, 0.5.0 in the parent's own CI) but that is harmless.
+
+## Minimal migration plan (ordered)
+
+1. Producer contract (F-01, F-12): make `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`; add `getQBitMetaData()` (artifactId `quick-search`, version from a `quick-search.properties` resource filtered by Maven), `getNamespace()`; move table/process/app building into package-scanned component producers (or `@QMetaDataProducingEntity` on the two entities plus small process/app producers); move customizer registration and client bootstrap into `postProduceActions`. Keep `withConfig`, `getConfig` and a `produce(QInstance)` override that adds to the instance and returns an empty multi-output, so `qqq-all DemoQBits.java:161` keeps compiling; mark them `@Deprecated`.
+2. Config surface (F-05, F-06, F-09): add `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`, `permissionRules` (app/tables/processes), `startupMode` (`DEGRADED` default, `FAIL_FAST`), `tableMetaDataCustomizer` setter, `getDefaultBackendNameForTables()` = `backendName`. Deprecate `enableScheduledProcesses` in favour of `enableIndexingProcesses`. Validation stays list-based and null-safe; add a `QInstanceValidatorPluginInterface` for late checks (F-07).
+3. Real-time path (F-02, F-03, F-04): add `QuickSearchRecordChangeListener implements RecordChangeListenerInterface`, registered via `qInstance.withRecordChangeListener` in `postProduceActions`; publish after commit through `event.getTransaction().addAfterCommitCallback(...)` when a transaction is present. Keep the three customizers only as a documented 4.0 fallback (`enableRealTimeIndexing` + a `legacyCustomizers` flag), composing with `MultiCustomizer` when a slot is occupied; delete them in 1.1.
+4. Runtime state (F-08, F-10): add `QuickSearchRuntimeService implements QRuntimeServiceInterface` (connect, `ensureIndexExists`, close) registered via `qInstance.withRuntimeService`; resolve the config from `SourceQBitAware.getSourceQBitConfig()` in steps and the listener; keep `QuickSearchQBitContext` as a deprecated facade over the instance-bound state for one release.
+5. Scheduling and basepull (F-05, F-14): attach `QScheduleMetaData` when `schedulerName` is set; capture the basepull run-start timestamp before the first query and store it minus an overlap; add primary-key `orderBy`/keyset paging in `querySourceTableBatch`.
+6. Permission-aware search (F-06): filter table configs by `PermissionsHelper.hasTablePermission(input, table, READ)` and pass the allowed `sourceTable` set to the OpenSearch query; verify hits for tables with record security locks via `QueryAction`; align `QuickSearchResult` with core `RecordSearchResult` (add `tableLabel`) ahead of the SPI proposal (F-11).
+7. Build (F-13): decide jacoco (bind `prepare-agent` or remove), drop the JUnit 5.10.0 pins, re-pin to `qbit-build-parent` 2.1.0 when released and retire the `qqq-snapshot` profile's BOM import; bump to 0.3.0 for the compat release, 1.0.0 after qqq-all removes its manual `addQBit`.
+8. Downstream: qqq-all PR removing `DemoQBits.java:162-163`, passing `withSchedulerName("demoScheduler")`, and asserting `instance.getQBits()` contains `quick-search` from the qbit itself; propose the `RecordSearchProviderInterface` SPI in qqq (F-11).

docs/audits/2026-10-04/02-opensearch-aws-compat.md

@@ -0,0 +1,160 @@
+# Audit 02: OpenSearch and AWS compatibility of qbit-quick-search
+
+Date: 2026-10-04. Repo reviewed read-only at `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` (branch `feature/GH-6-repin-and-drift-fix`). Compile experiments were run on a copy under the scratchpad, never in the repo.
+
+Files reviewed: `src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java` (CLIENT), `OpenSearchDocument.java`, `BulkIndexResult.java`, `QuickSearchQBitConfig.java` (CONFIG), `processes/IndexingUtils.java`, `actions/QuickSearchAction.java`, `pom.xml`, and the three `*IntegrationTest.java` classes.
+
+## 0. Resolved dependency facts (from `mvn dependency:tree` in the repo)
+
+| Artifact | Resolved | Note |
+| --- | --- | --- |
+| org.opensearch.client:opensearch-java | 2.10.0 | Pinned in pom.xml line 24. Its own POM declares httpclient5 5.3.1, httpcore5 5.2.4, jackson 2.17.0, and a runtime dependency on opensearch-rest-client 2.12.0 (https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/2.10.0/opensearch-java-2.10.0.pom) |
+| org.opensearch.client:opensearch-rest-client | 2.12.0 (runtime) | Pulled only because the 2.x client still ships the deprecated RestClientTransport; the 3.x client POM no longer lists it (https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/3.9.0/opensearch-java-3.9.0.pom) |
+| org.apache.httpcomponents.client5:httpclient5 | 5.6.3 | Managed by the qqq BOM |
+| org.apache.httpcomponents.core5:httpcore5 / httpcore5-h2 | 5.4.3 | Managed by the qqq BOM |
+| com.fasterxml.jackson.core:* | 2.21.5 | Managed by the qqq BOM |
+| software.amazon.awssdk:* (auth, regions, apache-client, netty-nio-client, ...) | 2.41.10 | Already on the classpath, but only transitively via qqq-backend-core -> quicksight. Not something this library should rely on without declaring it |
+
+Maven Central: latest opensearch-java is 3.10.0 (`<latest>3.10.0</latest>` in https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/maven-metadata.xml); the 2.x line ended at 2.26.0. The OpenSearch Java client docs now install 3.10.0 (https://docs.opensearch.org/latest/clients/java/).
+
+## 1. Research findings (external)
+
+### 1.1 opensearch-java client
+
+* Compatibility matrix: client 2.x.0 supports OpenSearch 1.x to 2.x; client 3.x.0 supports 1.x to 3.x. JDK 8/11/17/21 for both lines. (https://github.com/opensearch-project/opensearch-java/blob/main/COMPATIBILITY.md)
+* `ApacheHttpClient5Transport` is the default and recommended transport; `RestClientTransport` and `org.opensearch.client.RestClient` are deprecated and will be removed. (https://docs.opensearch.org/latest/clients/java/)
+* OpenSearch 3.0 brings HTTP/2 and `ApacheHttpClient5Transport` switches to HTTP/2 when available; `HttpVersionPolicy` can be forced through `setHttpClientConfigCallback`. (https://github.com/opensearch-project/opensearch-java/blob/main/USER_GUIDE.md)
+* 3.0.0 breaking changes (https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/CHANGELOG.md and https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/UPGRADING.md): more conservative URL path encoding, `searchAfter` and `Hit.sort` now `FieldValue`, `listAllPit` -> `getAllPits`, several type corrections, removal of ES-only namespaces (`features`, `shutdown`, `termsEnum`, `indices.upgrade`). Nothing in the transport builders or `JacksonJsonpMapper` changed. One undocumented change does hit this repo: `DeleteByQueryRequest.Builder.refresh(...)` takes the `Refresh` enum instead of `boolean` (see empirical test below).
+* Empirical compile check (scratch copy, `opensearch.version` set to 3.10.0): exactly one compile error, CLIENT line 427 `.refresh(true)` ("boolean cannot be converted to org.opensearch.client.opensearch._types.Refresh"). After changing it to `Refresh.True`, `mvn test-compile` and all 233 unit tests pass. Resolved tree with 3.10.0: httpcore5 5.4.3, httpclient5 5.6.3 (BOM), plus a new `commons-logging:1.3.6` transitive. The 3.x client POM is built against httpclient5 5.6 / httpcore5 5.4.2, so the BOM versions are in range.
+* httpcore5/httpclient5 constraint with the current 2.10.0 client: it was built against httpclient5 5.3.1 / httpcore5 5.2.4 but runs on 5.6.3 / 5.4.3 here; the repo already carries a workaround for the 5.6 content-compression behaviour (CLIENT lines 124 to 130). Moving to the 3.x client removes that version skew.
+* Elasticsearch-compatible endpoints: the compatibility matrix lists only OpenSearch. A maintainer answer on the forum says clients generally only bridge adjacent majors (ES 7.10 <-> OpenSearch 1.x) and that 2.x clients against ES 7.10 hit renamed APIs. (https://forum.opensearch.org/t/opensearch-api-compatibility-with-elastic-search/17647). Legacy Elasticsearch 7.10 domains on AWS should be declared out of scope.
+* SigV4: `AwsSdk2Transport(SdkHttpClient, endpointWithoutScheme, "es" | "aoss", Region, AwsSdk2TransportOptions)`; credentials from the default provider chain; `aws-crt-client` is the recommended SdkHttpClient and the AWS SDK `ApacheHttpClient` is discouraged (it cannot send bodies on GET/DELETE and `AwsSdk2Transport` throws `TransportException` for such calls). (https://docs.opensearch.org/latest/clients/java/#connecting-to-amazon-opensearch-service and https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md). The AWS SDK artifacts are compileOnly in the client, so the application must declare them (release notes for #628 in https://github.com/opensearch-project/opensearch-java/releases/tag/v3.0.0).
+
+### 1.2 Amazon OpenSearch Service (managed domains)
+
+* Engine versions offered today: OpenSearch 3.5, 3.3, 3.1, 2.19, 2.17, 2.15, 2.13, 2.11, 2.9, 2.7, 2.5, 2.3, 1.3 to 1.0, plus legacy Elasticsearch up to 7.10. 2.3 to 2.9 left standard support in Nov 2025; 2.11 to 2.17 end Nov 2027. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html)
+* Endpoint and TLS: "OpenSearch Service only accepts connections over port 80 (HTTP) or 443 (HTTPS)" (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html); TLS 1.2 required, 1.3 recommended (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/infrastructure-security.html). No custom port; endpoints are `https://search-<domain>-<id>.<region>.es.amazonaws.com` (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/createupdatedomains.html).
+* FGAC: master user is either an IAM principal or an internal-database user. With an internal master user "you can use HTTP basic authentication (as well as IAM credentials)". If the domain access policy names IAM principals, clients must sign with SigV4; "You can't sign a request with a username and password and IAM credentials", so AWS recommends an access policy that does not require signed requests when using basic auth. Master user roles are `all_access` and `security_manager`. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html)
+* Signing service name for domains is `es`; for Serverless it is `aoss` and `x-amz-content-sha256` is required. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-comparison.html)
+* Supported operations: the per-version operation lists for managed domains include `/_bulk`, `/_delete_by_query`, `/_refresh`, `/_reindex`, `/_update_by_query` and all index-path operations, so every call this library makes is available on a managed domain. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html)
+* Request payload limits: 10 MiB on t2/t3 and `*.large` instance types, 100 MiB on xlarge and above. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html)
+* Permissions on a FGAC domain follow the security plugin model (section 1.4).
+
+### 1.3 Amazon OpenSearch Serverless (AOSS)
+
+* Supported API table (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html): `aoss:CreateIndex` (PUT index), `aoss:DescribeIndex` (HEAD index, GET mappings/settings), `aoss:WriteDocument` (POST _bulk, DELETE index/_doc/id, PUT index/_doc/id "for search collection types only", POST index/_doc), `aoss:ReadDocument` (GET/POST _search, _count, _msearch, PIT). `_delete_by_query`, `_update_by_query`, `_reindex` and `_refresh` do not appear anywhere in the table. Aliases exist under `aoss:CreateCollectionItems` / `UpdateCollectionItems`.
+* Collection types: Search, Vector search, Time series. Custom document IDs and upserts are only allowed for search collections. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-overview.html and the Python note in https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html)
+* Settings: "The number of shards, number of intervals, and refresh interval are not modifiable and are handled by OpenSearch Serverless"; refresh interval is approximately 10 seconds for search and time series collections. (serverless-overview.html)
+* Clients must sign with service `aoss`, include `x-amz-content-sha256`, and "must be compatible with OpenSearch 3.x" (serverless-overview.html; the older serverless-clients.html page still says 2.x). The OpenSearch Java docs note AOSS does not support the `refresh` parameter and does not serve the root endpoint (https://docs.opensearch.org/latest/clients/java/#connecting-to-amazon-opensearch-serverless).
+* Data access policy minimum for an ingesting principal: `aoss:CreateIndex`, `aoss:WriteDocument`, `aoss:UpdateIndex` on `index/<collection>/<index>`; this library also needs `aoss:DescribeIndex` (exists check) and `aoss:ReadDocument` (search). (serverless-clients.html)
+* Quotas: 1000 indexes per collection (https://docs.aws.amazon.com/general/latest/gr/opensearch-service.html). No per-request payload limit is documented on that page.
+* Which of this library's calls fail on AOSS: the connection itself (basic auth transport, no SigV4), `deleteDocumentsForTable` (`_delete_by_query`, CLIENT line 392), `deleteDocumentsIndexedBefore` (`_delete_by_query` with `refresh=true`, CLIENT line 424), `refreshIndex` (`_refresh`, CLIENT line 579), and all indexing with custom `_id` unless the collection is of type Search. `ensureIndexExists`, bulk index/delete, single delete, and search with highlight and from/size are within the supported surface.
+
+### 1.4 Self-managed OpenSearch with the security plugin
+
+* Demo config and Docker image: since 2.12 a custom admin password in `OPENSEARCH_INITIAL_ADMIN_PASSWORD` is required or the node does not start; the password must be 8 to 100 chars, mixed classes, and rated "strong" by zxcvbn. `DISABLE_SECURITY_PLUGIN=true` disables the plugin; the demo config installs self-signed certificates. (https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/ and https://docs.opensearch.org/latest/security/configuration/demo-configuration/)
+* TLS with self-signed certs: the official Java client example builds an `SSLContext` (trust-all for demo, or a truststore) and installs it via `ClientTlsStrategyBuilder` + `PoolingAsyncClientConnectionManager` inside `setHttpClientConfigCallback`; the RestClient transport variant relies on the JVM truststore system properties. (https://docs.opensearch.org/latest/clients/java/#connecting-to-opensearch)
+* Auth backends: HTTP basic against the internal user database; client certificate (mTLS) via `plugins.security.ssl.http.clientauth_mode` and a `clientcert` auth domain mapped to roles by CN (https://docs.opensearch.org/latest/security/authentication-backends/client-auth/); JWT bearer tokens (HMAC, RSA, ECDSA, JWKS since 3.3) in the `Authorization` header (https://docs.opensearch.org/latest/security/authentication-backends/jwt/); API tokens with permissions bound to the token were introduced in OpenSearch 3.7 (https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-3.7.0.md).
+* Permissions for this library's operations (default action groups, https://docs.opensearch.org/latest/security/access-control/default-action-groups/): index-level `create_index` (`indices:admin/create`, `indices:admin/mapping/put`) for `ensureIndexExists`; `read` or `search` for `_search`; `write` (`indices:data/write*`, which covers index, bulk items, delete, and `delete/byquery`) and `delete`; `indices:admin/refresh*` is not in any default group except `manage` (`indices:admin/*`) and must be added for `refreshIndex`; cluster-level `cluster_composite_ops` for `indices:data/write/bulk`. The exists check (HEAD index) also needs an `indices:admin/*` read permission; verify the exact action name against the permissions reference when writing the README role.
+* Versions: 2.x is in maintenance (latest 2.19.6, maintained until 4.0 GA); 3.x is current (3.9.0 released 2026-09-29; 3.10.0 scheduled Nov 2026). (https://opensearch.org/releases/)
+
+### 1.5 Testcontainers
+
+* Docker Hub has `opensearchproject/opensearch` tags 2.19.0 to 2.19.6 and 3.0.0 to 3.9.0 (https://hub.docker.com/v2/repositories/opensearchproject/opensearch/tags).
+* The official module `org.opensearch:opensearch-testcontainers` is at 4.1.0 (requires Testcontainers 2.0.0+ and JDK 17+); 3.0.x works with Testcontainers 1.20.0+ and JDK 21. `OpenSearchContainer` disables security by default and offers `withSecurityEnabled()` with HTTPS plus generated credentials. (https://github.com/opensearch-project/opensearch-testcontainers/blob/main/README.md and https://repo1.maven.org/maven2/org/opensearch/opensearch-testcontainers/maven-metadata.xml)
+
+### 1.6 Server-side behaviours relevant to the mapping and query design
+
+* `index.max_result_window` default 10000 caps `from + size` (https://docs.opensearch.org/latest/install-and-configure/configuring-opensearch/index-settings/).
+* Search analyzer resolution order: query `analyzer`, field `search_analyzer`, index `default_search`, field `analyzer`, `standard`. The docs' edge n-gram example sets `analyzer: edge_ngram_analyzer` with `search_analyzer: standard` so the query is not n-grammed. (https://docs.opensearch.org/latest/analyzers/search-analyzers/)
+* Edge n-gram filter: `max_gram` too low means long search terms do not match; the docs suggest a `truncate` filter as search analyzer to mitigate. (https://docs.opensearch.org/latest/analyzers/token-filters/edge-ngram/)
+* Dynamic mapping: strings become `text` + `keyword` subfield, or `date` when `date_detection` (default on) matches `strict_date_optional_time`; JSON numbers become `long`/`float`; `index.mapping.total_fields.limit` defaults to 1000; `flat_object` or `dynamic_templates` are the recommended defences. (https://docs.opensearch.org/latest/mappings/ and https://docs.opensearch.org/latest/mappings/mapping-explosion/)
+
+## 2. Compatibility matrix
+
+| Target | Works today? | What breaks | Change needed |
+| --- | --- | --- | --- |
+| OpenSearch 2.x self-managed, security disabled (the integration-test setup) | Yes | Nothing; this is the only tested configuration (image 2.11.0) | Test against 2.19.x as well (F10) |
+| OpenSearch 2.x self-managed, security plugin, basic auth over TLS | Partly | Demo or private-CA certificates fail the JVM default trust check; no truststore or insecure option exists (CONFIG exposes only `useSsl`). Only basic auth is wired; no client-cert, JWT or API-token path | Add TLS trust configuration (F1), document required roles (F13); optional auth-header or SSLContext hooks (F3) |
+| OpenSearch 3.x self-managed | Not supported by the pinned client | opensearch-java 2.10.0 is documented for 1.x to 2.x only; the server negotiates HTTP/2 with the Apache 5 transport | Bump to opensearch-java 3.10.0; one-line change at CLIENT line 427 (F2); add 3.x to the IT matrix (F10) |
+| AWS managed domain 2.x or 3.x, FGAC with internal-database master user, HTTP basic auth | Yes for 2.x (3.x after F2) | Needs `opensearchPort = 443`, `useSsl = true`, and a domain access policy that does not require signed requests. AWS certificates are publicly trusted so TLS works. Every API the library calls is on the managed-domain supported list. Payload limit 10 MiB on small instances with count-only batching (F8); no timeouts (F9) | F2 for 3.x domains, F8, F9, README guidance |
+| AWS managed domain, IAM master user or IAM principals in the access policy (SigV4) | No | Client only does basic auth; AWS rejects unsigned requests when IAM principals are in the policy; no `AwsSdk2Transport` path, region, or service-name configuration | Add an optional SigV4 transport mode: `AwsSdk2Transport` with service `es`, `Region`, default credential chain, `aws-crt-client`; declare the AWS SDK artifacts as optional dependencies (F3) |
+| Amazon OpenSearch Serverless (AOSS) | No | Transport (SigV4 `aoss`, `x-amz-content-sha256`); `_delete_by_query` used by FullReindexStep and ReconcileIndexStep; `_refresh` used by ReconcileIndexStep; custom `_id` only on Search collections; refresh lag ~10 s; client must be 3.x | Declare unsupported for 1.0. Supporting it later means: SigV4 transport (F3), replace delete-by-query with search + bulk delete, replace refresh with a bounded wait, and require Search collections (F4) |
+| Legacy Elasticsearch 7.10 domains or other ES-compatible endpoints | Untested, not documented | Client matrix lists only OpenSearch; renamed APIs across the 7.10 to 2.x gap | Declare out of scope |
+
+## 3. Findings
+
+Severity: High = breaks a target platform or returns wrong results; Medium = operational risk or avoidable failure; Low = hygiene.
+
+| ID | Severity | Finding | Evidence | Recommended change | 1.0 blocker |
+| --- | --- | --- | --- | --- | --- |
+| F1 | High | No TLS trust configuration. `useSsl` only flips the scheme; there is no truststore path, PEM CA, or insecure toggle, so self-managed clusters with demo or private-CA certificates fail the handshake. Hostname verification cannot be relaxed either | CLIENT lines 98 to 139; CONFIG lines 54, 420 to 444; official pattern in https://docs.opensearch.org/latest/clients/java/#connecting-to-opensearch | Add `trustStorePath`/`trustStorePassword` (or `caCertificatePem`) and an explicit `insecureSkipTlsVerify` flag; build an `SSLContext` and install it through `ClientTlsStrategyBuilder` + `PoolingAsyncClientConnectionManager` in the existing `setHttpClientConfigCallback`. Log a warning when verification is disabled | Yes |
+| F2 | High | opensearch-java pinned to 2.10.0 (2.x line, documented for OpenSearch 1.x to 2.x). AWS already offers 3.1 to 3.5 and 3.x is the current community line. Empirical check: bumping to 3.10.0 yields a single compile error at CLIENT line 427 (`refresh(true)` must become `refresh(Refresh.True)`); 233 unit tests pass afterwards. 3.x also drops the transitive `opensearch-rest-client` and matches the BOM's httpclient5 5.6 / httpcore5 5.4 | pom.xml line 24; COMPATIBILITY.md; CHANGELOG/UPGRADING links in section 1.1; scratch compile log | Bump `opensearch.version` to 3.10.0, fix line 427, re-run ITs; keep the `disableContentCompression()` workaround until proven unnecessary | Yes |
+| F3 | High (for AWS IAM) | Only HTTP basic auth is supported. No SigV4 (`AwsSdk2Transport`), no region or service name, no bearer-token or client-certificate hooks. An IAM master user or any IAM principal in the domain access policy makes the domain unreachable | CLIENT lines 109 to 137; CONFIG lines 52, 53; fgac.html; clients/java docs | Introduce an `authMode` (BASIC, AWS_SIGV4, NONE) plus `awsRegion`, `awsServiceName` (default `es`), optional credentials provider; build `AwsSdk2Transport` with `AwsCrtHttpClient`. Declare `software.amazon.awssdk:aws-crt-client` and `auth` as `<optional>true</optional>` rather than relying on the quicksight transitive. Optionally expose a `Consumer<HttpAsyncClientBuilder>` hook for JWT/API-token headers and mTLS | No (declare unsupported in README) |
+| F4 | High (for AOSS) | AOSS cannot be targeted: SigV4 `aoss` missing; `_delete_by_query` (CLIENT 392, 424) and `_refresh` (CLIENT 579) are not in the AOSS supported-operations table; custom `_id` requires a Search collection; `refresh=true` parameter unsupported | serverless-genref.html; serverless-overview.html; CLIENT lines 388 to 445, 575 to 585 | Declare AOSS unsupported for 1.0. For a later release: F3, implement table wipe and stale-document removal as search (PIT or `search_after`) + bulk delete, replace `refreshIndex` with a polling wait, document the Search-collection requirement and the `aoss:CreateIndex/DescribeIndex/WriteDocument/ReadDocument/UpdateIndex` data-access policy | No |
+| F5 | High | Edge n-gram analyzer (min 2, max 20) is set as the field `analyzer` with no `search_analyzer`, so the query string is also n-grammed. A search for "search" becomes se, sea, sear, searc, search and matches every document containing any token starting with "se", inflating hits and flattening relevance. Tokens longer than 20 chars only index their first 20 grams, and single-character terms never match | CLIENT lines 173 to 192 (`searchableText` property), 472 to 475 (must clause on `searchableText`); search-analyzers doc | Add a second analyzer (`standard` tokenizer + `lowercase`) and set it as `search_analyzer` on `searchableText`; consider `truncate` to 20 in the search analyzer or `preserve_original` so long terms still match; keep `min_gram 2` but document it. Requires index recreation (bump an index-version suffix or document a full reindex) | Yes |
+| F6 | High | `fieldValues` is a dynamic `object` that receives raw `QRecord` values (`Integer`, `BigDecimal`, `Instant`, `LocalDate`, `Boolean`, `String`). Dynamic mapping turns them into `long`/`float`/`date`/`boolean`; the first table to index a field name fixes its type for the whole index, so a `status` that is a string in one table and an integer in another causes `mapper_parsing_exception` on the second table. The boost `multi_match` clauses target `fieldValues.<field>` without `lenient: true`, so a non-numeric or non-date search term against a numeric or date `fieldValues` field makes the whole search request fail. Field count also grows with every configured field across all tables toward `index.mapping.total_fields.limit` (1000) | IndexingUtils lines 144 to 156; CLIENT lines 192, 525 to 565; mappings docs (date detection on by default; dynamic rules); mapping-explosion doc | Map `fieldValues` with a `dynamic_templates` rule that maps every `fieldValues.*` string to `text` (edge n-gram analyzer + standard search analyzer) and set `date_detection: false`; convert values to strings in `IndexingUtils` (`getValueString`) so types can never conflict; set `lenient(true)` on the boost `multi_match` queries; consider `flat_object` if per-field boosting is not needed | Yes |
+| F7 | Medium | `limit` and `offset` are unbounded; `from + size > 10000` returns a 400 (`index.max_result_window`) and large `size` values are accepted verbatim | QuickSearchAction lines 46 to 79; CLIENT 497 to 502; index-settings doc | Cap `limit` (for example 100) and reject or clamp `offset + limit > 10000` with a clear error; document that deep paging is not supported | No |
+| F8 | Medium | Bulk batching is count-based only (`bulkBatchSize` default 500) with no byte cap. On AWS small instance types the HTTP payload limit is 10 MiB; wide records with large text fields can exceed it and the whole batch fails with 413 | CLIENT lines 241 to 274; CONFIG line 58; AWS limits.html | Flush a batch when its serialized size approaches a configurable byte limit (default 5 MiB) in addition to the count; surface 413 clearly | No |
+| F9 | Medium | No connect, response or connection-request timeouts and no pool sizing; a hung node blocks the caller indefinitely; single host only, no failover list | CLIENT lines 105 to 139; no `RequestConfig`, `setConnectionManager`, or timeout call anywhere in `src/main/java` | Add `connectTimeoutMs`, `responseTimeoutMs`, `maxConnections` to CONFIG and apply via `setDefaultRequestConfig` and `PoolingAsyncClientConnectionManagerBuilder`; accept a list of hosts (`ApacheHttpClient5TransportBuilder.builder(HttpHost...)`) | Yes |
+| F10 | Medium | Integration tests pin `opensearchproject/opensearch:2.11.0` (Oct 2023). Nothing exercises 2.19.x or any 3.x, although AWS offers 2.19 and 3.1 to 3.5 and the community line is 3.9 | OpenSearchIntegrationTest line 76; IndexDriftIntegrationTest line 81; ConfigDrivenIndexingIntegrationTest line 76; releases page; Docker Hub tags | Parameterize the image via a system property (`-Dopensearch.image=...`) and run two CI jobs: `2.19.6` and `3.9.0`. Optionally adopt `org.opensearch:opensearch-testcontainers` 3.0.2 (Testcontainers 1.20+, JDK 21) now and 4.1.0 after moving to Testcontainers 2.x | Yes |
+| F11 | Low | ITs disable security via `plugins.security.disabled=true` while also passing `OPENSEARCH_INITIAL_ADMIN_PASSWORD=Admin123!`. Harmless today, but if security is ever enabled that password is unlikely to pass the zxcvbn "strong" check required since 2.12, and the documented image switch is `DISABLE_SECURITY_PLUGIN=true` | the three IT classes lines 79 to 85; demo-configuration doc | Use `DISABLE_SECURITY_PLUGIN=true` (and `DISABLE_INSTALL_DEMO_CONFIG=true`); drop the password, or when adding a secured IT, generate a strong one and connect over HTTPS with F1 in place | No |
+| F12 | Low | `opensearch-rest-client:2.12.0` and `httpcore5` 5.2.x come in transitively from the 2.10.0 client even though `RestClientTransport` is never used; version skew with the BOM's httpclient5 5.6.3 already required the compression workaround | dependency tree; CLIENT lines 124 to 130 | Resolved by F2; otherwise exclude `opensearch-rest-client` | No |
+| F13 | Low | No documented least-privilege role for the security plugin or AWS FGAC | none in repo | README: index pattern `<opensearchIndexName>`, groups `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*` and an `indices:admin/*` read for the exists check; cluster `cluster_composite_ops`. For AWS FGAC map the same to the basic-auth user; for AOSS list the five `aoss:*` permissions | No |
+| F14 | Low | Composite `_id` is `sourceTable + ":" + recordId`; `recordId` is mapped `keyword` (correct for term filters and mixed PK types). A table name containing ":" would be ambiguous, and `_id` is limited to 512 bytes | OpenSearchDocument lines 184 to 187; CLIENT line 188 | Validate table names against ":" in CONFIG; keep `keyword` | No |
+| F15 | Info | Against 3.x servers the Apache 5 transport negotiates HTTP/2 automatically; nothing in the code forces a version | USER_GUIDE.md HTTP/2 note | No change; mention in README; `HttpVersionPolicy.FORCE_HTTP_1` is available if a proxy misbehaves | No |
+
+## 4. Recommended "supported platforms" statement for the 1.0 README
+
+Supported (tested in CI):
+
+* Self-managed OpenSearch 2.19.x and 3.x (latest minor), reached over HTTP or HTTPS with a certificate the JVM trusts or a configured truststore, with the security plugin disabled or with HTTP basic authentication against the internal user database. Required role: `create_index`, `read`, `write`, `delete` on the quick-search index plus `indices:admin/refresh*`, and the cluster-level `cluster_composite_ops` group.
+* Amazon OpenSearch Service managed domains running OpenSearch 2.x or 3.x with fine-grained access control, an internal-database master user (or a mapped internal user), HTTP basic authentication, `opensearchPort = 443`, `useSsl = true`, and a domain access policy that does not require signed requests. Keep `bulkBatchSize` small enough for the instance type's HTTP payload limit (10 MiB on t3 and `*.large`).
+
+Not supported in 1.0:
+
+* IAM / SigV4 authentication to Amazon OpenSearch Service (IAM master user or IAM principals in the access policy).
+* Amazon OpenSearch Serverless (requires SigV4 with service `aoss` and does not offer `_delete_by_query` or `_refresh`, which the full-reindex and reconcile processes use).
+* Legacy Elasticsearch 7.10 domains and other Elasticsearch-compatible endpoints.
+* OpenSearch 1.x.
+
+Pre-conditions for the statement above: F1, F2, F5, F6, F9 and F10 must land first; the statement should name the exact image tags the CI matrix runs.
+
+## 5. Source list
+
+* https://github.com/opensearch-project/opensearch-java/blob/main/COMPATIBILITY.md
+* https://docs.opensearch.org/latest/clients/java/
+* https://github.com/opensearch-project/opensearch-java/blob/main/USER_GUIDE.md
+* https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md
+* https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/CHANGELOG.md
+* https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/UPGRADING.md
+* https://github.com/opensearch-project/opensearch-java/releases/tag/v3.0.0
+* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/maven-metadata.xml
+* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/2.10.0/opensearch-java-2.10.0.pom
+* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/3.9.0/opensearch-java-3.9.0.pom
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/infrastructure-security.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-overview.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-comparison.html
+* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html
+* https://docs.aws.amazon.com/general/latest/gr/opensearch-service.html
+* https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+* https://docs.opensearch.org/latest/security/configuration/demo-configuration/
+* https://docs.opensearch.org/latest/security/access-control/default-action-groups/
+* https://docs.opensearch.org/latest/security/authentication-backends/client-auth/
+* https://docs.opensearch.org/latest/security/authentication-backends/jwt/
+* https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-3.7.0.md
+* https://opensearch.org/releases/
+* https://hub.docker.com/v2/repositories/opensearchproject/opensearch/tags
+* https://github.com/opensearch-project/opensearch-testcontainers/blob/main/README.md
+* https://docs.opensearch.org/latest/install-and-configure/configuring-opensearch/index-settings/
+* https://docs.opensearch.org/latest/analyzers/search-analyzers/
+* https://docs.opensearch.org/latest/analyzers/token-filters/edge-ngram/
+* https://docs.opensearch.org/latest/mappings/
+* https://docs.opensearch.org/latest/mappings/mapping-explosion/
+* https://forum.opensearch.org/t/opensearch-api-compatibility-with-elastic-search/17647

docs/audits/2026-10-04/03-authentication-security.md

@@ -0,0 +1,156 @@
+# Audit 03: Authentication, secrets handling, and connection security
+
+Repo: `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` at `7cd94ae` (branch `feature/GH-6-repin-and-drift-fix`), read-only.
+QQQ reference: `/Users/james.maes/Git.Local/QRunIO/qqq` at tag `v4.1.0-RC.1`.
+Resolved runtime versions (from `mvn -B dependency:tree`): opensearch-java 2.10.0, opensearch-rest-client 2.12.0 (runtime), httpclient5 5.6.3, httpcore5 5.4.3, jackson 2.21.5, log4j 2.25.5, AWS SDK v2 2.41.10 (via qqq-backend-core), AWS SDK v1 1.12.797 (via qqq-backend-core).
+
+Paths below are relative to `src/main/java/com/kingsrook/qbits/quicksearch/` unless stated otherwise.
+
+## Findings
+
+| ID | Severity | Finding | Evidence | Recommended change | 1.0 blocker |
+|----|----------|---------|----------|--------------------|-------------|
+| A1 | Critical | `QuickSearchAction` returns `tableName`, `recordId`, `recordLabel` and a `highlightSnippet` (fragments of `searchableText`, which is the concatenation of every configured field value) for every indexed table, with no check of the caller's session, table permission, or record security locks. The caller-supplied `tableName` filter is also unchecked. Any authenticated (or anonymous, if the instance allows it) caller can enumerate labels and field values across all indexed tables. | `actions/QuickSearchAction.java:81-121` (reads client and all `discoveredTables` from the static context, maps every hit straight to output); `opensearch/QuickSearchOpenSearchClient.java:466-510` (only filter is the optional caller-supplied `sourceTable` term); `processes/IndexingUtils.java:83,136-165` (searchableText and fieldValues built from all configured fields) | See section "A1 fix" below: make `QuickSearchInput` extend `AbstractActionInput`, compute the set of tables the session may read with `PermissionsHelper.hasTablePermission(input, table, READ)`, reject or ignore a requested `tableName` outside that set, pass the allowed set into the client as a `terms` filter on `sourceTable`, then post-filter hits per table through `QueryAction` by primary key so record security locks apply. Add negative tests. | Yes |
+| A2 | High | HTTP basic credentials are sent in cleartext when `useSsl` is false (the default), and nothing in `validate()` objects to username/password combined with plain HTTP. The README example and the qqq-all `DemoQBits` default `OPENSEARCH_SSL` to false. | `QuickSearchQBitConfig.java:54` (`useSsl = false`), `:103-112` (only checks that username and password are set together); `opensearch/QuickSearchOpenSearchClient.java:98-99,113-119`; `README.md:303`; qqq-all `DemoQBits.java:156-158` | In `validate()`, add an error when `authMode` is BASIC (or username/password set) and TLS is off, unless an explicit `allowPlaintextCredentials` (dev only) is true. For 1.0 change the default scheme to HTTPS (see config sketch) so that opting into plaintext is deliberate. | Yes |
+| A3 | High | The password lives as a plain `String` on a `Serializable` `QBitConfig`, and the downstream host attaches that same object to `QBitMetaData` which is stored in `QInstance.getQBits()`. Anything that serializes the `QInstance` (for example `QInstanceAdapter.qInstanceToJson`, used by dev tooling and tests) or any code that walks `getQBits()` can read the password through the public getter. No `@JsonIgnore`, no redaction, no `toString()` override (the default `Object.toString()` happens not to leak it, but only by accident). | `QuickSearchQBitConfig.java:52-53,389-392`; qqq `QBitMetaData.java` (field `QBitConfig config`, `getConfig()`); qqq `QInstance.java:114,1800-1836`; qqq `adapters/QInstanceAdapter.java:46,57`; qqq-all `DemoQBits.java:157,162-163`; qqq-all `FullQBitsTest.java:78` (casts the config back out of `getQBits()`) | Do not keep the raw secret on the metadata object. Options, cheapest first: (1) mark `getOpensearchPassword()` (and any token/secret getter) `@JsonIgnore` and add a `toString()` that redacts; (2) accept `${env.X}` / `${prop.X}` references and resolve them with `QMetaDataVariableInterpreter` only inside the client constructor, so the config object stores the reference string, not the value (this is exactly how `RDBMSBackendMetaData.enrich()`, `MongoDBBackendMetaData.enrich()` and `S3BackendMetaData.enrich()` handle `password`/`secretKey`); (3) longer term, resolve credentials through a `QCodeReference` provider so no secret ever sits on metadata. Do (1) and (2) for 1.0. | Yes |
+| A4 | Medium | Only two auth modes exist: none and HTTP basic. There is no AWS SigV4 support, which blocks Amazon OpenSearch Service domains using IAM (and OpenSearch Serverless entirely), and there is no way to inject a bearer token, API-key header, or client certificate without forking the client. | `opensearch/QuickSearchOpenSearchClient.java:94-147` (hard-wires `ApacheHttpClient5TransportBuilder` + `BasicCredentialsProvider`) | Add an `authMode` enum (NONE, BASIC, AWS_SIGV4) and a transport-customizer SPI for the rest. `AwsSdk2Transport` ships in opensearch-java 2.10.0 and the AWS SDK v2 core modules are already on the compile classpath via qqq-backend-core, so SigV4 is mostly wiring (details in section 1). | Yes (SigV4); No (bearer, API key, mTLS via SPI) |
+| A5 | Medium | TLS trust is not configurable. The transport uses `SSLContext.getDefault()` (JVM trust store) and the httpclient5 default hostname verifier, so certificates are verified correctly by default, but a private CA, a custom trust store, or client-certificate material can only be supplied through JVM system properties. There is no explicit, loud "insecure" switch for local development, which pushes developers toward global `-Djavax.net.ssl.trustStore` hacks or disabling the security plugin. | opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` bytecode: `ClientTlsStrategyBuilder.create().setSslContext(SSLContext.getDefault())`; `opensearch/QuickSearchOpenSearchClient.java:105-137` never sets a TLS strategy | Add `tls.caCertificatePath` (PEM) or `tls.trustStorePath` + `tls.trustStorePassword` (reference-interpreted), `tls.hostnameVerification` (default true), and `tls.insecureSkipVerify` (default false; refuse unless host is loopback or `allowInsecureInProduction` is set; log a WARN at startup). Build an `SSLContext` via `SSLContextBuilder.loadTrustMaterial(...)` and pass it through `ClientTlsStrategyBuilder` on a `PoolingAsyncClientConnectionManager`. | No (strongly recommended) |
+| A6 | Medium | The three produced processes (`quickSearchBasepullIndex`, `quickSearchFullReindex`, `quickSearchReconcileIndex`), the two operational tables, and the `quickSearchAdmin` app carry no `QPermissionRules`, so they inherit `QInstance.defaultPermissionRules` (QQQ default is `NOT_PROTECTED`). In a host that has not tightened the instance default, any user can trigger a full reindex (which wipes the index first, a search blackout) or read/edit the run tables. | `QuickSearchQBitProducer.java:457-549` (tables and processes built without `withPermissionRules`), `:246-260` (app); `grep PermissionRules src/main` returns nothing; qqq `QInstance.java:140` (`defaultPermissionRules = QPermissionRules.defaultInstance()`); qqq `QJavalinProcessHandler.java:404,418` (middleware enforces whatever rules the process declares) | Set explicit rules in the producer: processes and app `HAS_ACCESS_PERMISSION`, tables `READ_WRITE_PERMISSIONS`, with a stable `permissionBaseName` (for example `quickSearchAdmin`) so one grant covers the admin surface. Make it overridable via an `adminPermissionRules` config field; never default to `NOT_PROTECTED`. | Yes |
+| A7 | Medium | Transport timeouts are the opensearch-java defaults: connect 1 s, response 30 s, max 10 connections per route / 30 total, automatic httpclient retries disabled. None are configurable. 1 s connect is tight for cross-region or cold AWS endpoints; 30 s response is marginal for `deleteByQuery(... refresh=true)` on large tables (`deleteDocumentsIndexedBefore`, `deleteDocumentsForTable`) and for 500-document bulk requests on a slow cluster. There is no explicit user-agent. | opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` constants `DEFAULT_CONNECT_TIMEOUT_MILLIS=1000`, `DEFAULT_RESPONSE_TIMEOUT_MILLIS=30000`, `DEFAULT_MAX_CONN_PER_ROUTE=10`, `DEFAULT_MAX_CONN_TOTAL=30`, and `HttpAsyncClientBuilder.disableAutomaticRetries()` in its `createHttpClient`; `opensearch/QuickSearchOpenSearchClient.java:105-137` sets no `RequestConfigCallback` | Add `connectTimeoutMillis` (default 5000), `responseTimeoutMillis` (default 60000), `maxConnections` (default 30) to config; apply via `setRequestConfigCallback` and `PoolingAsyncClientConnectionManagerBuilder`. Set a default header `User-Agent: qbit-quick-search/<version>` so cluster-side logs can attribute traffic. Keep retries off at the HTTP layer; the indexing steps already record per-item failures. | No |
+| A8 | Medium | The OpenSearch client is never closed on host shutdown. `QuickSearchOpenSearchClient` is `Closeable` and `SynchronousIndexEventPublisher.close()` calls `client.close()`, but nothing invokes `publisher.close()` or `client.close()` outside tests; the static `QuickSearchQBitContext` has no lifecycle hook. The pooled connections and the httpclient5 I/O reactor thread leak until JVM exit, and a hot-reload of the `QInstance` (which re-runs `produce`) creates a second transport while the first stays open. | `QuickSearchQBitContext.java:34-37,152-158`; `QuickSearchQBitProducer.java:159-203`; `publisher/SynchronousIndexEventPublisher.java:179-181`; `opensearch/QuickSearchOpenSearchClient.java:595-612` | In `produce`, close any previously stored client/publisher before replacing them, and register a `Runtime.getRuntime().addShutdownHook` (or the QQQ instance shutdown hook if the host exposes one) that calls `QuickSearchQBitContext.shutdown()`. | No |
+| A9 | Low | Search terms are logged at DEBUG (`"term", normalizedTerm`). Search terms are frequently PII (names, emails, order numbers). Not a secret leak, but worth knowing before enabling DEBUG in production. | `actions/QuickSearchAction.java:89` | Log term length or a hash at DEBUG; log the raw term only at TRACE. | No |
+| A10 | Low | `QuickSearchQBitConfig implements QBitConfig` (which `extends Serializable`) but holds a non-`Serializable` `IndexEventPublisher` and a `List<Class<?>>`. Java serialization of the config (which QQQ permits by contract) would throw `NotSerializableException` when a custom publisher is set. Not a security issue, but it is the same contract that A3 relies on. | `QuickSearchQBitConfig.java:45,60,63`; qqq `QBitConfig.java` (`public interface QBitConfig extends Serializable`) | Mark `indexEventPublisher` `transient` or replace it with a `QCodeReference`. | No |
+| A11 | Low | `limit` and `offset` are unbounded. A caller can request `limit=100000`; OpenSearch rejects `from+size > index.max_result_window` (10000) with an error that surfaces as a `QException`, and smaller but large pages pull `fieldValues` for every hit into heap. Combined with A1 this makes bulk enumeration trivial. | `actions/QuickSearchAction.java:78-79` | Clamp `limit` to a `maxLimit` (default 100) and `offset + limit` to 10000, and document it. | No |
+| A12 | Info | No committed production credentials. Test fixtures use placeholders (`"user"`, `"secret"`, `"pass"`, `"admin"`) and the Testcontainers `OPENSEARCH_INITIAL_ADMIN_PASSWORD=Admin123!` with `plugins.security.disabled=true`, so the password is never used. No `.env`, properties, or YAML with secrets in the tree; `.gitignore` excludes `.env` and `.env.local`. The README example hard-codes `"secretPassword"` as a literal, which teaches the wrong pattern. | `src/test/.../QuickSearchQBitConfigTest.java:229-278,509-555`; `src/test/.../QuickSearchOpenSearchClientTest.java:89-90`; `src/test/.../OpenSearchIntegrationTest.java:75-81` (and the two other integration tests); `README.md:115-125` | Change the README example to `.withOpensearchPassword("${env.OPENSEARCH_PASSWORD}")` once A3 lands. | No |
+| A13 | Info | Logging and exception messages do not include the host, port, username or password. `validate()` emits only field names, `backendName` and table names; the client's `QException`s include `indexName`, document ids, the search term, and the underlying `e.getMessage()` (which for connection failures from httpclient5 can include `host:port`, acceptable). | `QuickSearchQBitConfig.java:73-159`; `opensearch/QuickSearchOpenSearchClient.java:145,199,222,351,376,401,443,508,583`; `QuickSearchQBitProducer.java:121,166,180` | Keep as is; add the `toString()` redaction from A3 so a future `LOG.info("config", config)` cannot regress this. | No |
+| A14 | Info | Supply chain: no concrete CVE affects the resolved runtime set. httpclient5 5.6.3 is past the fix for CVE-2025-27820 (PSL/hostname-verification bypass in 5.4.0 to 5.4.2, fixed 5.4.3; see https://nvd.nist.gov/vuln/detail/CVE-2025-27820 and https://github.com/advisories/GHSA-73m2-qfq3-56cx). jackson-core 2.21.5 is past CVE-2025-52999 (nesting-depth DoS, fixed 2.15.0; https://nvd.nist.gov/vuln/detail/CVE-2025-52999). log4j 2.25.5 has no open advisories. No published advisory was found for opensearch-java 2.10.0 or opensearch-rest-client 2.12.0 (searched NVD and GitHub advisories on 2026-10-04). Note, not a CVE: qqq-backend-core still drags in AWS SDK for Java v1 1.12.797, which reached end of support on 2025-12-31 (https://aws.amazon.com/blogs/developer/announcing-end-of-support-for-aws-sdk-for-java-v1-x-on-december-31-2025/); that is a qqq concern, not this repo's, and this QBit should use SDK v2 only. | `pom.xml:22` (`opensearch.version 2.10.0`), `:74-83` (httpclient5 and jackson unpinned, versions from the qqq BOM); `mvn -B dependency:tree` output | Nothing to pin. Consider moving opensearch-java to the 2.x line that matches the target cluster (the client's `opensearchVersion` is 2.12.0) when the next qqq BOM bump happens; keep httpclient5 and jackson BOM-managed as ADR-0007 already requires. | No |
+
+## 1. Auth modes: today versus 1.0
+
+| Mode | Supported today | What it needs | Notes |
+|------|-----------------|---------------|-------|
+| None (local dev, security plugin disabled) | Yes (omit username/password) | Nothing | Should stay the default only for `localhost`; see A2. |
+| HTTP basic (security plugin internal user, or Amazon OpenSearch fine-grained access control with an internal master user) | Yes | Nothing new; fix A2 (require TLS) and A3 (`${env.}` references) | Current implementation uses `BasicCredentialsProvider` scoped to the host, which is correct. Consider pre-emptive auth (`BasicScheme` in a `HttpClientContext`) to save one 401 round trip per connection; optional. |
+| AWS IAM SigV4, service `es` (Amazon OpenSearch Service domain with IAM) | No | `org.opensearch.client.transport.aws.AwsSdk2Transport` (already inside opensearch-java 2.10.0); `software.amazon.awssdk:sdk-core`, `auth`, `regions`, `http-client-spi` (already compile-scope via qqq-backend-core at 2.41.10); one `SdkHttpClient` implementation declared explicitly, `software.amazon.awssdk:apache-client` (currently only transitive runtime scope) or `url-connection-client`; `software.amazon.awssdk:sts` if assume-role is wanted (not currently on the classpath) | `new AwsSdk2Transport(httpClient, endpointHost, "es", Region.of(region), AwsSdk2TransportOptions.builder().setCredentials(provider).build())`. Default provider is `DefaultCredentialsProvider` (env, profile, container, instance role). Assume-role: `StsAssumeRoleCredentialsProvider`. Reference: https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md and https://docs.opensearch.org/latest/clients/java/ |
+| AWS IAM SigV4, service `aoss` (OpenSearch Serverless) | No | Same as above with signing service name `"aoss"` | Serverless rejects some operations this QBit uses; verify `refreshIndex()` and `deleteByQuery(... refresh=true)` against the supported-operations list before claiming support (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html). Reference for `aoss`: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html |
+| Bearer / JWT token | No | `ApacheHttpClient5TransportBuilder.setDefaultHeaders(new BasicHeader("Authorization", "Bearer " + token))` for a static token; a `Supplier<String>` plus an `HttpRequestInterceptor` for rotating tokens | The security plugin supports JWT as an HTTP authenticator (https://docs.opensearch.org/latest/security/authentication-backends/authc-index/). Static tokens fit a header field; rotation needs the SPI below. |
+| API key header | No | Same mechanism as bearer, custom header name and value | Not a native OpenSearch security-plugin concept (OpenSearch has no Elasticsearch-style API keys); treat as "arbitrary static header", covered by the SPI. |
+| Client certificate mTLS | No | `SSLContextBuilder.loadKeyMaterial(keyStore, keyPassword)` plus `loadTrustMaterial`, passed through `ClientTlsStrategyBuilder` to a `PoolingAsyncClientConnectionManager` set on the `HttpAsyncClientBuilder` | Security plugin side: `plugins.security.ssl.http.clientauth_mode: OPTIONAL` or `REQUIRE` (https://docs.opensearch.org/latest/security/authentication-backends/client-auth/). Fits naturally under the `tls` block (`keyStorePath`, `keyStorePassword`). |
+
+Recommended minimum for 1.0: `authMode` enum with three values, NONE, BASIC, AWS_SIGV4 (with `awsRegion`, `awsServiceName` defaulting to `es`, optional `awsAssumeRoleArn`), plus one escape hatch, `transportCustomizer` as a `QCodeReference` to an interface such as
+
+    public interface OpenSearchTransportCustomizer
+    {
+       default ApacheHttpClient5TransportBuilder customizeTransport(ApacheHttpClient5TransportBuilder b) { return b; }
+       default HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder b) { return b; }
+    }
+
+That covers local, self-managed with the security plugin, and both AWS flavours natively, and lets bearer, API key and mTLS be added by hosts without a fork. Do not add BEARER, API_KEY or MTLS enum values in 1.0; they would each need a token-refresh or key-store story that the SPI defers.
+
+The AWS path is a different `OpenSearchTransport` implementation, so `QuickSearchOpenSearchClient` should build an `OpenSearchTransport` from a small factory keyed on `authMode` and hold the interface type rather than `ApacheHttpClient5Transport` (`opensearch/QuickSearchOpenSearchClient.java:77`).
+
+Least privilege for whatever identity is used: the QBit needs, on its one index, `indices:admin/create`, `indices:admin/exists` (or `indices:admin/get`), `indices:admin/refresh`, `indices:data/write/index`, `indices:data/write/bulk*`, `indices:data/write/delete`, `indices:data/write/delete/byquery`, and `indices:data/read/search`. It needs no cluster-level privileges. Document a role template; the index holds copies of field values from every indexed table, so the index itself should be treated as sensitive data.
+
+## 2. TLS
+
+- Can trust be configured: no. The transport is built with `SSLContext.getDefault()` and the httpclient5 default hostname verifier (confirmed from the opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` bytecode). Certificates are verified against the JVM trust store by default, hostname verification is on, and there is no code path to weaken or extend either.
+- Custom CA or trust store: only through `-Djavax.net.ssl.trustStore` system properties on the host JVM, which is global and awkward in containers. Add the `tls` block from the sketch.
+- Insecure flag for dev: none exists. This is good for safety but in practice developers running OpenSearch with its default self-signed demo certs will reach for a global workaround. Provide `tls.insecureSkipVerify` with guard rails (refuse for non-loopback hosts unless an explicit second flag is set; WARN at startup) rather than leaving it to ad hoc JVM flags.
+- `useSsl` default false: acceptable only for a local, no-auth cluster. It is not acceptable combined with credentials (A2). For 1.0 prefer a single `url` (or `scheme`) field defaulting to `https`, and make `validate()` reject BASIC or AWS_SIGV4 over `http` unless `allowPlaintextCredentials` is set. AWS endpoints are HTTPS only, so AWS_SIGV4 should force `https`.
+
+## 3. Secrets handling
+
+- Storage: `opensearchPassword` is a plain `String` on a `Serializable` config (`QuickSearchQBitConfig.java:53`). `QuickSearchQBitProducer.produce` stores the same object in the static `QuickSearchQBitContext` (`QuickSearchQBitProducer.java:200`), and the downstream qqq-all host attaches it to `QBitMetaData` and `QInstance.addQBit` (`DemoQBits.java:162-163`). The password therefore lives in three places for the JVM lifetime.
+- Exposure through QQQ metadata endpoints: not today. `MetaDataAction` builds `MetaDataOutput` from `QFrontendTableMetaData`, `QFrontendProcessMetaData`, `QFrontendAppMetaData`, widgets, `environmentValues`, and supplemental metadata; it never touches `QInstance.getQBits()` or backend metadata, so `/metaData` does not carry QBit configs. Two caveats: `QInstanceAdapter.qInstanceToJson` (qqq `adapters/QInstanceAdapter.java:46,57`) serializes the whole `QInstance` including `qBits`, and Jackson will emit `opensearchPassword` through its public getter; and `MetaDataOutput.environmentValues` is sent to the frontend and is populated from every `QQQ_ENV_*` environment variable (qqq `QInstance.java:190-216`), so an operator must never name an OpenSearch secret `QQQ_ENV_...`. Document that.
+- Logging: no `LOG` call prints host, port, username or password (full list in A13). `validate()` and `produce()` print only field names, `backendName`, class names and table names. There is no `toString()` on the config, so an accidental `LOG.info("...", config)` would print `QuickSearchQBitConfig@hash` today; add an explicit redacting `toString()` so that stays true if someone adds one.
+- Recommendation (ordered): (1) interpret `opensearchUsername`, `opensearchPassword` and any future token or store-password fields with `QMetaDataVariableInterpreter.interpret()` at client-construction time only, so hosts write `"${env.OPENSEARCH_PASSWORD}"` and the metadata object never holds the value; this is the established QQQ pattern (`RDBMSBackendMetaData.enrich()`, `MongoDBBackendMetaData.enrich()`, `S3BackendMetaData.enrich()`); `${prop.X}` and `${literal.X}` come for free, and `??` null-coalescing lets `"${env.X}??${prop.X}"` work. (2) `@JsonIgnore` on the secret getters and a redacting `toString()`. (3) For 1.0 documentation, state that the raw password form is deprecated and will be rejected (or at least warned about) in a later release.
+
+## 4. QQQ-side authorization
+
+### A1 fix (search)
+
+Exact leak: `QuickSearchAction.execute` reads every discovered table from `QuickSearchQBitContext` and returns `tableName`, `recordId`, `recordLabel` and `highlightSnippet` for every hit (`actions/QuickSearchAction.java:81-121`). The snippet is cut from `searchableText`, which `IndexingUtils.buildSearchableText` builds from all configured fields (`processes/IndexingUtils.java:83,136`). So a user with read permission on `order` but not `customer` who searches "smith" receives customer labels and fragments of customer field values (emails, phone numbers, whatever was configured). The caller-supplied `tableName` filter is passed straight into the query (`QuickSearchOpenSearchClient.java:481-487`) so it can also be used to target one table deliberately.
+
+Minimal fix, in order:
+
+1. Make `QuickSearchInput extends AbstractActionInput` (`actions/QuickSearchInput.java:25` is a plain class today). `PermissionsHelper.hasTablePermission(AbstractActionInput, String, TablePermissionSubType)` needs it, and it gives the action the session via `QContext`.
+2. Compute `allowedTables = discoveredTables.filter(t -> PermissionsHelper.hasTablePermission(input, t.getTableName(), TablePermissionSubType.READ))`. QQQ 4.1's own `RecordSearchAction` does exactly this check (qqq `actions/tables/RecordSearchAction.java:110`); mirror it, including the `Capability.TABLE_QUERY` check.
+3. If `input.tableName` is set and not in `allowedTables`, return the empty output (do not throw; do not reveal that the table exists). If `allowedTables` is empty, return empty without calling OpenSearch.
+4. Change `QuickSearchOpenSearchClient.search` to take `Collection<String> allowedTables` and always add `filter: { terms: { sourceTable: allowedTables } }` to the bool query, in addition to the optional single-table term. Only build boost `should` clauses for allowed tables.
+5. Optional but recommended (initiative D3 already decided yes): group hits by `sourceTable`, run one `QueryAction` per table with `primaryKeyField IN (ids)` and `shouldOmitHiddenFields`, keep only hits whose id came back. `QueryAction` itself does not check table permission (it only calls `ActionHelper.validateSession`, qqq `QueryAction.java:127`), so step 2 is still required; what the re-read adds is record security locks (`RecordSecurityLock` handling in `JoinsContext`), which is why `RecordSearchAction` goes through `QueryAction` too (qqq `RecordSearchAction.java:66,177`). Over-fetch (for example `limit * 2`, capped) and document that `totalHits` is an upper bound.
+6. Tests: a session without READ on table B searching a term that matches rows in A and B gets only A hits; a session with a record security lock sees only its own rows after step 5; a requested `tableName` the session cannot read yields empty.
+
+### Processes, tables and app
+
+Today none of the produced metadata declares permission rules (A6). For 1.0 the producer should set, by default and overridable through config:
+
+- `quickSearchBasepullIndex`, `quickSearchFullReindex`, `quickSearchReconcileIndex`: `QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName("quickSearchAdmin")`. Full reindex deletes the table's documents before rebuilding (search blackout), so it must not be runnable by arbitrary users.
+- `quickSearchIndex`, `quickSearchIndexRun`: `READ_WRITE_PERMISSIONS` (operators may need to flip `enabled` or edit intervals; everyone else should not see run error messages, which can include backend exception text).
+- `quickSearchAdmin` app: `HAS_ACCESS_PERMISSION` with the same base name so a single `quickSearchAdmin.hasAccess` grant lights up the whole admin surface.
+- Scheduled runs: when D4 lands (`schedulerName` required), basepull runs under the scheduler's system session, not a user's, so the process permission only gates manual triggering. Say so in the README.
+
+The customizers run inside the acting user's transaction and session; they index what that user inserted or updated and delete what that user deleted, so they need no extra checks. `QuickSearchPostUpdateCustomizer` re-reads via `QueryAction` in the same session (`customizers/QuickSearchPostUpdateCustomizer.java:154`), which is correct.
+
+## 5. Transport hygiene
+
+| Aspect | Current value | Source | Assessment |
+|--------|---------------|--------|------------|
+| Connect timeout | 1000 ms | opensearch-java builder default | Too tight for AWS or cross-AZ; make configurable, default 5 s |
+| Response (socket) timeout | 30 000 ms | builder default | Bounded, good; configurable, default 60 s for bulk and delete-by-query |
+| Connection-request (pool wait) timeout | httpclient5 `RequestConfig.DEFAULT` (3 min) | not set | Bounded; fine |
+| Max connections | 10 per route / 30 total | builder default | Fine for one host; expose `maxConnections` |
+| HTTP-level retries | disabled (`disableAutomaticRetries()`) | builder | Good; indexing steps handle failures per item |
+| Node failover | single `HttpHost`, dead-host denylist with backoff | `QuickSearchOpenSearchClient.java:99`; transport `DeadHostState` | OK for one endpoint; allow a list of hosts for self-managed clusters |
+| Compression | client-side decompression disabled, transport handles it | `QuickSearchOpenSearchClient.java:122-131` | Correct workaround for httpclient5 5.6 |
+| User-Agent | opensearch-java default | not set | Add `qbit-quick-search/<version>` |
+| Close on shutdown | never called in production | A8 | Fix |
+| Keep-alive / idle eviction | httpclient5 defaults | not set | Acceptable |
+
+## 6. Supply chain
+
+See A14. Resolved compile/runtime set has no concrete published CVE at the audit date. The one cited fix the stack already has is CVE-2025-27820 in httpclient5 5.4.0 to 5.4.2 (resolved 5.6.3 is clear). Keep httpclient5 and jackson BOM-managed (ADR-0007). `h2 2.2.220`, Testcontainers 1.21.4, JUnit, AssertJ and Mockito are test scope only.
+
+## Recommended 1.0 auth configuration
+
+Field names on `QuickSearchQBitConfig` (fluent `withX` setters as today). All `String` values that can hold secrets or paths are run through `QMetaDataVariableInterpreter.interpret()` inside the client factory, never at `validate()` or `produce()` time, so the config object stores references.
+
+    // Connection
+    opensearchUrl            String   "https://search.example.com:443"  (preferred; replaces host/port/useSsl)
+    opensearchHost/Port      String/Integer  (kept for compatibility; deprecated)
+    useSsl                   Boolean  (kept; ignored when opensearchUrl is set)
+    opensearchHosts          List<String>  optional, same scheme, for self-managed multi-node clusters
+    opensearchIndexName      String   (unchanged)
+
+    // Authentication
+    authMode                 enum OpenSearchAuthMode { NONE, BASIC, AWS_SIGV4 }   default NONE
+    opensearchUsername       String   BASIC only; "${env.OPENSEARCH_USER}"
+    opensearchPassword       String   BASIC only; "${env.OPENSEARCH_PASSWORD}"; @JsonIgnore getter; redacted in toString()
+    awsRegion                String   AWS_SIGV4; "${env.AWS_REGION}" if omitted
+    awsServiceName           String   AWS_SIGV4; "es" (default) or "aoss"
+    awsAssumeRoleArn         String   AWS_SIGV4, optional; uses StsAssumeRoleCredentialsProvider when set
+    allowPlaintextCredentials Boolean default false; validate() errors on BASIC over http unless true
+
+    // TLS
+    tls.caCertificatePath    String   PEM bundle to trust in addition to or instead of the JVM store
+    tls.trustStorePath       String   JKS/PKCS12 alternative to caCertificatePath
+    tls.trustStorePassword   String   reference-interpreted
+    tls.keyStorePath         String   client certificate for mTLS (optional)
+    tls.keyStorePassword     String   reference-interpreted
+    tls.hostnameVerification Boolean  default true
+    tls.insecureSkipVerify   Boolean  default false; validate() rejects for non-loopback hosts unless allowInsecureInProduction is true; WARN at startup
+
+    // Transport
+    connectTimeoutMillis     Integer  default 5000
+    responseTimeoutMillis    Integer  default 60000
+    maxConnections           Integer  default 30
+    transportCustomizer      QCodeReference<OpenSearchTransportCustomizer>  escape hatch for bearer, API key, custom interceptors
+
+    // Authorization inside QQQ
+    searchRequiresTableReadPermission  Boolean default true (no switch to turn it off in 1.0; listed so the behaviour is documented)
+    applyRecordSecurityLocks           Boolean default true; when true, hits are re-read through QueryAction by primary key
+    adminPermissionRules     QPermissionRules  default HAS_ACCESS_PERMISSION, base name "quickSearchAdmin", applied to the app and the three processes
+    tablePermissionRules     QPermissionRules  default READ_WRITE_PERMISSIONS, applied to quickSearchIndex and quickSearchIndexRun
+
+`validate()` additions: `authMode` consistency (BASIC needs both username and password; AWS_SIGV4 needs a resolvable region and forbids username/password; NONE with username/password set is an error, not a silent fallback), TLS consistency (trust store needs password or must be PKCS12 without one; `insecureSkipVerify` rules above), and the plaintext-credentials rule. Validation messages must never echo values, only field names, which is already the convention in the file.
+
+Deprecation path: keep `withOpensearchHost/Port/UseSsl/Username/Password` working for 0.x hosts; infer `authMode = BASIC` when username is set and `authMode` is null, and log a one-line WARN recommending `${env.}` references when a password value does not start with `${`.

docs/audits/2026-10-04/04-feature-completeness.md

@@ -0,0 +1,213 @@
+# Audit 04: Feature completeness for Quick Search 1.0
+
+Scope: `qbit-quick-search` at `feature/GH-6-repin-and-drift-fix` (content-identical to `develop` 000e5bb), 29 main classes, 29 test classes (233 unit tests, 12 integration tests). Compared against the draft PRD `docs/initiatives/quick-search-1.0.md`, qqq-all `DemoQBits.java`, Voyage, and QQQ `v4.1.0-RC.1` `RecordSearchAction`.
+
+Paths below are relative to `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1/src/main/java/com/kingsrook/qbits/quicksearch/` unless stated. Core paths are relative to `/Users/james.maes/Git.Local/QRunIO/qqq` at tag `v4.1.0-RC.1`.
+
+Verdict in one line: the QBit has a working happy path (index, bulk, search, reconcile) but is missing most of what makes a search feature trustworthy in a multi-table, multi-user, long-running host: no scheduling, no permissions, no timeouts, no mapping/config drift handling, no admin UI affordances, a single global static context, and a search query shape that was never tuned. The PRD covers roughly half of the gaps found here.
+
+## 1. Area-by-area table
+
+| # | Area | Status | Evidence | What 1.0 needs |
+|---|---|---|---|---|
+| 1a | Real-time insert/update/delete customizers | Partial | `customizers/QuickSearchPostInsertCustomizer.java:66-105`, `...PostUpdateCustomizer.java:81-155` (re-reads full record in the update's transaction), `...PostDeleteCustomizer.java:67-118`. All three swallow every exception at WARN (`:99-102`, `:117-120`, `:112-115`) and return records unchanged. Registration throws when the slot is occupied (`QuickSearchQBitProducer.java:571-579`; core `QTableMetaData.java:651-653` throws `IllegalArgumentException`). | `MultiCustomizer` composition (PRD E1). Decide and document the failure contract (swallow vs. fail write), plus a failed-event record so swallowed failures are not lost (see 4). Honour `quickSearchIndex.enabled` (see 3). |
+| 1b | Bulk inserts with many records | Partial | Insert customizer builds one `IndexEvent` per record and the publisher bulk-indexes in `bulkBatchSize` chunks (`publisher/SynchronousIndexEventPublisher.java:93-127`, `opensearch/QuickSearchOpenSearchClient.java:252-272`). Whole batch runs inside the host's insert call; no size-based (bytes) batching, no async. A 50k-row bulk load blocks the insert for the full OpenSearch round trips. | Document that bulk loads should disable real-time indexing or rely on basepull; add byte-size cap per bulk request (OpenSearch `http.max_content_length` default 100 MB). |
+| 1c | Null/blank searchable fields | Complete | `processes/IndexingUtils.java:96-99` skips null/blank in text; `:150-156` omits null from `fieldValues`; `:138-141` returns null doc when PK null. Unit tested (`IndexingUtilsTest`). | Nothing. |
+| 1d | Very large text fields | Missing | No truncation or length cap anywhere (`IndexingUtils.buildSearchableText`); `recordLabel`/`searchableText` mapped as plain `text` (`QuickSearchOpenSearchClient.java:189-190`). A CLOB field is sent whole; edge-ngram 2..20 on a 1 MB field explodes token count. Highlight returns default fragments only. | Per-field `maxLength` in `SearchableFieldConfig`/`@QuickSearchField` with a sane default (e.g. 10k chars), documented. |
+| 1e | Tables without a `modifyDate`-style field | Missing | Default `basepullTimestampField = "modifyDate"` (`annotations/QuickSearchable.java:54`, `SearchableTableConfig.java:34`). `QuickSearchQBitConfig.validate` (`:73-157`) never checks that the timestamp field or the searchable fields exist on the table. Basepull then fails per table at query time, caught at `BasepullIndexStep.java:128-132` and logged. qqq-all's `customer` table config does not declare one (`qqq-all DemoQBits.java:159-160`). | Validate field existence at produce (fail fast); allow `basepullTimestampField = null` meaning "no incremental basepull, reconcile only". |
+| 1f | Non-integer primary keys | Partial | `recordId` is a String everywhere (`OpenSearchDocument.java:35`, `QuickSearchResult.java:28`); keyset paging uses `GREATER_THAN` on the PK (`ReconcileIndexStep.java:222-238`), which works for strings/UUIDs only if the backend sorts them consistently. Untested. | Integration test with a STRING PK table; document ordering assumption. |
+| 1g | Composite keys | Missing | Single `primaryKeyField` model (`QuickSearchableTableConfig.java:25`). | Out of scope for 1.0; state it in README. |
+| 1h | Table removed from config (orphaned documents) | Missing | Reconcile iterates `getDiscoveredTables()` only (`ReconcileIndexStep.java:98`); documents for a table no longer configured are never deleted and are still returned by an unfiltered search (`QuickSearchOpenSearchClient.java:481-487` filter only when `tableName` given). `quickSearchIndex` rows for removed tables stay `ACTIVE`. | Reconcile (or a startup check) should delete-by-query every `sourceTable` not in the discovered set and mark the index row `ORPHANED`/disabled. |
+| 1i | Table renamed / prefix changed | Missing | `sourceTable` is the QQQ table name; `tableNamePrefix` applies only to the two operational tables and processes (`QuickSearchQBitConfig.applyPrefix`). Renaming a source table leaves old documents behind (same as 1h). Changing `opensearchIndexName` silently starts an empty index (`ensureIndexExists` creates it, `:173-192`) with no reindex trigger. | Same mechanism as 1h plus an "index is empty but source has rows" warning in the admin table. |
+| 1j | Config changes to fields/weights/analyzer | Missing | `searchableFieldsJson` is written once when the index row is created (`AbstractIndexingStep.java:167-175`) and never compared again. `ensureIndexExists` returns early if the index exists (`QuickSearchOpenSearchClient.java:167-171`), so a pre-existing index with a different analyzer or mapping is used as-is. Weights are query-time boosts on `fieldValues.*` (`:526-565`), so weight changes need no reindex, but field additions do. | Compare current config to `searchableFieldsJson` at produce/basepull and set `status = NEEDS_REINDEX`; verify the live mapping's analyzer settings and refuse to start (or warn loudly) on mismatch. |
+| 1k | Index name collisions on a shared cluster | Missing | One flat index per config, name is whatever the host passes (`QuickSearchQBitConfig.java:51`). Two apps with the same default would share documents; there is no `appId`/environment field in the document or filter. `fieldValues` is a single dynamic object shared by every table (`:192`): two tables with the same field name but different types (e.g. `code` INTEGER in one, STRING in another) produce `mapper_parsing_exception` bulk-item failures for the second table. | Document required index naming per environment; namespace `fieldValues` per table (`fieldValues.<table>.<field>`) or map all values as `keyword`/`text` strings to avoid type conflicts. |
+| 1l | Basepull catch-up | Partial | Works per enabled row (`BasepullIndexStep.java:74-134`). Defects: never scheduled (no `withSchedule` anywhere; `QuickSearchQBitProducer.java:510-518`); `lastBasepullTime` is set to `Instant.now()` after the run (`:268-279`), not the run's start, so rows modified during the run but after their page was read are skipped forever; `GREATER_THAN` on a seconds-precision DB column vs. millisecond `Instant` misses same-second rows; limit/skip paging without order-by (`AbstractIndexingStep.java:358-359`) is nondeterministic under concurrent writes. | PRD E2 covers scheduling and ordered paging. Add: capture `runStart` before querying and use it (minus a small overlap) as the new watermark; use `GREATER_THAN_OR_EQUALS` with the overlap. |
+| 1m | Full reindex | Partial | Wipes the table's documents first (`FullReindexStep.java:152`) causing a search blackout; limit/skip paging without order-by (`:162`); does not honour `enabled`; `recordCount` written only on success (`:191-204`). | PRD E2 (alias swap) covers the blackout. |
+| 1n | Reconcile | Complete for its design | Keyset paging (`ReconcileIndexStep.java:222-238`), refresh then delete-stale by `indexedAt` (`:191-193`), aborts stale-delete on any failure (`:195-198`). Relies on cross-node clock sync (class Javadoc `:62-63`). Not scheduled. Does not handle orphaned tables (1h). | Schedule it (PRD E2); add orphan-table sweep. |
+| 2a | Term normalisation and short/empty terms | Partial | `IndexingUtils.normalizeSearchText` lowercases and collapses whitespace (`:58-66`); blank returns empty output (`QuickSearchAction.java:67-74`). No minimum length: a 1-char term yields zero edge-ngram tokens (min_gram 2) so it silently returns nothing; no maximum length (core caps at 100). | Enforce min 2 / max 100 characters with a clear message, matching core. |
+| 2b | Multi-word queries | Missing (untuned) | `multi_match` on `searchableText` with no `operator`, no `minimum_should_match`, no `type` (`QuickSearchOpenSearchClient.java:472-475`). Default is OR, so "blue widget" matches every document containing "bl", "blu", "wi", ... and ranks by ngram overlap. | Set `operator = AND` (or `minimum_should_match = "2<75%"`) and type `best_fields` or `cross_fields`; cover with an integration test using two-word terms. |
+| 2c | Phrase matching | Missing | No `match_phrase` clause; quoted input is passed through as-is. | Optional for 1.0; document absence. |
+| 2d | Prefix/typeahead behaviour | Partial (defective) | `searchableText` uses the edge-ngram analyzer for both indexing and search; no `search_analyzer` is set (`:190`). The query term is therefore also edge-ngrammed: "order" becomes or/ord/orde/order, so "or" matches as strongly as the full word and relevance degrades with term length. The boost should-clauses hit `fieldValues.*`, which are dynamically mapped with the standard analyzer, so field weights only apply to whole-token matches, never to prefix matches. | Set `search_analyzer: standard` (lowercase only) on `searchableText`; either map `fieldValues.*` with the same edge-ngram analyzer via a dynamic template or drop field-level boosting into per-field sub-fields of `searchableText`. |
+| 2e | Fuzziness | Missing | None configured. | 1.x (PRD explicitly out of scope). |
+| 2f | Per-table result limits and grouping | Missing | Single flat ranked list; `tableName` filter accepts one table (`QuickSearchInput.java:29`). One hot table swamps the others. Core has `limitPerTable` (default 5, max 25). | Add `tableNames` (list) and `limitPerTable` (use an OpenSearch `terms` aggregation with `top_hits`, or `collapse`); needed to map onto core `RecordSearchInput`. |
+| 2g | Result fields | Partial | `QuickSearchResult` has `tableName, recordId(String), recordLabel, score, highlightSnippet` (`actions/QuickSearchResult.java:27-31`). No `tableLabel`, no typed `recordId`, no route/URL hint, no display fields, no `relation` on total. Core `RecordSearchResult` has `tableLabel` and `Serializable recordId`. | Add `tableLabel` (from `QInstance.getTable(name).getLabel()`), typed `recordId`, and optionally the matched field name. |
+| 2h | Max page depth | Missing | `limit` has no upper bound (`QuickSearchAction.java:78`); `from + size > 10000` hits `index.max_result_window` and surfaces as a raw QException. | Cap `limit` (e.g. 100) and `offset + limit` (e.g. 10000); return a clean error. |
+| 2i | Highlight quality | Partial | Highlight only on `searchableText` with defaults (`QuickSearchOpenSearchClient.java:491-492`); fragments joined with "..." (`QuickSearchAction.java:104-109`). Snippets include the `fieldName: ` prefix when `includeLabel` is set (`IndexingUtils.java:106-108`), exposing internal field names; no `pre/post_tags` control; nothing stops a snippet from a field the user may not see (PRD E3 mentions this). | Configurable tags and fragment size; strip label prefixes or use the QQQ field label. |
+| 2j | Sorting options | Missing | Score only. | Fine for 1.0; document. |
+| 2k | Total hit accuracy | Missing | `track_total_hits` not set, so `totalHits` caps at 10,000 and `hasMore` is computed from the capped value (`QuickSearchAction.java:123-124`); `relation` is ignored. | Set `track_total_hits` to a bounded number and expose `totalHitsIsLowerBound`. |
+| 2l | Permissions / session | Missing | No `PermissionsHelper`, no `QContext` use anywhere in `actions/` or `opensearch/`. Core `RecordSearchAction.java` checks `TablePermissionSubType.READ` and `Capability.TABLE_QUERY` per table and personalises fields. | PRD E3. |
+| 3a | `quickSearchIndex` fields | Partial | `id, tableName, enabled, basepullIntervalMinutes, basepullTimestampField, searchableFieldsJson, lastBasepullTime, lastFullReindexTime, recordCount, status, createDate, modifyDate` (`QuickSearchQBitProducer.java:456-471`). `status` is always "ACTIVE" (`AbstractIndexingStep.java:175`) and never updated; no `lastError`, `lastRunStatus`, `documentCount` (from OpenSearch), `lastReconcileTime`, `indexName`. No PVS, no sections, no `isRequired`, no `isEditable=false` on system-managed fields, no labels beyond defaults. | Add `lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`; make tracking fields read-only; add field sections and a tableName PVS. |
+| 3b | `quickSearchIndexRun` fields | Partial | `quickSearchIndexId, runType, status, startTime, endTime, recordsProcessed, recordsIndexed, errorCount, errorMessage` (`:482-497`). `errorMessage` is a plain STRING holding only the first error (`BasepullIndexStep.java:254-257`); no per-item error table; `quickSearchIndexId` has no PVS/join so the admin UI shows an integer. | Join/PVS to `quickSearchIndex`; store error list (JSON or child table); set `errorMessage` max length. |
+| 3c | Run records for every path | Partial | Written for basepull, full reindex, reconcile (`createRunRecord`/`completeRunRecord`). Not written for real-time customizer failures (they only WARN) nor for "skipped, not due" basepull iterations. Nothing is scheduled, so there are no scheduled runs today. | A failed-event table or a `REALTIME_FAILURE` run row per failed publish (see 4c). |
+| 3d | Retention/cleanup of run history | Missing | No purge; unbounded growth once scheduled (every 5 minutes per table). | Scheduled purge (e.g. keep 30 days or N rows per index) in the same scheduled process. |
+| 3e | `enabled` honoured at runtime | Partial | Only basepull filters `enabled = true` (`BasepullIndexStep.java:89-92`). Customizers, full reindex, reconcile and search ignore it, so disabling a table in the admin UI still indexes it on every write and still returns its rows. | Customizers and processes consult the index row (cached, refreshed per run); search excludes disabled tables via a `terms` filter. |
+| 3f | Status reporting / counts | Partial | `recordCount` updated only on full reindex/reconcile success; no live document count; no health field. | See 3a. |
+| 3g | Admin widgets or dashboards | Missing | App is two tables plus three processes (`QuickSearchQBitProducer.java:247-258`); no widgets, no icon. | A small dashboard widget (per-table status, last run, failures) is a 1.x nicety; an icon and labels are 1.0 polish. |
+| 3h | Process input screens | Missing | All three processes are a single backend step with no frontend step and no input `QFieldMetaData` (`:510-549`), yet full reindex and reconcile read `tableName` from input (`FullReindexStep.java:73`, `ReconcileIndexStep.java:93`). From the admin UI there is no way to supply it, so UI-triggered runs always hit every table. README's "or through the QQQ admin UI" is therefore only half true. | Add a frontend step with an optional `tableName` field backed by a PVS of discovered tables, plus a summary screen showing counts (reconcile already emits `recordsIndexed`/`documentsRemoved`). |
+| 3i | Labels/help text | Partial | Table and process labels set; no field labels/help, no app icon/section. | Polish pass. |
+| 4a | Logging | Partial | QLogger with structured pairs; failures at WARN. Basepull logs "not due" at INFO per table per run (`BasepullIndexStep.java:118-120`), which is noisy every 5 minutes. | Downgrade to DEBUG; add one INFO summary per run. |
+| 4b | Metrics / health check | Missing | No counters, no timers, no `ping`/cluster-health call; the client constructor never connects (`QuickSearchOpenSearchClient.java:94-147`), so "client created" says nothing about reachability. | Expose a health method (indices.exists or cluster health) and surface it in the admin table or a QQQ health endpoint; count publish failures. |
+| 4c | OpenSearch down at startup vs. runtime | Partial (hazardous) | Startup: client build rarely fails; `ensureIndexExists` failure is WARN-and-continue (`QuickSearchQBitProducer.java:174-181`), publisher is still created (`:192-195`), host boots "healthy". Runtime: every host insert/update/delete calls OpenSearch synchronously; failures are swallowed (customizers) so the host write succeeds and the index silently diverges; `QuickSearchAction` throws. No timeouts are configured on the transport (no `setConnectTimeout`/`setResponseTimeout`/`RequestConfig` anywhere), so a black-holed host blocks each write for the OS connect timeout (tens of seconds to minutes) and a slow cluster blocks indefinitely. | Fail-fast at produce with opt-out (PRD E1). Add explicit connect/response timeouts in config (e.g. 2s/10s). Record failed events (see 4e). |
+| 4d | Retry/backoff | Missing | None; a transient 429/503 on bulk is a hard failure. | Bounded retry with jitter on bulk 429/5xx inside the client (sync path), documented. |
+| 4e | Dead-letter / failed-event capture | Missing | Failed real-time events vanish after a WARN. Basepull only recovers rows whose `modifyDate` moves; a failed delete is recovered only by reconcile. | A `quickSearchFailedEvent` table (table, recordId, action, error, attempts) replayed by the scheduled process; this is the cheapest correct answer without an async queue. |
+| 4f | Circuit breaking | Missing | None. | Simple breaker in `SynchronousIndexEventPublisher` (open after N consecutive failures, half-open after T seconds; skip and record events while open). With timeouts (4c) this is what keeps a down cluster from slowing every host write. |
+| 5a | Index name per environment | Partial | Host supplies the name; qqq-all reads `OPENSEARCH_INDEX` (`DemoQBits.java:155`). No validation of OpenSearch naming rules (lowercase, no spaces). | Validate name; document per-environment convention. |
+| 5b | Analyzer customisation | Missing | Hard-coded `standard` tokenizer + `lowercase` + `edge_ngram(2,20)` (`QuickSearchOpenSearchClient.java:176-185`). | Expose `minGram/maxGram` and an optional analyzer-settings hook (see 6c). |
+| 5c | Refresh policy | Missing | Bulk requests use default (near-real-time, ~1s); only tests call `refreshIndex`. Hosts that read-after-write will not see the row. | `refreshPolicy` config (none/wait_for) for the real-time path; document the default. |
+| 5d | `sourceBatchSize` vs `bulkBatchSize` | Complete but undocumented | Source page size for QQQ queries (`:1000`) vs documents per bulk request (`:500`) (`QuickSearchQBitConfig.java:58-59`). README table describes both correctly. | Note interaction (one source page = ceil(1000/500) bulk calls). |
+| 5e | Timeouts | Missing | See 4c. | Add. |
+| 5f | Multiple QBit instances / multiple indexes | Missing | Static `QuickSearchQBitContext` (`:34-37`); a second `produce()` overwrites the first; customizers and steps resolve state statically. | PRD E1 (instance-scoped holder). |
+| 5g | Excluded tables | Missing | No way to exclude, only to include; `enabledByDefault=false` still registers customizers and creates the row. | Fine if `enabled` is honoured (3e). |
+| 5h | Per-table record label strategy | Partial | `recordLabelFormat` only on config-driven tables (`SearchableTableConfig.java:36-37`); annotation path uses `QRecord.getRecordLabel()` which is null unless the query generated it (`IndexingUtils.java:142`). Customizers receive records from Insert/Update actions where `recordLabel` is typically not populated, so real-time-indexed documents often have a null `recordLabel`. Core falls back to `QValueFormatter.formatRecordLabel`. | Fall back to `QValueFormatter.formatRecordLabel(table, record)` in `buildDocument`; allow label format on the annotation too. |
+| 5i | Field type handling (dates, decimals, booleans, PVS labels) | Missing | Raw values only: `getValueString` for text (`IndexingUtils.java:94`) and raw `Object` into `fieldValues` (`:150-154`). Dates index as ISO instants, booleans as "true", PVS fields as their raw id. No display values, no PVS translation. Core sets `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` on its queries. | Index display values (translate PVS, format dates) into `searchableText` and keep raw values in `fieldValues`; batch processes already go through `QueryAction` so this is a flag; customizers need a re-read with display values (update already re-reads). |
+| 5j | Flag semantics | Partial | `enableScheduledProcesses=false` removes all three processes including the manual full reindex and reconcile (`QuickSearchQBitProducer.java:228-232`), so a host cannot have on-demand processes without the basepull. | Split into `enableBasepullProcess` and `enableMaintenanceProcesses`, or always register processes and gate only the schedule. |
+| 6a | `IndexEventPublisher` SPI | Partial | `publishIndexEvents`, `publishDeleteEvents`, `close()` (`publisher/IndexEventPublisher.java:40-60`). `IndexEvent` carries `tableName, recordId, action, QRecord` (`IndexEvent.java:30-33`): the full record snapshot, not a built document, so an async consumer can rebuild the document later via `IndexingUtils.buildDocument(record, tableConfig)` only if it has the table config (currently only through the static context). There is no event timestamp, sequence or record version, so a queue consumer cannot order an INDEX after a later DELETE. `close()` is never invoked by the producer or any shutdown hook. The publisher receives no `QSession`/tenant information. | Add `occurredAt` (Instant) and an optional `version` to `IndexEvent`; pass the table config map (or a resolver) into the publisher constructor contract; register `close()` with the host lifecycle; document ordering expectations. |
+| 6b | Document builder hook | Missing | `IndexingUtils.buildDocument` is static and called directly from publisher and steps. | `DocumentBuilder` interface with a default implementation, settable on `QuickSearchQBitConfig` (lets hosts add computed text, strip PII, index display values). |
+| 6c | Custom analyzer hook | Missing | Hard-coded in `ensureIndexExists`. | `IndexSettingsCustomizer` (consumer of `CreateIndexRequest.Builder`) or at least `minGram/maxGram/searchAnalyzer` config. |
+| 6d | Result post-processing hook | Missing | `QuickSearchAction.execute` maps hits inline (`:95-121`). | `ResultCustomizer` to add route hints or filter; also where permission filtering would plug in. |
+| 7a | Unit coverage | Partial | 233 unit tests across all packages; happy paths and null/empty guards well covered (see test list in section 4). JaCoCo `check` is configured (`pom.xml:188-212`) but with no `prepare-agent` execution, so the 0.70/0.90 gates never run (PRD notes this). | Bind `prepare-agent`; keep gates. |
+| 7b | Integration coverage | Partial | 12 tests in `OpenSearchIntegrationTest`, `IndexDriftIntegrationTest`, `ConfigDrivenIndexingIntegrationTest`. Every search uses the single whole-word term "widget" or "zzzznonexistent"; no prefix, multi-word, boost-ordering, highlight-content, pagination-beyond-first-page-correctness, basepull, or scheduling test. | See section 4 list. |
+| 7c | README promises not met | Partial | "Scheduled basepull" (README:14, 30, 287) is not scheduled. "Prefix indexed text with the field label" (README:171, 331) prefixes the field *name* (`IndexingUtils.java:108`). "Or through the QQQ admin UI" for per-table reindex (README:254, 271) has no input screen. Dependency snippet shows `0.2.0-SNAPSHOT` (README:79). qbit-crm/qbit-wms companion section (README:333-341) describes integrations that do not exist. "70%+ instruction coverage (JaCoCo enforced)" (README:362) is not enforced. "Admin Observability" (README:20) is two raw tables. | PRD E5 rewrite; add the items above to its checklist. |
+| 7d | CHANGELOG / migration notes | Missing | No `CHANGELOG.md` at repo root; no migration doc. | PRD E5. |
+| 7e | Configuration reference accuracy | Partial | README table (291-312) matches `QuickSearchQBitConfig` fields but omits the `enableScheduledProcesses` side effect (5j), the `indexEventPublisher` lifecycle, and says nothing about timeouts, refresh, or required cluster privileges. | Fix with E5. |
+| 7f | Javadoc on public API | Mostly complete | Flower-box Javadoc on nearly every public member. Gaps: `IndexingUtils.buildDocument` documents `fieldWeights` as "reserved for future use" (`:130`) although weights drive query boosts; `QuickSearchQBitProducer` public constants (`:54-57`) undocumented; no `package-info.java`; `IndexEventPublisher` does not state thread-safety or ordering guarantees. | Small pass with E5. |
+
+## 2. Comparison with QQQ 4.1 core `RecordSearchAction`
+
+Source: `qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/RecordSearchAction.java`, `model/actions/tables/search/RecordSearch{Input,Output,Result}.java`, `QTableMetaData.searchFields` (`QTableMetaData.java:100, 2013-2060`), Javalin `RecordSearchExecutor` (passes `QInputSource.USER`).
+
+Core has, Quick Search lacks:
+
+| Capability | Core | Quick Search |
+|---|---|---|
+| Table-level READ permission check | `PermissionsHelper.hasTablePermission(..., READ)` per table; unauthorised tables skipped silently (no probing) | None |
+| Capability check (`TABLE_QUERY`) | Yes | N/A (reads from index) but no equivalent gate |
+| Field personalisation; hidden/masked fields excluded | `TableMetaDataPersonalizerAction`, `getIsHidden`, `needsMasked` | None; whatever is configured is indexed and highlighted |
+| Record-level security locks | Inherited from `QueryAction` | None |
+| Per-table limit | `limitPerTable` default 5, max 25 | Global `limit`, no max |
+| Multi-table filter | `tableNames` list | Single `tableName` |
+| Term validation | Blank -> `QBadRequestException`; max 100 chars | Blank -> empty output; no max |
+| Query timeout | 30s per table | None |
+| Read-replica hint | `QueryHint.MAY_USE_READ_ONLY_BACKEND` | N/A |
+| Result shape | `tableName, tableLabel, Serializable recordId, recordLabel` | `tableName, String recordId, recordLabel, score, highlightSnippet`; no `tableLabel` |
+| Record label fallback | `QValueFormatter.formatRecordLabel` | Null when the record had no label |
+| Display values / PVS translation | `shouldGenerateDisplayValues`, `shouldTranslatePossibleValues` | Raw values |
+| Deterministic ordering | Order by PK | Score (fine) |
+| Integer term matching | Integral fields match numeric terms exactly | Numbers indexed as text tokens; "42" prefix-matches "4200" |
+| Metadata declaration | `QTableMetaData.withSearchFields(...)` is first-class metadata the Next UI can read | Separate annotation/config model; `searchFields` on the table is ignored |
+| UI surface | `POST /qqq/v1/search` and the Next UI search box | None |
+
+Quick Search has, core lacks:
+
+| Capability | Quick Search | Core |
+|---|---|---|
+| Relevance scoring and field weights | BM25 with per-field boosts | None (PK order) |
+| Prefix/typeahead | Edge-ngram (with the defect in 2d) | `CONTAINS` (LIKE `%term%`), which is a full scan on large tables |
+| Highlights | Yes | No |
+| Pagination and total | `offset/limit`, `totalHits`, `hasMore` | Top-N per table only |
+| Cross-table ranking | Single ranked list | Per-table lists concatenated |
+| Performance isolation | Query hits OpenSearch, not the OLTP backend | One LIKE query per searchable table per keystroke |
+| Operational history | Run tables | None |
+| Async-capable transport | `IndexEventPublisher` SPI | N/A |
+
+Integration consequence (PRD E4/D1): to back the Next UI, a `QuickSearchRecordSearchProvider` must accept `RecordSearchInput` (`tableNames`, `limitPerTable`, `inputSource`) and return `RecordSearchResult` (`tableLabel`, typed `recordId`). Quick Search today cannot satisfy `limitPerTable` or `tableNames` without the changes in 2f, and cannot honour the permission semantics core users will assume without E3. Also, a host that sets `QTableMetaData.searchFields` for core and `@QuickSearchable` for the QBit now maintains two field lists; 1.0 should at least default the QBit's field list from `searchFields` when present.
+
+## 3. Prioritised gap list
+
+Severity: P0 = blocks a credible 1.0; P1 = should ship in 1.0; P2 = 1.0 if cheap, else 1.x; P3 = 1.x.
+
+| ID | Sev | Gap | Evidence | Recommended scope |
+|---|---|---|---|---|
+| G01 | P0 | No connect/response timeouts on the OpenSearch transport; a down or black-holed cluster blocks every host write for the OS timeout, indefinitely for a hung response | `QuickSearchOpenSearchClient.java:94-147`; grep for timeout in `src/main` returns nothing | 1.0. Add `connectTimeoutMs`/`responseTimeoutMs` config with conservative defaults. Not in PRD. |
+| G02 | P0 | Search ignores permissions, hidden/masked fields, record locks | `actions/QuickSearchAction.java`, no `PermissionsHelper` in repo; core does all three | 1.0 (PRD E3). Extend E3 to exclude hidden/masked fields at index time, not just at search time. |
+| G03 | P0 | Basepull and reconcile never scheduled | No `withSchedule`; `QuickSearchQBitProducer.java:510-518` | 1.0 (PRD E2, D4). |
+| G04 | P0 | Edge-ngram analyzer applied at query time too (no `search_analyzer`); multi_match defaults to OR with no `minimum_should_match`; field boosts never apply to prefix matches | `QuickSearchOpenSearchClient.java:190, 472-475, 526-565` | 1.0. Set `search_analyzer`, `operator=AND`, dynamic template for `fieldValues.*`. Requires a reindex (mapping change) so it must land before 1.0's first index. Not in PRD. |
+| G05 | P0 | Startup is best-effort; host boots healthy with search broken | `QuickSearchQBitProducer.java:160-181` | 1.0 (PRD E1). |
+| G06 | P1 | Real-time failures are swallowed with no record; no retry, no breaker, no dead-letter | `customizers/*:99-120`; `SynchronousIndexEventPublisher` | 1.0: failed-event table replayed by the scheduled process, plus bounded retry on bulk 429/5xx. Breaker can be 1.x if timeouts (G01) ship. Not in PRD. |
+| G07 | P1 | `quickSearchIndex.enabled` ignored by customizers, full reindex, reconcile and search | Only `BasepullIndexStep.java:89-92` filters on it | 1.0. Not in PRD. |
+| G08 | P1 | Basepull watermark set after the run; same-second rows and rows modified mid-run are skipped forever | `BasepullIndexStep.java:268-279`, `:206-209` | 1.0 alongside E2 ordered paging; capture start time, apply overlap. Not in PRD (E2 mentions only ordering). |
+| G09 | P1 | `fieldValues` shared dynamic object across tables causes mapping type conflicts when two tables share a field name with different types | `QuickSearchOpenSearchClient.java:192`; `IndexingUtils.java:150-154` | 1.0 (mapping change, same reindex as G04): namespace per table or map as keyword. Not in PRD. |
+| G10 | P1 | Static JVM-global context; one QBit instance per JVM | `QuickSearchQBitContext.java:34-37` | 1.0 (PRD E1). |
+| G11 | P1 | Customizer slot collision fails produce | `QuickSearchQBitProducer.java:571-579`; core `QTableMetaData.java:651-653` | 1.0 (PRD E1). |
+| G12 | P1 | Full reindex blackout; unordered paging | `FullReindexStep.java:152, 162` | 1.0 (PRD E2, D5). |
+| G13 | P1 | No validation that searchable fields and the basepull timestamp field exist on the table; silent PK fallback to "id" for tables missing from the QInstance | `QuickSearchQBitConfig.java:73-157`; `QuickSearchQBitProducer.java:349, 418` | 1.0 (fold into E1 fail-fast validation). Partly in PRD ("fail-fast config validation") but field existence is not named. |
+| G14 | P1 | Orphaned documents when a table is removed from config or renamed; reconcile only sweeps configured tables | `ReconcileIndexStep.java:98`; `QuickSearchOpenSearchClient.java:481-487` | 1.0 (small addition to reconcile). Not in PRD. |
+| G15 | P1 | No config-drift detection: `searchableFieldsJson` never compared; existing index mapping/analyzer never verified | `AbstractIndexingStep.java:167-175`; `QuickSearchOpenSearchClient.java:167-171` | 1.0: compare and set `NEEDS_REINDEX`; verify analyzer on existing index and warn. Not in PRD. |
+| G16 | P1 | Result model lacks `tableLabel`, typed `recordId`, `tableNames`/`limitPerTable` input; cannot map onto core `RecordSearchResult` cleanly | `actions/QuickSearchInput.java:28-31`, `QuickSearchResult.java:27-31` | 1.0 (needed for PRD E4 even if the SPI slips). PRD E4 names the mapping but not the input-side gaps. |
+| G17 | P1 | Processes have no input screen; `tableName` cannot be supplied from the admin UI | `QuickSearchQBitProducer.java:510-549`; `FullReindexStep.java:73` | 1.0: frontend step with optional tableName PVS. Not in PRD. |
+| G18 | P1 | `totalHits` capped at 10,000 silently; `limit`/`offset` unbounded | `QuickSearchAction.java:78-79, 123-124` | 1.0: bound limit/offset, set `track_total_hits`, expose lower-bound flag. Not in PRD. |
+| G19 | P1 | Raw values indexed; PVS ids, booleans, dates not searchable by their display form; `recordLabel` often null on real-time path | `IndexingUtils.java:94, 142, 150-154` | 1.0: index display values, fallback to `QValueFormatter.formatRecordLabel`. Not in PRD. |
+| G20 | P1 | `enableScheduledProcesses=false` also removes manual full reindex and reconcile | `QuickSearchQBitProducer.java:228-232` | 1.0: split the flag (backwards compatible: keep old name as alias). Not in PRD. |
+| G21 | P2 | Run history grows unbounded once scheduled; no purge | No purge code | 1.0 if trivial (one delete-by-filter in the scheduled step), else 1.1. Not in PRD. |
+| G22 | P2 | `quickSearchIndex` lacks `lastRunStatus`, `lastError`, `documentCount`, `lastReconcileTime`; `status` never changes; no PVS/joins/sections | `QuickSearchQBitProducer.java:456-497`; `AbstractIndexingStep.java:175` | 1.0 polish with E5; schema additions are cheap before 1.0 and painful after. Not in PRD. |
+| G23 | P2 | Health check and metrics absent | No ping/health call | 1.0: a health method surfaced in the admin table; metrics 1.x. Not in PRD. |
+| G24 | P2 | No term min/max length; 1-char term returns nothing silently | `QuickSearchAction.java:67-74` | 1.0 (trivial, align with core 100-char cap). Not in PRD. |
+| G25 | P2 | Large text fields unbounded | `IndexingUtils.buildSearchableText` | 1.0: per-field `maxLength` default. Not in PRD. |
+| G26 | P2 | `IndexEvent` has no timestamp/version; publisher `close()` never called; publisher cannot resolve table configs without the static context | `publisher/IndexEvent.java:30-33`; `IndexEventPublisher.java:60` | 1.0 if the SPI is to be stable at 1.0 (PRD says "the SPI stays"): add `occurredAt`, pass config resolver, wire `close()`. Otherwise mark SPI experimental in 1.0. Not in PRD. |
+| G27 | P2 | No document-builder, analyzer or result hooks | `IndexingUtils` static; `ensureIndexExists` hard-coded | 1.1 unless needed for E3 (a result hook is the natural place for permission post-filtering). Not in PRD. |
+| G28 | P2 | Refresh policy not configurable; read-after-write misses | Bulk requests default | 1.1; document the 1s default in 1.0. Not in PRD. |
+| G29 | P2 | Highlight snippets expose field names via `includeLabel` prefix; README says "label" | `IndexingUtils.java:106-108` | 1.0 docs fix at least; use QQQ field label in 1.0 if cheap. |
+| G30 | P3 | Fuzziness, phrase queries, synonyms, language analyzers, sort options | Not present | 1.x (PRD out of scope). |
+| G31 | P3 | Composite keys, multi-index layouts | Not present | 1.x (PRD out of scope). |
+| G32 | P1 | Tests: JaCoCo gate inert; integration suite exercises one whole-word term only; no basepull, scheduling, permission, typeahead, multi-word, boost-order, non-integer PK, OpenSearch-down, mapping-conflict tests | `pom.xml:188-212`; test list in section 4 | 1.0 (PRD E1 covers the agent; add the concrete tests below). |
+| G33 | P1 | Docs: README promises (scheduled basepull, admin UI per-table reindex, coverage enforced, qbit-crm/wms integrations, field "label"), no CHANGELOG or migration notes | `README.md:14, 30, 79, 171, 254, 271, 287, 333-341, 362` | 1.0 (PRD E5). |
+
+## 4. Test inventory and concrete untested behaviours
+
+Unit (233): `QuickSearchQBitConfigTest` (34), `QuickSearchQBitProducerTest` (23), `IndexingUtilsTest` (19), `QuickSearchActionTest` (16), `QuickSearchOpenSearchClientTest` (13, constructor/unavailable-server only), `SynchronousIndexEventPublisherTest` (12), `BasepullIndexStepTest` (9), `QuickSearchPostUpdateCustomizerTest` (9), `QuickSearchQBitContextTest` (9), `FullReindexStepTest` (8), `QuickSearchPostDeleteCustomizerTest` (8), `BulkIndexResultTest` (8), `QuickSearchableTableConfigTest` (8), `QuickSearchDTOsTest` (7), `QuickSearchAnnotationsTest` (7), `ReconcileIndexStepTest` (6), `QuickSearchPostInsertCustomizerTest` (6), `SearchableFieldConfigTest` (6), `SearchableTableConfigTest` (6), `IndexEventTest` (4), `OpenSearchDocumentTest` (4), `QuickSearchIndexTest` (4), `QuickSearchIndexRunTest` (4), `RequiresDockerConditionTest` (3).
+
+Integration (12, Testcontainers OpenSearch 2.11): `OpenSearchIntegrationTest` (index created, full reindex, search returns, pagination, no results), `ConfigDrivenIndexingIntegrationTest` (discovery, full reindex, label format, no results), `IndexDriftIntegrationTest` (partial update keeps fields, delete with missing doc, reconcile repairs orphan/missing/stale).
+
+Concretely untested:
+
+1. Query shape: no test asserts the OpenSearch request (operator, boosts, highlight, from/size). `QuickSearchOpenSearchClient.search` is only exercised end-to-end with "widget".
+2. Typeahead: README's "ord finds order" is never tested; nor that "or" does not outrank "order".
+3. Multi-word terms and `minimum_should_match` behaviour.
+4. Field weight ordering (a weight-3 field match ranks above a weight-1 match).
+5. Highlight fragment content against a real cluster.
+6. Basepull against a real cluster; watermark semantics; timestamp filter with seconds-precision columns; rows modified mid-run.
+7. Any scheduling (nothing to test yet).
+8. Permission denial (nothing to test yet).
+9. Non-integer and string primary keys through reconcile keyset paging.
+10. OpenSearch down at runtime during a host insert (only a mocked publisher-throws test exists); produce with an unreachable host beyond `ensureIndexExists` failing.
+11. Pre-existing index with a different mapping/analyzer.
+12. Two tables sharing a field name with different types (mapping conflict).
+13. `totalHits` above 10,000; `offset + limit` above `max_result_window`.
+14. `enableScheduledProcesses=false` behaviour; `tableNamePrefix` applied to processes/app end to end.
+15. Duplicate `@QuickSearchable(tableName)` across two entity classes (only annotation-vs-config duplicates are checked, `QuickSearchQBitProducer.java:144`).
+16. Large text fields; `includeLabel` text appearing in snippets.
+17. Publisher `close()` lifecycle.
+18. `enabled=false` row: customizers/search still index and return (would currently fail if written as a negative test, which is the point).
+
+## 5. Items the draft PRD does not mention
+
+The PRD's five epics cover: 4.1 contract and fail-fast (E1), scheduling, ordered paging, alias-swap reindex, per-item run errors (E2), permissions (E3), core SPI and result mapping (E4), docs and release (E5). It does not mention:
+
+1. Transport timeouts (G01). The single largest operational risk; without it, "OpenSearch down" slows every host write.
+2. Search query quality: `search_analyzer`, `operator`/`minimum_should_match`, boosts not applying to prefixes (G04). This changes the mapping and should be bundled with the alias-swap work so 1.0 starts with the right index.
+3. `fieldValues` cross-table mapping conflicts (G09). Same reindex.
+4. Honouring `enabled` outside basepull (G07).
+5. Basepull watermark correctness (G08). E2 says "order-by primary key" but not the start-time watermark or overlap.
+6. Failed-event capture, retry, circuit breaker (G06). E2's "run records capture per-item errors" covers batch runs only; real-time failures still vanish.
+7. Orphaned documents for removed/renamed tables (G14).
+8. Config/mapping drift detection and `NEEDS_REINDEX` status (G15).
+9. Validation that configured fields and the timestamp field exist on the table; `basepullTimestampField = null` for tables without one (G13). E1 says "fail-fast config validation" generically.
+10. Process input screens with a table PVS (G17). Without it, the "Or through the QQQ admin UI" README promise stays false after E5.
+11. `totalHits` cap, `limit`/`offset` bounds, term length bounds (G18, G24).
+12. Display values and PVS translation at index time; `recordLabel` fallback (G19). Core does both; a UI backed by Quick Search will show different labels than core for the same record.
+13. Input-side parity with core (`tableNames`, `limitPerTable`) and `tableLabel` on results (G16). E4 mentions only the output mapping.
+14. The `enableScheduledProcesses` flag removing manual processes (G20).
+15. Run-history retention (G21) and the admin table's missing status/error/count fields (G22). Schema changes are cheap before 1.0.
+16. Health check (G23).
+17. `IndexEvent` ordering metadata and publisher lifecycle (G26). The PRD keeps the SPI but does not say whether it is stable; as it stands an async implementation cannot be correct.
+18. Large-field truncation (G25) and refresh policy (G28).
+19. Defaulting the QBit's field list from `QTableMetaData.searchFields` so hosts do not maintain two lists.
+20. Voyage as a consumer: the local Voyage checkout (`/Users/james.maes/Git.Local/QRunIO/Voyage`, develop, main, circleci-project-setup) contains no reference to Quick Search or OpenSearch on any ref; the PRD's "Voyage (pins 0.2.0)" stakeholder row should be re-verified or dropped. qqq-all is the only confirmed consumer (`DemoQBits.java:151-163`, config-driven `customer` name/email, hand-registered `QBitMetaData` 0.2.1-SNAPSHOT, no `schedulerName` while the SFTP QBit on the same line uses `withSchedulerName("demoScheduler")`).
+
+## 6. Suggested adjustments to the epics
+
+- E1: add field-existence validation, OpenSearch naming validation, timeouts in config, split of `enableScheduledProcesses`.
+- E2: add start-time watermark with overlap, orphan-table sweep in reconcile, config-drift detection, run-history purge, failed-event table replayed by the scheduled run. Bundle the mapping changes (search_analyzer, operator, fieldValues namespacing, display values) into the alias-swap work so the first 1.0 index is right.
+- E3: add index-time exclusion of hidden/masked fields; honour `enabled` in search.
+- E4: add `tableNames`, `limitPerTable`, `tableLabel`, typed `recordId`, term bounds; default fields from `searchFields`.
+- E5: add process input screens, admin table fields, health method, README corrections listed in 7c, and an explicit "SPI stability" statement for `IndexEventPublisher`.

docs/audits/2026-10-04/05-correctness-failures.md

@@ -0,0 +1,91 @@
+# 05. Correctness and silent-failure review: qbit-quick-search
+
+Scope: every file under `src/main/java/com/kingsrook/qbits/quicksearch` and `src/test/java`, read in full, cross-checked against QQQ `v4.1.0-RC.1` (`InsertAction`, `UpdateAction`, `DeleteAction`, `QueryAction`, `QueryInput`, `QRecord`, `ValueUtils`, `TableCustomizerInterface`, `RDBMSQueryAction`, `RDBMSInsertAction`, `RDBMSUpdateAction`, `RDBMSDeleteAction`, `RDBMSTransaction`, `QInstance`, `ProcessLockUtils`) and the opensearch-java 2.10.0 jar in the local Maven repo. Read-only; no repo file was modified.
+
+Paths below are relative to `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1/src/main/java/com/kingsrook/qbits/quicksearch/` unless prefixed with `qqq:` (relative to `/Users/james.maes/Git.Local/QRunIO/qqq/` at tag `v4.1.0-RC.1`).
+
+## Framework facts this review depends on (verified)
+
+- POST_INSERT / POST_UPDATE / POST_DELETE customizers run synchronously, in the calling thread, immediately after the backend DML and before `execute()` returns (`qqq:qqq-backend-core/.../actions/tables/InsertAction.java:286`, `UpdateAction.java:268`, `DeleteAction.java:370-390`).
+- When the caller passes a `QBackendTransaction`, the RDBMS module runs the DML on that transaction's connection (`qqq:qqq-backend-module-rdbms/.../RDBMSInsertAction.java:73-79`, `RDBMSUpdateAction.java:85-91`, `RDBMSDeleteAction.java:87-93`). `RDBMSTransaction` sets `autoCommit=false` (`RDBMSTransaction.java:55`) and only the caller's later `commit()` (`:87`) or `rollback()` (`:137`) ends it. So the customizers run inside an open transaction.
+- A customizer exception never propagates to the caller; QQQ catches it and attaches a `QWarningMessage("An error occurred after the insert/update/delete: ...")` to each output record (`InsertAction.java:344-350`, `UpdateAction.java:291-297`, `DeleteAction.java:418-426`).
+- Post-delete receives the pre-fetched old records minus those with errors (`DeleteAction.java:376`); post-update receives the (possibly sparse) updated records plus `Optional<oldRecordList>`.
+- RDBMS paging emits `LIMIT n OFFSET m` and emits `ORDER BY` only when the filter has order-bys (`qqq:qqq-backend-module-rdbms/.../RDBMSQueryAction.java:340-354`). There is no default ordering, so unordered paging is not stable.
+- `QueryAction` sets `QRecord.recordLabel` only when `queryInput.getShouldGenerateDisplayValues()` is true (`qqq:.../QueryAction.java:503-506`); the default is `false` (`qqq:.../QueryInput.java:56`).
+- `QRecord.getValueString` is `ValueUtils.getValueAsString`, which returns `String.valueOf(value)` for Instant, LocalDate, BigDecimal, Boolean, and the possible-value id (not the label) for enums (`qqq:.../ValueUtils.java:67-90`).
+- `QInstance.addTable` throws `IllegalArgumentException` on a duplicate name (`qqq:.../QInstance.java addTable`).
+- `org.opensearch.client.opensearch._types.OpenSearchException extends java.lang.RuntimeException` (javap on opensearch-java 2.10.0). The client wrapper only catches `IOException`.
+
+## Findings
+
+| ID | Severity | Finding | Failure scenario | Evidence | Recommended fix | 1.0 blocker |
+|----|----------|---------|------------------|----------|-----------------|-------------|
+| F01 | Critical | Real-time indexing happens inside the caller's open transaction, before commit. | Host code inserts an order inside a `QBackendTransaction`, the post-insert customizer indexes it, the host then rolls back: OpenSearch now holds a document for a row that never existed, and it is only removed by the next reconcile (basepull cannot see deletes). Post-update is worse: it re-reads the row through the same transaction (`setTransaction`) and indexes uncommitted values. | `customizers/QuickSearchPostInsertCustomizer.java:97`; `customizers/QuickSearchPostUpdateCustomizer.java:114,151`; `customizers/QuickSearchPostDeleteCustomizer.java:109`; `qqq:InsertAction.java:286`; `qqq:RDBMSInsertAction.java:73-79`; `qqq:RDBMSTransaction.java:55,87` | QQQ has no post-commit hook, so: (a) when `input.getTransaction() != null`, buffer events keyed by the transaction object and flush them from a wrapper the host calls after `commit()` (ship a `QuickSearchTransactionListener` helper), or at minimum (b) document loudly that real-time indexing is pre-commit and that reconcile is the correction path; and make reconcile the recommended scheduled job. | Yes (fix or document; today the README implies the index tracks the table) |
+| F02 | Critical | Real-time indexing failures are fully silent to the host. | OpenSearch is down for ten minutes; every insert/update/delete in that window logs one WARN line and succeeds normally; no run record, no counter, no record warning. QQQ would have attached a `QWarningMessage` to the returned records if the exception had been allowed to escape, but the customizers swallow it first. The index silently drifts until someone runs reconcile. | `customizers/QuickSearchPostInsertCustomizer.java:99-102`; `QuickSearchPostUpdateCustomizer.java:117-120`; `QuickSearchPostDeleteCustomizer.java:112-115`; `publisher/SynchronousIndexEventPublisher.java:160-171` (throwIfAnyFailed is caught by the customizers and goes nowhere); `qqq:InsertAction.java:344-350` | Remove the blanket catch in the three customizers (QQQ already guarantees the write is not failed; the host gets a warning on the records). Additionally record the failure durably: increment a failure counter on the `quickSearchIndex` row (new `realTimeErrorCount`/`lastRealTimeError`), or write the failed (table, id) pairs to a small dead-letter table that basepull/reconcile drains. | Yes |
+| F03 | High | Basepull and FullReindex page with `LIMIT/OFFSET` and no `ORDER BY`. | While FullReindex pages through 1M rows, a concurrent insert or delete shifts the physical order: a row moves from page 101 to page 100 (already read) and is never indexed, or is read twice. Because FullReindex first deletes every document for the table, a skipped row disappears from search until the next full run. Postgres/MySQL do not guarantee a stable order for repeated unordered `LIMIT/OFFSET` queries even without writes. | `processes/AbstractIndexingStep.java:358-359`; `processes/BasepullIndexStep.java:218-262`; `processes/FullReindexStep.java:158-185`; `qqq:RDBMSQueryAction.java:340-354` | Replace offset paging with the keyset loop already written in `processes/ReconcileIndexStep.java:222-238` (order by PK ascending, `PK > lastKey`). For basepull, keyset on PK with the timestamp criterion added. Delete `querySourceTableBatch` once both callers migrate. | Yes |
+| F04 | High | Basepull window is anchored at the end of the run and compared against the application clock, so records modified during the run are lost. | Run starts at 10:00:00, pages for 90 s; a row updated and committed at 10:00:30 is on a page already read with its old values, or not yet visible; at 10:01:30 `lastBasepullTime = now()`; the next run filters `modifyDate > 10:01:30` and never sees the 10:00:30 change. Any DB/app clock skew widens or flips the gap. `GREATER_THAN` also drops rows whose `modifyDate` equals the boundary exactly. | `processes/BasepullIndexStep.java:198-209` (filter), `:268-279` (`lastBasepullTime = Instant.now()` after the loop); `processes/FullReindexStep.java:196` (same end-of-run stamp) | Capture `windowStart = Instant.now()` before the first query and persist that (as `ReconcileIndexStep.java:142,274` already does), subtract a configurable overlap (default: the basepull interval, or 5 minutes) to absorb skew and long transactions, and use `GREATER_THAN_OR_EQUALS`. Re-indexing is idempotent, so overlap is cheap. | Yes |
+| F05 | High | No document versioning; last writer to OpenSearch wins, not last committer. | Thread A updates row 7 (v1) and, in its post-update, reads v1 back; thread B updates row 7 (v2), commits, indexes v2; thread A's bulk arrives last and overwrites with v1. Same race between FullReindex/Reconcile reading an old page and a real-time write that lands between the read and the bulk. Also applies to delete-then-reinsert of the same id across threads. | `opensearch/QuickSearchOpenSearchClient.java:263-267` (`IndexOperation` with no `version`/`versionType`/`ifSeqNo`); `processes/IndexingUtils.java:164` (`indexedAt = now()` is informational only) | Use external versioning: `versionType(External)` with `version = modifyDate.toEpochMilli()` (or a DB version column) so stale writes are rejected with 409 and counted, not applied. Short of that, document that basepull/reconcile is the convergence mechanism (true only once F04 is fixed). | No, if F04 is fixed and the limitation is documented |
+| F06 | High | Search-time analysis uses the edge_ngram analyzer, so query terms are n-grammed too. | Mapping sets `analyzer("quick_search_analyzer")` with no `search_analyzer`, so OpenSearch analyzes the query with the same edge_ngram filter. Searching `stella` becomes `st OR ste OR stel OR stell OR stella` and matches every document containing any token that starts with `st`. Scores still favour the full term, but `totalHits`, `hasMore`, and pages 2+ are mostly noise. | `opensearch/QuickSearchOpenSearchClient.java:177-190` | Add a second analyzer (`standard` tokenizer + `lowercase`) and set it as `searchAnalyzer` on `searchableText`. Requires index recreation (bump the index name or add a mapping-version check in `ensureIndexExists`, which currently returns early if the index exists at `:165-171` and never reconciles mappings). | Yes |
+| F07 | High (behaviour of OpenSearch assumed from its documented default `lenient=false`; verify with the Docker test) | Boost clauses query `fieldValues.<field>` for every configured field, and `fieldValues` is a dynamic object; a non-string searchable field breaks every search. | A table marks `quantity` (Integer) or `createDate` (Instant) as a `@QuickSearchField`. The first indexed document maps `fieldValues.quantity` as `long`. The next search for `widget` issues `multi_match` against that field with a non-numeric term; OpenSearch returns `query_shard_exception: failed to create query ... For input string: "widget"` and the whole bool query (not just the should clause) fails. Separately, dynamic mapping is first-value-wins: a `sku` whose first value looks like a date maps as `date`, and later SKUs fail per-item with `mapper_parsing_exception`. | `opensearch/QuickSearchOpenSearchClient.java:192` (dynamic `object`), `:536-561` (per-field multi_match, no `lenient`); `processes/IndexingUtils.java:150-155` (raw `Object` values copied into `fieldValues`) | Either set `.lenient(true)` on the should-clause multi_match queries, or define a `dynamic_templates` rule mapping everything under `fieldValues.*` as `text` with the search analyzer, or stringify values when building `fieldValues`. | Yes |
+| F08 | High | `recordLabel` is null for every annotation-configured table. | `buildDocument` copies `record.getRecordLabel()`, but the records come from `QueryAction` calls that never set `shouldGenerateDisplayValues`, and from `InsertAction` output, neither of which populates the label. Only tables with `recordLabelFormat` (config-driven path) get a label. Search results for annotation-based tables show no label, which is the primary display field. | `processes/IndexingUtils.java:142`; `processes/AbstractIndexingStep.java:361-365`; `processes/ReconcileIndexStep.java:233-237`; `customizers/QuickSearchPostUpdateCustomizer.java:149-154`; `qqq:QueryAction.java:503-506`; `qqq:QueryInput.java:56` | In `buildDocument`, when `recordLabelFormat` is absent, call `QValueFormatter.formatRecordLabel(QContext.getQInstance().getTable(tableName), record)`; or set `shouldGenerateDisplayValues(true)` on every source query and translate possible values (also fixes F13). | Yes |
+| F09 | High | FullReindex wipes the table's documents first; a mid-run failure leaves a partially empty index with no automatic recovery. | `deleteDocumentsForTable` runs, then OpenSearch returns 429 on batch 40 of 1000: `OpenSearchException` (unchecked) escapes `indexDocuments`, the run is marked FAILED, 96 percent of the table is unsearchable until someone reruns. Also `deleteByQuery` is called with default `conflicts=abort` and no `refresh`, so a real-time write racing the delete raises `version_conflict_engine_exception` and fails the run after a partial wipe. The FAILED run record from the bulk-item path carries `errorMessage = null`. | `processes/FullReindexStep.java:152` (wipe), `:160-185` (loop), `:210` (null errorMessage on FAILED), `:215-220`; `opensearch/QuickSearchOpenSearchClient.java:388-403` (no `conflicts(Proceed)`, no `refresh`) | Prefer the reconcile algorithm (index into place, then delete stale) for the "full" process too, or build into a fresh index and swap an alias. At minimum: pass the first bulk error into `completeRunRecord`, set `conflicts(Proceed)`, and document the blackout. | Yes for the errorMessage and conflicts fixes; the wipe design should be documented or replaced before 1.0 |
+| F10 | Medium | `OpenSearchException` (unchecked) bypasses every `catch(IOException)` in the client and reaches `QuickSearchAction` callers raw. | Someone deletes the index (`index_not_found_exception`), or a client sends `offset=20000` (`max_result_window`), or F07 fires: `QuickSearchAction.execute` throws a `RuntimeException` rather than the declared `QException`; UI/API layers that only handle `QException` surface a 500 with the raw OpenSearch JSON. In the steps and customizers the `catch(Exception)` blocks do catch it, so only the search path is exposed. | `opensearch/QuickSearchOpenSearchClient.java:197,220,349,374,399,441,506,581` (all `catch(IOException)`); `actions/QuickSearchAction.java:91` | Catch `OpenSearchException` alongside `IOException` in every client method and wrap in `QException` with `error().reason()`. | Yes (small) |
+| F11 | Medium | Search input has no upper bound on `limit` or `offset`. | `limit=100000` builds a 100k-hit response (memory and `max_result_window` error); `offset+limit > 10000` is rejected by OpenSearch and surfaces as F10. `totalHits` is also capped at 10000 by `track_total_hits` default, so `hasMore` is wrong beyond that. | `actions/QuickSearchAction.java:78-79`; `opensearch/QuickSearchOpenSearchClient.java:497-502` | Clamp `limit` to a configurable max (e.g. 100) and reject or clamp `offset` so `from + size <= 10000`; set `trackTotalHits(true)` only if exact counts matter. | Yes (small) |
+| F12 | Medium | `quickSearchIndex.enabled` is honoured only by basepull; the row's `basepullTimestampField` is never read. | An admin disables a table in the UI; the next FullReindex or Reconcile still indexes it. An admin edits `basepullTimestampField` on the row; basepull keeps using the annotation value. | `processes/FullReindexStep.java:90-96`; `processes/ReconcileIndexStep.java:98-106`; `processes/BasepullIndexStep.java:113` (read) vs `:207` (uses `tableConfig.getBasepullTimestampField()`) | Check `enabled` in all three steps (or make the field read-only and document). Pick one source of truth for the timestamp field and remove the other. | No (document) |
+| F13 | Medium | Basepull on a table whose config was removed but whose row and QQQ table still exist silently scans everything and indexes nothing. | `getTableConfig` returns null; the filter stays null (full scan), `buildDocument` is skipped for every row, the run completes `COMPLETED` with `recordsProcessed=N, recordsIndexed=0`, and `lastBasepullTime` advances. | `processes/BasepullIndexStep.java:184,202-209,236-240,268` | If `tableConfig == null`, mark the run FAILED (or the row `status=INACTIVE`) and skip. | No |
+| F14 | Medium | `ensureIndexRowExists` assumes a unique constraint on `tableName` that the produced metadata does not declare. | Two replicas run the first basepull together; both see no row, both insert; without a DB unique index there is no exception, the recheck is never reached, and the table now has two rows. Basepull then processes the table twice per cycle (two run records, two `lastBasepullTime` stamps); FullReindex and Reconcile use `get(0)` arbitrarily. | `processes/AbstractIndexingStep.java:143-195`; `QuickSearchQBitProducer.java:457-475` (no `withUniqueKey`) | Add `.withUniqueKey(new UniqueKey("tableName"))` to the table metadata so QQQ's pre-insert unique check fires, and ship DDL with a unique index. | Yes (metadata only) |
+| F15 | Medium | Per-table basepull failures are logged at WARN and the process still reports success. | OpenSearch rejects one table's bulk every run; the scheduler sees a green process forever. | `processes/BasepullIndexStep.java:124-132` | Collect failures and throw (or set an output value and a FAILED status on the step) after the loop; expose counts via `output.addValue` as Reconcile does. | Yes (small) |
+| F16 | Medium | `produce()` tolerates OpenSearch being unreachable and starts with `client=null, publisher=null`. | Misconfigured host/port in production: the app boots healthy, every write logs "publisher is null", every search throws "client is not initialized". Nothing in the operational tables records this. | `QuickSearchQBitProducer.java:159-167,172-182,187-195`; `customizers/*.java:74-79` | Add `failFastOnStartup` (default true) to the config, or at least retry client creation lazily on first use and surface a health check. | No (document) |
+| F17 | Medium | Static singleton context: re-produce leaks transports, and two QInstances in one JVM share one context. | `produce()` called twice (tests, qqq-all, hot reload): the first `ApacheHttpClient5Transport` and its IO reactor threads are never closed (only `clear()` nulls references; main code never calls `publisher.close()` or `client.close()` outside tests). A second QInstance overwrites `config/client/discoveredTables`, so instance A's customizers index into instance B's index. Calling `produce()` twice on the same QInstance throws from `addTable` after the static context has already been replaced. | `QuickSearchQBitContext.java:34-37,152-158`; `QuickSearchQBitProducer.java:200-215`; `qqq:QInstance.java addTable` | Close the previous client when replacing it; register a shutdown hook or expose `QuickSearchQBit.shutdown()`; document the one-QInstance-per-JVM constraint; longer term key the context by `QInstance` identity. | No (document; close-on-replace is trivial and worth doing) |
+| F18 | Medium | `String.format(recordLabelFormat, ...)` with `""` substituted for null throws on non-`%s` specifiers and poisons the whole batch. | `recordLabelFormat="%s #%d"` and `orderNumber` null: `IllegalFormatConversionException` (unchecked) escapes `buildDocument`; in `SynchronousIndexEventPublisher.publishIndexEvents` this happens before any bulk call, so every document in the batch (all tables) is dropped and the customizer logs one WARN. In the steps it fails the table's run. | `processes/IndexingUtils.java:194-200`; `publisher/SynchronousIndexEventPublisher.java:113-120` | Validate the format at `produce()` time against the field types; wrap per-record `buildDocument` in try/catch, count as a failure, continue. | No |
+| F19 | Medium | Indexed text is raw `String.valueOf`: PVS ids instead of labels, ISO instants, `true/false`, unbounded length, no HTML stripping. | A `status` PVS field indexes `3` not `Shipped`, so users cannot search by status label; a `notes` CLOB of 5 MB is sent whole (500 per bulk can exceed `http.max_content_length` and fail the batch with a 413 that surfaces as F10 in the steps). | `processes/IndexingUtils.java:94,150-155`; `qqq:ValueUtils.java:67-90` | Query with `shouldGenerateDisplayValues(true)` and `shouldTranslatePossibleValues(true)` and index `getDisplayValue(field)`; truncate each value to a configurable max (e.g. 32 KB); strip tags for `HTML` adorned fields. | No |
+| F20 | Low | Public `buildSearchableText` dereferences `includeLabels` without a null check. | Any external caller passing `null` for `includeLabels` gets an NPE; the table-config overload guards it, the 6-arg overload does not. | `processes/IndexingUtils.java:83-115` (`:106`) vs `:190` | Null-guard inside `buildSearchableText`. | No |
+| F21 | Low | Document ids are `sourceTable:recordId`, not namespaced by `tableNamePrefix` or instance. | Two apps pointed at the same `opensearchIndexName` with a table named `order` overwrite each other's documents, and either app's FullReindex/Reconcile deletes the other's documents (`deleteDocumentsForTable`, `deleteDocumentsIndexedBefore` filter on `sourceTable` only). Changing `tableNamePrefix` does not change ids (good: no orphan churn). A table name containing `:` can collide with another (table, id) pair only in contrived cases. | `opensearch/OpenSearchDocument.java:184-186`; `opensearch/QuickSearchOpenSearchClient.java:392-397,428-437` | Document one index per QInstance (validate at produce time if the index already holds a different app's marker document), or include an app/prefix token in `sourceTable`. | No |
+| F22 | Low | Reconcile's orphan detection depends on `indexedAt` written by whichever node indexed the document, using that node's clock. | Node B's clock is 3 minutes behind node A; node A starts reconcile at T; node B inserts a row at T+1 min and indexes it with `indexedAt = T-2 min`; reconcile deletes it as an orphan. Basepull repairs it only if its `modifyDate` (DB clock) is `> reconcileStart` and the row still exists. The comment at `:62-63` acknowledges the assumption. The orphan delete itself uses `delete_by_query` with `conflicts(Proceed)` and `refresh`, so there is no 10000-document cap and no scroll needed. | `processes/ReconcileIndexStep.java:142,192`; `opensearch/QuickSearchOpenSearchClient.java:420-445`; `processes/IndexingUtils.java:164` | Replace the time cutoff with a per-run token (`reconcileRunId` stored on each document; delete `sourceTable = X AND reconcileRunId != current`), or subtract a skew margin from the cutoff. | No |
+| F23 | Low | `RUNNING` run records are never swept; `completeRunRecord` failures are WARN-only. | JVM killed mid-run (or DB blip at completion): the row stays `RUNNING` indefinitely; operators cannot tell a hung run from a dead one. | `processes/AbstractIndexingStep.java:264-292,310-334` | On step start, mark `RUNNING` rows older than N hours as `ABANDONED`; or add a heartbeat column. | No |
+| F24 | Low | FullReindex aborts remaining tables on the first table failure; basepull continues. Inconsistent semantics. | A reindex-all with 5 tables fails on table 2; tables 3 to 5 are untouched and no run records exist for them. | `processes/FullReindexStep.java:90-96,215-220` vs `processes/BasepullIndexStep.java:124-132` | Pick one policy (continue and aggregate is friendlier for a reindex-all) and apply it to all three steps. | No |
+| F25 | Low | `normalizeSearchText` lowercases client-side even though both index and (intended) search analyzers lowercase; harmless but misleading once F06 is fixed. The highlight join and `totalHits` fallback are correct: `fragments` emptiness is checked before `String.join("...")`, and when `total()` is null `totalHits = results.size()` so `hasMore` is false. | None. | `actions/QuickSearchAction.java:76,102-110,123-124` | Leave the whitespace collapsing, drop the lowercase, or keep as belt-and-braces with a comment. | No |
+
+### Items checked and found acceptable
+
+- `ReconcileIndexStep` keyset paging works for non-numeric keys and gaps: both the `ORDER BY` and the `GREATER_THAN` comparison are evaluated by the database, so string collation is consistent; a null primary key fails fast (`processes/ReconcileIndexStep.java:178-185`). Not verified for `byte[]` (binary UUID) keys.
+- `ReconcileIndexStep` when the orphan delete throws after a successful re-index: run FAILED, index row untouched, documents are all current; the next run simply repeats. Acceptable.
+- `ReconcileIndexStep` race with a concurrent real-time update (stale page wins, F05) is repaired by the next basepull because `lastBasepullTime` is set to `reconcileStart` (`:274`), provided F04's overlap is in place.
+- `deleteDocuments` treats `not_found` as success (bulk delete items with status 404 carry no `error`), matching the comment at `opensearch/QuickSearchOpenSearchClient.java:283-284`.
+- `BulkIndexResult` accounting per item is correct; partial bulk application is reported, not hidden, on the batch path.
+- Post-delete correctly skips records with errors and null ids (`customizers/QuickSearchPostDeleteCustomizer.java:95-99`); post-update correctly re-reads the full row rather than indexing the sparse update (`QuickSearchPostUpdateCustomizer.java:132-155`), though through the open transaction (F01).
+
+## Missing tests
+
+Unit tests use `MemoryBackendModule` with a mocked `QuickSearchOpenSearchClient`; integration tests use the memory backend plus a Testcontainers OpenSearch. No test uses an RDBMS backend or a `QBackendTransaction`, so none of the transaction, ordering, or SQL-paging hazards are exercised.
+
+Covered today (for calibration): post-update sparse-record re-read (`QuickSearchPostUpdateCustomizerTest.testUpdateAction_partialUpdate_reindexesFullRecord`, `IndexDriftIntegrationTest.testPartialUpdate_unchangedFieldsStaySearchable`); publisher throws on item failures (`SynchronousIndexEventPublisherTest.testPublish*_itemFailures_throwsQException`); customizers swallow publisher exceptions (`*CustomizerTest.*_publisherThrows_returnsRecordsWithoutThrowing`, which is testing the F02 behaviour as desired); reconcile keyset paging, stale delete skipped on errors, run FAILED (`ReconcileIndexStepTest`, `IndexDriftIntegrationTest.testReconcile_repairsOrphanMissingAndStaleDocuments`); basepull due/not-due and FAILED on client throw (`BasepullIndexStepTest`); full reindex wipe-then-index ordering and FAILED on throw (`FullReindexStepTest`); search mapping, defaults, hasMore (`QuickSearchActionTest`); config-driven `recordLabelFormat` label (`ConfigDrivenIndexingIntegrationTest.testSearchResults_recordLabelFormatApplied`).
+
+Missing, by finding:
+
+1. F01: `QuickSearchPostInsertCustomizerTest` (or a new `TransactionIntegrationTest` against H2 via `qqq-backend-module-rdbms`): insert inside a `QBackendTransaction`, roll back, assert no `publishIndexEvents` call (today it will be called, so this test documents the defect until fixed). Same for update and delete.
+2. F02: `QuickSearchPostInsertCustomizerTest`: when the publisher throws, assert the returned records carry a warning (after the fix) or that a failure counter/dead-letter row was written; today's `*_publisherThrows_returnsRecordsWithoutThrowing` tests lock in the silent behaviour and should be inverted.
+3. F03: `FullReindexStepTest` and `BasepullIndexStepTest`: capture every `QueryInput` passed to the source table and assert each filter has an `ORDER BY` on the primary key and uses a keyset criterion (no `skip`). Integration variant: insert a row from another thread between pages (use a `Answer` on the mocked client to trigger the insert during `indexDocuments`) and assert every id was indexed exactly once.
+4. F04: `BasepullIndexStepTest.testLastBasepullTime_isCapturedBeforeQuerying`: insert a record with `modifyDate` between the step's start and end (same trick: insert from within the mocked `indexDocuments`), run twice, assert it is indexed on the second run. `testBasepullFilter_isGreaterThanOrEquals_withOverlap`: assert criterion operator and that the bound is `lastBasepullTime - overlap`.
+5. F05: `QuickSearchOpenSearchClientTest` or `OpenSearchIntegrationTest.testStaleWriteRejected`: index v2 of a document, then v1 with an older version, assert v2 survives (requires the external-version fix).
+6. F06: `OpenSearchIntegrationTest.testSearch_doesNotMatchOnSharedPrefixOnly`: index "stella" and "stone", search "stella", assert `totalHits == 1`. Also `testEnsureIndexExists_upgradesMappingWhenVersionChanges`.
+7. F07: `OpenSearchIntegrationTest.testSearch_tableWithIntegerAndDateSearchableFields_doesNotError`: add an `INTEGER` and a `DATE_TIME` field to a searchable table, reindex, search a word. `testIndex_dynamicMappingConflictOnFieldValues_reportedPerItem` for the first-value-wins mapping case.
+8. F08: `OpenSearchIntegrationTest.testSearchResults_annotationTable_haveRecordLabel`: assert `getRecordLabel()` is not null for the annotation-based `testEntity` table (currently it would fail). `IndexingUtilsTest.testBuildDocument_withoutFormat_usesTableRecordLabelFields`.
+9. F09: `FullReindexStepTest.testBulkItemFailures_runRecordHasErrorMessage` (currently `errorMessage` is null on that path). `QuickSearchOpenSearchClientTest`/integration: `deleteDocumentsForTable` proceeds on version conflicts.
+10. F10: `QuickSearchOpenSearchClientTest.testSearch_indexMissing_throwsQException` (start a container, delete the index, search); `QuickSearchActionTest.testOpenSearchException_isWrappedAsQException` using a mocked client that throws `OpenSearchException`.
+11. F11: `QuickSearchActionTest.testLimit_aboveMax_isClamped`, `testOffsetPlusLimit_aboveMaxResultWindow_isRejectedOrClamped`.
+12. F12: `FullReindexStepTest.testDisabledIndexRow_isSkipped`, `ReconcileIndexStepTest.testDisabledIndexRow_isSkipped`, `BasepullIndexStepTest.testRowTimestampField_isUsedOverAnnotation` (or the inverse, once a source of truth is chosen).
+13. F13: `BasepullIndexStepTest.testIndexRowWithoutTableConfig_runMarkedFailedAndNoScan`.
+14. F14: `QuickSearchQBitProducerTest.testIndexTable_hasUniqueKeyOnTableName`; `BasepullIndexStepTest.testEnsureIndexRowExists_concurrentFirstRun_createsOneRow` (two threads).
+15. F15: `BasepullIndexStepTest.testOneTableFails_stepReportsFailure` (assert the step throws or sets an output value after processing the other tables).
+16. F16: `QuickSearchQBitProducerTest.testProduce_openSearchUnreachable_failsFastWhenConfigured`.
+17. F17: `QuickSearchQBitProducerTest.testProduce_twice_closesPreviousClient` (spy on client), `testProduce_sameQInstanceTwice_doesNotLeaveContextHalfReplaced`.
+18. F18: `IndexingUtilsTest.testBuildDocument_recordLabelFormatWithNumericSpecifierAndNullValue_doesNotThrow`; `SynchronousIndexEventPublisherTest.testOneBadRecord_doesNotDropWholeBatch`.
+19. F19: `IndexingUtilsTest.testBuildSearchableText_possibleValueField_usesLabel`, `testBuildSearchableText_instantAndBigDecimal_formatting`, `testBuildSearchableText_veryLongValue_truncated`.
+20. F20: `IndexingUtilsTest.testBuildSearchableText_nullIncludeLabels_doesNotThrow`.
+21. F22: `ReconcileIndexStepTest.testDocumentIndexedByLaggingClock_duringRun_notDeleted` (requires the run-token design).
+22. F23: `AbstractIndexingStep` sweeper test: a `RUNNING` run older than the threshold is marked `ABANDONED` on the next step start.
+
+## Summary of blockers for 1.0
+
+Must fix or explicitly document before 1.0: F01 (pre-commit indexing), F02 (silent real-time failures), F03 (unordered paging), F04 (basepull window), F06 (search analyzer), F07 (non-text searchable fields break search), F08 (null record labels), F09 (errorMessage and conflicts on full reindex), F10 (unchecked OpenSearchException), F11 (limit/offset bounds), F14 (unique key on tableName), F15 (basepull hides failures). Everything else can ship with a documented limitation.

docs/audits/2026-10-04/README.md

@@ -0,0 +1,108 @@
+# Deep-dive audit: QBit Quick Search for 1.0 on QQQ 4.1
+
+Date: 2026-10-04. Code: `develop` `000e5bb` (identical to `feature/GH-6-repin-and-drift-fix`), 0.2.1-SNAPSHOT. References: QQQ `v4.1.0-RC.1`, opensearch-java 2.10.0, OpenSearch and AWS documentation as of this date. Five parallel code-grounded reviews; the raw reports with file:line evidence are the numbered files beside this one. Every claim below was spot-checked against source where it changed a decision; one agent claim (that Voyage does not use the QBit) was wrong and is corrected here.
+
+## Verdict
+
+The QBit compiles and passes its 233 unit and 12 integration tests on QQQ 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1 with no source change, so it is **API-compatible with QQQ 4.1 today**. It is **not feature complete or production-safe for a 1.0**: the index can hold uncommitted writes, failures are silent, basepull loses records and is never scheduled, search leaks data across tables, the search analyzer inflates matches, the OpenSearch client line is end-of-life, and only HTTP basic auth over an optional TLS connection is supported. None of these are hard; most are a few days each. The draft PRD covered about half of them. The PRD is updated alongside this audit.
+
+| Dimension | Score (1 to 10) | One line |
+|---|---|---|
+| QQQ 4.1 API compatibility | 9 | Builds and tests green on RC.1 (see section 1 for the contract gaps that remain) |
+| QQQ 4.1 QBit contract | 3 | Plain producer, no `QBitMetaData`, no schedule, no permission rules, static context, slot collisions |
+| Index trustworthiness | 3 | Pre-commit indexing, silent failures, lossy basepull window, unordered paging, wipe-first reindex |
+| Search quality and safety | 3 | No permissions, query terms edge-ngrammed, OR semantics, non-text fields break search, unbounded paging |
+| OpenSearch and AWS compatibility | 4 | 2.x self-managed and AWS FGAC basic auth work; 3.x needs a client bump; IAM SigV4 and Serverless unsupported |
+| Authentication and secrets | 4 | Basic only, plaintext credentials allowed, password on serializable metadata, no TLS trust config |
+| Operability | 4 | No timeouts, no health check, no failed-event capture, admin tables thin, processes without input screens |
+| Tests and docs | 5 | Good unit coverage of happy paths; JaCoCo gate inert; integration suite tests one word; README over-promises |
+
+## 1. QQQ 4.1 integration readiness
+
+Full detail with file:line evidence in `01-qqq-4.1-integration.md`. The QBit compiles and tests green on 4.1.0-RC.1, but it uses none of the 4.1 contract and misses three 4.1 APIs that solve its hardest problems. All three are absent from 4.0.0 (verified by tag), so a 1.0 that adopts them requires qqq 4.1 (decision D10).
+
+| Topic | Today | 4.1 answer | Refs |
+|---|---|---|---|
+| Producer contract | Plain class; `produce(QInstance)` adds tables, processes and app directly; no `QBitMetaData`; qqq-all stamps one by hand with version `0.2.1-SNAPSHOT` and its test asserts artifactId `quick-search` | `QBitMetaDataProducer<QuickSearchQBitConfig>` (3 of 4 sibling QBits use it); `getQBitMetaData()` with the real version from a Maven-filtered resource; component producers or `@QMetaDataProducingEntity`; registration in `postProduceActions`; keep `withConfig()` and a `produce(QInstance)` override that adds to the instance so qqq-all and Voyage keep compiling | 01 F-01, F-12 |
+| Real-time hook and transactions | Three per-table POST_* customizers; run inside the caller's open transaction before commit; `withCustomizer` throws `IllegalArgumentException` on an occupied slot | `RecordChangeListenerInterface` (instance-level, `appliesTo(table, type)`, covers INSERT/UPDATE/DELETE, cannot fail the write, carries old records and the transaction) registered with `QInstance.withRecordChangeListener`; publish through `QBackendTransaction.addAfterCommitCallback` when a transaction is present (discarded on rollback; `RDBMSTransaction` runs them after JDBC commit); reference implementation `qqq-esb` `EsbRecordChangeListener`. `MultiCustomizer` only if customizers are kept as a 4.0 fallback; instance-level `withTableCustomizer` is unsuitable because `DeleteAction` never consults it | 01 F-02, F-03, F-04 |
+| Scheduling | No `QScheduleMetaData` or `BasepullConfiguration`; `QApplicationLauncher` starts the scheduler only if some process has a schedule; validator requires repeat-or-cron plus an existing scheduler name | `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression` on the config; attach schedules only when `schedulerName` is set; keep the per-table `quickSearchIndex` watermark (core `BasepullConfiguration` is keyed per process, not per table) but anchor it at run start minus an overlap | 01 F-05 |
+| Permissions | No `PermissionsHelper` in search; no `QPermissionRules` on produced metadata (enricher default `NOT_PROTECTED`) | Mirror core `RecordSearchAction` (table READ filter, `QueryAction` for locks); `permissionRules` config defaulting to `HAS_ACCESS_PERMISSION` for processes and app, `READ_INSERT_EDIT_DELETE_PERMISSIONS` for the operational tables | 01 F-06 |
+| Validation timing | Backend lookup works for the qqq-all ordering; package scanning would instantiate the producer with a null config and NPE; `QBitConfig.validate` swallows thrown exceptions (only collected errors fail); `QInstanceValidator` has no qbit checks | `isEnabled()` false when config is null; keep list-based validate null-safe; a `QInstanceValidatorPluginInterface` for late checks (backend, searchable tables, scheduler) | 01 F-07 |
+| Runtime state and lifecycle | Static `QuickSearchQBitContext`; OpenSearch connect and index creation happen during metadata production; nothing closes the transport | `QRuntimeServiceInterface` (`start` connects and ensures the index, `stop` closes), registered with `qInstance.withRuntimeService`; config resolved from `SourceQBitAware.getSourceQBitConfig()` on produced tables and processes (the webhooks pattern); `startupMode` `DEGRADED` or `FAIL_FAST`. Note: qqq-all's `FullQBitsTest` builds the full instance with OpenSearch at `127.0.0.1:1`, so either `DEGRADED` stays the default or qqq-all's test changes | 01 F-08, F-09, F-10 |
+| Core search overlap | `RecordSearchExecutor` hard-wires `new RecordSearchAction()`; no SPI | Propose `QInstance.withRecordSearchProvider(QCodeReference)` plus `RecordSearchProviderInterface { appliesTo(table); search(input, table, term, limit) }`, consulted after core's permission check | 01 F-11 |
+| Build | JaCoCo `check` has no `prepare-agent` and no version; JUnit pinned 5.10.0 while parent manages `junit-bom` 6.0.1; parent 2.1.0 not yet tagged | Bind the agent with an explicit version; drop the JUnit pins; re-pin to parent 2.1.0 when released and retire the snapshot profile's BOM import | 01 F-13 |
+
+Ordered migration plan (from report 01): 1 contract and compat shims; 2 config surface; 3 record-change listener with after-commit publish; 4 runtime service and `SourceQBitAware` config resolution; 5 schedules, watermark, keyset paging; 6 permission-aware search; 7 build and parent 2.1.0 re-pin; 8 qqq-all follow-up (remove the manual `addQBit`, pass `withSchedulerName("demoScheduler")`) and the qqq SPI proposal.
+
+## Status (2026-10-04, branch feature/GH-1-0-release-readiness)
+
+All 25 blockers below are implemented on the release branch (commits 90cd09d through the admin-screen commit). Verification: 307 unit tests and 18 Testcontainers integration tests green against qqq 4.1.0-RC.1 and OpenSearch 2.19.6, JaCoCo 85% instruction / 96% class. Remaining before the 1.0.0 tag: parent 2.1.0 re-pin when published, qqq-all follow-up (remove its manual `addQBit`, set `schedulerName`, `startupMode DEGRADED` for its test), CI image matrix job, and the qqq core search SPI proposal (1.1).
+
+## 2. Blockers for 1.0, deduplicated across reports
+
+Severity: Critical = wrong or leaked data; High = breaks a target platform or a core promise; Medium = operational risk. IDs reference the raw reports (A = auth, C = compatibility, F = correctness, G = features).
+
+| # | Sev | Blocker | Evidence | Fix in brief | Refs |
+|---|---|---|---|---|---|
+| B1 | Critical | Real-time indexing runs inside the host's open transaction, before commit; a rollback leaves a ghost document, and post-update re-reads uncommitted values through the same transaction (javalin single-record endpoints pass no transaction and autocommit, so this bites ETL, bulk and transactional host code) | `customizers/*` :97, :114, :151, :109; qqq `InsertAction.java:286`, `RDBMSTransaction.java:55,87`, `StreamedETLExecuteStep.java:204-218` | Replace the customizers with a `RecordChangeListenerInterface` (4.1) and publish through `QBackendTransaction.addAfterCommitCallback` (4.1, discarded on rollback) when a transaction is present; publish immediately otherwise | F01, 01 F-02, F-03 |
+| B2 | Critical | Real-time indexing failures are fully silent: customizers catch everything and log WARN; nothing reaches the host records or any table | `customizers/*` :99-120; `SynchronousIndexEventPublisher.java:160-171` | The record-change listener cannot fail the write by design; record failed (table, id, action) rows in a failed-event table that the scheduled run replays, and surface counts on `quickSearchIndex` | F02, G06 |
+| B3 | Critical | Search returns labels and highlight snippets from every indexed table with no session, table-permission or record-lock check; the `tableName` filter is caller-controlled | `actions/QuickSearchAction.java:81-121`; `QuickSearchOpenSearchClient.java:481-487` | `QuickSearchInput extends AbstractActionInput`; filter tables with `PermissionsHelper.hasTablePermission(READ)` as core `RecordSearchAction.java:110` does; `terms` filter on `sourceTable`; re-read hits via `QueryAction` by primary key for record locks | A1, G02 |
+| B4 | High | Basepull and full reindex page with limit/skip and no order-by; concurrent writes skip or duplicate rows, and full reindex wiped first so skipped rows vanish | `AbstractIndexingStep.java:358-359`; qqq `RDBMSQueryAction.java:340-354` | Reuse the keyset loop already in `ReconcileIndexStep.java:222-238` | F03, G12 |
+| B5 | High | Basepull watermark is `Instant.now()` at the end of the run with a strict greater-than, so rows modified during the run or in the boundary second are lost forever | `BasepullIndexStep.java:198-209, 268-279` | Capture run start before the first query, persist that minus an overlap, use greater-than-or-equal; reconcile already does this at `:142, :274` | F04, G08 |
+| B6 | High | Basepull and reconcile are never scheduled; README calls basepull "scheduled" | no `withSchedule` anywhere; `QuickSearchQBitProducer.java:510-518` | `QScheduleMetaData` on both processes with a `schedulerName` config (decision D4) | G03 |
+| B7 | High | `searchableText` has the edge-ngram analyzer but no `search_analyzer`, so query terms are n-grammed too: "stella" matches anything starting with "st"; `multi_match` defaults to OR with no `minimum_should_match` | `QuickSearchOpenSearchClient.java:177-190, 472-475` | Add a standard+lowercase search analyzer, `operator=AND` or `minimum_should_match`; mapping change, so bundle with the index-versioning work | F06, C5, G04 |
+| B8 | High | `fieldValues` is one dynamic object shared by all tables with raw typed values; same-named fields of different types cause `mapper_parsing_exception`, and boost clauses without `lenient` make a text term against a numeric field fail the whole search | `QuickSearchOpenSearchClient.java:192, 536-561`; `IndexingUtils.java:150-155` | `dynamic_templates` mapping `fieldValues.*` to text, `date_detection: false`, stringify values, `lenient(true)` | F07, C6, G09 |
+| B9 | High | `recordLabel` is null for every annotation-configured table because no source query sets `shouldGenerateDisplayValues` and insert output has no label; PVS fields index ids, not labels | `IndexingUtils.java:94, 142`; qqq `QueryAction.java:503-506` | Fall back to `QValueFormatter.formatRecordLabel`; query with display values and PVS translation | F08, F19, G19 |
+| B10 | High | Full reindex wipes the table's documents first; a mid-run failure leaves a partial index with no recovery; delete-by-query uses `conflicts=abort`; FAILED runs from the bulk path have a null error message | `FullReindexStep.java:152, 160-185, 210`; `QuickSearchOpenSearchClient.java:388-403` | Build into a fresh index and swap an alias (decision D5) or reuse the reconcile algorithm; `conflicts(Proceed)`; pass the first bulk error into the run record | F09, G12 |
+| B11 | High | No TLS trust configuration (no CA, trust store, or guarded insecure flag); `useSsl` defaults false and basic credentials over plain HTTP pass validation | `QuickSearchOpenSearchClient.java:98-139`; `QuickSearchQBitConfig.java:54, 103-112` | `tls` block (CA path, trust store, hostname verification, guarded `insecureSkipVerify`); reject credentials over http unless `allowPlaintextCredentials`; prefer a single `opensearchUrl` defaulting to https | C1, A2, A5 |
+| B12 | High | Password is a plain String on a Serializable config that hosts attach to `QBitMetaData`; `QInstanceAdapter.qInstanceToJson` would emit it | `QuickSearchQBitConfig.java:52-53`; qqq-all `DemoQBits.java:162-163` | Interpret `${env.X}` references with `QMetaDataVariableInterpreter` at client construction (the RDBMS/Mongo/S3 backend pattern); `@JsonIgnore` secret getters; redacting `toString()` | A3 |
+| B13 | High | opensearch-java pinned at 2.10.0, the end-of-life 2.x line documented for OpenSearch 1.x to 2.x; AWS already offers 3.1 to 3.5 | `pom.xml:24`; compatibility matrix | Bump to 3.10.0; the only compile break is `.refresh(true)` to `.refresh(Refresh.True)` at `QuickSearchOpenSearchClient.java:427` (verified: 233 tests pass after) | C2 |
+| B14 | High | Startup is best-effort: an unreachable cluster leaves `client=null`, the host boots healthy, every write logs and every search throws; connect and index creation are side effects inside metadata production | `QuickSearchQBitProducer.java:159-195` | Move connect, `ensureIndexExists` and close into a `QRuntimeServiceInterface` (4.1) with `startupMode` `FAIL_FAST` or `DEGRADED`; qqq-all's `FullQBitsTest` currently relies on degraded behaviour, so the default is a decision (PRD D11) | F16, G05, 01 F-09, F-10 |
+| B15 | Medium | No connect/response timeouts or pool sizing exposed; opensearch-java defaults are 1 s connect and 30 s response; a hung node blocks every host write | `QuickSearchOpenSearchClient.java:105-139` | `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `User-Agent`; multi-host list | C9, A7, G01 |
+| B16 | Medium | Processes, operational tables and the admin app declare no `QPermissionRules`, so they inherit the instance default (`NOT_PROTECTED` unless the host tightened it); anyone can trigger a full reindex | `QuickSearchQBitProducer.java:457-549` | `HAS_ACCESS_PERMISSION` on processes and app with base name `quickSearchAdmin`; `READ_WRITE_PERMISSIONS` on tables; overridable | A6 |
+| B17 | Medium | `limit` and `offset` unbounded; `from + size > 10000` fails with a raw error; `totalHits` silently capped at 10000; no term length bounds | `QuickSearchAction.java:67-79, 123-124` | Clamp limit (100) and window (10000), bound term length (2 to 100) matching core, `track_total_hits` bounded with a lower-bound flag | F11, C7, A11, G18, G24 |
+| B18 | Medium | `OpenSearchException` is unchecked and bypasses every `catch(IOException)`, so search callers get a raw RuntimeException | `QuickSearchOpenSearchClient.java` all client methods; `QuickSearchAction.java:91` | Catch `OpenSearchException` and wrap in `QException` | F10 |
+| B19 | Medium | `quickSearchIndex.tableName` has no unique key, so concurrent first runs on two replicas create duplicate rows and the table is processed twice per cycle | `AbstractIndexingStep.java:143-195`; `QuickSearchQBitProducer.java:457-475` | `withUniqueKey(new UniqueKey("tableName"))` plus DDL note | F14 |
+| B20 | Medium | Per-table basepull failures are logged at WARN and the process reports success forever | `BasepullIndexStep.java:124-132` | Aggregate failures and fail the step or set a FAILED status | F15 |
+| B21 | Medium | `quickSearchIndex.enabled` is honoured only by basepull; customizers, full reindex, reconcile and search ignore it | `BasepullIndexStep.java:89-92` only | Consult the row (cached) in all paths; exclude disabled tables from search | F12, G07 |
+| B22 | Medium | Integration tests pin OpenSearch 2.11.0 (2023) and search one whole word; nothing covers 2.19.x or 3.x, typeahead, multi-word, boosts, basepull, scheduling or permissions; JaCoCo check is skipped (no `prepare-agent`) | `*IntegrationTest.java:76-85`; `pom.xml:188-212` | Parameterize the image, run 2.19.6 and 3.9.0 in CI, bind the JaCoCo agent, add the tests listed in reports 04 and 05 | C10, G32 |
+| B23 | Medium | No orphan sweep for tables removed from config or renamed; no detection that config or mapping drifted from the live index (`ensureIndexExists` returns early if the index exists) | `ReconcileIndexStep.java:98`; `QuickSearchOpenSearchClient.java:167-171`; `AbstractIndexingStep.java:167-175` | Reconcile deletes `sourceTable` not in the discovered set; compare config to `searchableFieldsJson` and set `NEEDS_REINDEX`; verify analyzer settings on an existing index | G14, G15 |
+| B24 | Medium | Validation never checks that searchable fields or the basepull timestamp field exist on the table; tables without `modifyDate` fail at run time, caught and logged | `QuickSearchQBitConfig.java:73-157` | Fail fast at produce; allow `basepullTimestampField = null` meaning reconcile-only | G13 |
+| B25 | Medium | Processes have no input screen, so `tableName` cannot be supplied from the admin UI; `enableScheduledProcesses=false` also removes the manual full reindex and reconcile | `QuickSearchQBitProducer.java:228-232, 510-549` | Frontend step with an optional table PVS; split the flag | G17, G20 |
+
+Also from report 01: drop the local JUnit 5.10.0 pins in favour of the parent's `junit-bom` 6.0.1 (01 F-13); add `getDefaultBackendNameForTables()` and a settable `tableMetaDataCustomizer` on the config so hosts can set RDBMS backend details the way qqq-all does for the other QBits (01 answer 1); use the `QBitMetaData.namespace` as the default `tableNamePrefix` (01 F-12).
+
+Non-blocking but recommended before 1.0: external document versioning from `modifyDate` so stale writes are rejected (F05); failed-event replay plus bounded retry on bulk 429/5xx (G06); byte-size cap on bulk batches for the AWS 10 MiB payload limit on small instances (C8); close the client and publisher on shutdown and on re-produce (A8, F17); `IndexEvent` needs `occurredAt` and a version if the publisher SPI is to be called stable (G26); run-history purge (G21); admin table fields `lastRunStatus`, `lastErrorMessage`, `documentCount`, `lastReconcileTime` (G22); health method (G23); per-field `maxLength` (G25); mark `indexEventPublisher` transient on the Serializable config (A10).
+
+## 3. OpenSearch and AWS compatibility matrix
+
+| Target | Works today | What breaks | Needed |
+|---|---|---|---|
+| OpenSearch 2.x self-managed, security disabled | Yes | Nothing; the only tested configuration (image 2.11.0) | Test 2.19.x too |
+| OpenSearch 2.x self-managed, security plugin, basic auth over TLS | Partly | Demo or private-CA certificates fail the JVM trust check; basic auth only | B11; documented least-privilege role |
+| OpenSearch 3.x self-managed | No (client line) | opensearch-java 2.x is documented for 1.x to 2.x; server negotiates HTTP/2 | B13; add 3.x to the IT matrix |
+| AWS managed domain 2.x or 3.x, FGAC, internal-database master user, basic auth | Yes for 2.x, 3.x after B13 | Needs port 443, `useSsl`, and an access policy that does not require signed requests; 10 MiB payload limit on t3 and `*.large` | B13, byte-capped bulks, README guidance |
+| AWS managed domain with IAM master user or IAM principals in the access policy (SigV4) | No | Unsigned requests rejected; no `AwsSdk2Transport`, region or service name | `authMode=AWS_SIGV4` with `AwsSdk2Transport`, service `es`, default credential chain, optional assume-role; declare `software.amazon.awssdk:aws-crt-client` and `auth` as optional dependencies (decision D6) |
+| Amazon OpenSearch Serverless | No | SigV4 service `aoss` plus `x-amz-content-sha256`; `_delete_by_query` and `_refresh` are not in the AOSS supported-operations table (used by full reindex and reconcile); custom `_id` only on Search collections; refresh lag about 10 s; client must be 3.x | Declare unsupported for 1.0; later: SigV4, replace delete-by-query with search plus bulk delete, replace refresh with a bounded wait |
+| Legacy Elasticsearch 7.10 domains, OpenSearch 1.x | Untested | Client matrix lists only OpenSearch 1.x to 3.x; renamed APIs | Out of scope |
+
+Recommended 1.0 supported-platforms statement: self-managed OpenSearch 2.19.x and 3.x latest minor over HTTP or HTTPS with the security plugin disabled or HTTP basic auth; Amazon OpenSearch Service managed domains 2.x and 3.x with fine-grained access control, an internal-database master user, basic auth, port 443 and TLS. Not supported in 1.0: IAM SigV4 (unless D6 says otherwise), Serverless, Elasticsearch endpoints, OpenSearch 1.x. Least-privilege role on the one index: `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*` and an `indices:admin/*` read for the exists check, and the cluster-level `cluster_composite_ops` group for `_bulk`. Sources are listed in report 02.
+
+## 4. Authentication options that make sense for 1.0
+
+Today: none, or HTTP basic. Recommended minimum: an `authMode` enum with `NONE`, `BASIC`, `AWS_SIGV4` (`awsRegion`, `awsServiceName` default `es`, optional `awsAssumeRoleArn`), plus one escape hatch, a `transportCustomizer` `QCodeReference` to an interface with `customizeTransport(ApacheHttpClient5TransportBuilder)` and `customizeHttpClient(HttpAsyncClientBuilder)`, which lets hosts add bearer or JWT tokens, API-key headers and mTLS without a fork. Do not add `BEARER`, `API_KEY` or `MTLS` enum values in 1.0; each needs a token-refresh or key-store story the SPI defers. All secret-bearing strings are resolved with `QMetaDataVariableInterpreter` inside the client factory so hosts write `"${env.OPENSEARCH_PASSWORD}"` and metadata never holds the value. Validation: BASIC needs both username and password; AWS_SIGV4 needs a resolvable region and forbids username/password; credentials over http rejected unless `allowPlaintextCredentials`; `insecureSkipVerify` refused for non-loopback hosts unless a second explicit flag is set, with a WARN at startup. Full field sketch with defaults is in report 03, section "Recommended 1.0 auth configuration". The deprecation path keeps `withOpensearchHost/Port/UseSsl/Username/Password` working and infers `BASIC` when a username is set.
+
+Supply chain: no concrete CVE in the resolved set (httpclient5 5.6.3, jackson 2.21.5, log4j 2.25.5). qqq-backend-core still pulls AWS SDK for Java v1 1.12.797, which reached end of support 2025-12-31; a qqq concern, and this QBit should use SDK v2 only.
+
+## 5. Feature completeness versus QQQ 4.1 core search
+
+QQQ 4.1 added `RecordSearchAction`, `QTableMetaData.searchFields` and `POST /qqq/v1/search`, consumed by the Next UI. Core has: table READ permission and capability checks, field personalisation (hidden and masked fields excluded), record locks via `QueryAction`, `limitPerTable` (5 default, 25 max), a `tableNames` list, a 100-character term cap, a 30 s timeout, `tableLabel`, typed `recordId`, label fallback and display values. Quick Search has: relevance scoring with weights, typeahead (defective, B7), highlights, offset pagination with totals, cross-table ranking, OLTP isolation, run history and the publisher SPI. To back the Next UI (PRD E4) Quick Search must first accept `tableNames` and `limitPerTable` and return `tableLabel` and a typed `recordId` (G16), and it should default its field list from `searchFields` so hosts do not maintain two lists. Core has no provider SPI; see decision D1.
+
+## 6. Consumers (verified)
+
+- qqq-all `DemoQBits.java:151-163` (origin/develop): config-driven `customer` with `name` and `email`, hand-registered `QBitMetaData` at `0.2.1-SNAPSHOT`, no `schedulerName`, `OPENSEARCH_SSL` defaults false. Its `FullQBitsTest` casts the config back out of `QInstance.getQBits()`.
+- Voyage `pom.xml:249-253` (origin/develop): pins `qbit-quick-search:0.2.0` from GitHub Packages and produces the QBit in `OrderAppMetaDataProvider.produceQuickSearchQBit` for `crmContact` and `crmCompany` when `OPENSEARCH_HOST` is set. Report 04's statement that Voyage has no reference was based on a stale local checkout and is wrong.
+
+## 7. Missing tests (consolidated)
+
+Reports 04 and 05 list them by class and scenario. The themes: a transaction test against H2 through `qqq-backend-module-rdbms` (insert, roll back, assert no publish); inverted customizer tests (publisher throws, records carry a warning or a failed-event row exists); keyset paging assertions on every source query; watermark captured before querying with an overlap; stale-write rejection once versioning lands; "stella" does not match "stone"; integer and date searchable fields do not break search; annotation tables have a record label; full-reindex FAILED run has an error message; `OpenSearchException` wrapped; limit and window clamps; disabled rows skipped everywhere; unique key on `tableName`; one failing table fails the basepull step; produce twice closes the previous client; recordLabelFormat with a null numeric; PVS label indexing; permission denial; string primary keys through reconcile; OpenSearch down during a host insert; pre-existing index with a different mapping; two tables sharing a field name with different types; image matrix 2.19.6 and 3.9.0.

docs/initiatives/quick-search-1.0.md

@@ -0,0 +1,174 @@
+---
+tracker_key: "QRun-IO/qbit-quick-search#12"
+project: qbit-quick-search
+status: released-rc
+updated: 2026-10-09
+---
+
+# Initiative: QBit Quick Search 1.0 on QQQ 4.1
+
+> Stakeholder-readable PRD. This markdown is canonical; the tracker mirrors it. Scope claims are grounded in the code at `develop` (`000e5bb`) and in QQQ `v4.1.0-RC.1`, and in the deep-dive audit of 2026-10-04 (`docs/audits/2026-10-04/README.md`, blockers B1 to B25).
+
+## Status
+
+`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). `1.0.0-RC.2` followed on 2026-10-06 with the review follow-ups #13 to #17. Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
+
+## Executive Summary
+
+QBit Quick Search gives QQQ applications OpenSearch-backed full-text search across tables. It is at 0.2.1-SNAPSHOT, published publicly as 0.2.0, pinned by Voyage and bundled by `qqq-all`. The July 2026 audit found the index could not be trusted to converge with the source database. PR #7 (merged 2026-09-26) fixed update clobbering, silent bulk failures and added a reconcile process. The 2026-10-04 deep-dive confirmed the QBit builds and passes all tests on QQQ 4.1.0-RC.1 unchanged, and found 25 remaining blockers across trust, security, search quality, platform compatibility and operability.
+
+1.0 means three things. First, a standard QQQ 4.1 QBit: built on the 4.1 build parent, registered through `QBitMetaDataProducer`, scheduled through QQQ's scheduler, composable with other QBits on the same tables, fail-fast at startup. Second, a trustworthy index: no uncommitted or lost writes, no silent failures, deterministic batch jobs, a reindex that keeps serving results, and search that honours QQQ permissions. Third, a credible platform story: current OpenSearch client, self-managed 2.x and 3.x plus Amazon OpenSearch Service managed domains, TLS trust configuration, secrets by reference, and an authentication model that covers basic auth and, if decision D6 says so, AWS IAM SigV4.
+
+The 4.1 release train does not wait for 1.0. A compatibility-only 0.3.0 rides the proposed `qbit-build-parent` 2.1.0 chain (qqq#921). 1.0 follows on the same parent once the epics below land.
+
+## Business Objective
+
+Ship a search QBit that QQQ adopters (qqq-all demo users, Voyage, future Business Platform tenants) can run in production without a reconciliation babysitter, against the OpenSearch deployments they actually have, and that the Next UI can expose directly.
+
+## Problem Statement
+
+Current state at `develop` as of 2026-10-04:
+
+| Area | State | Evidence |
+|---|---|---|
+| Build | Parent `qbit-build-parent:2.0.0` (qqq 4.0.0). Opt-in `-Pqqq-snapshot` profile. | `pom.xml` |
+| 4.1 compatibility | 233 unit tests pass on 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1; `mvn verify` on RC.1 with Docker passes all 12 integration tests. No source change needed. | surefire and failsafe reports |
+| Coverage gate | JaCoCo `check` is skipped (no `prepare-agent`), so the 0.70 / 0.90 thresholds never run. | `mvn verify` output |
+| Open PRs | #8 (orb 0.6.8, tag-only GA publish, Apache checkstyle, AssertJ 3.27.7): CI green, awaiting review. #1 superseded by #8. | GitHub |
+| Release chain | QQQ 4.1.0-RC.1 published 2026-10-02. Parent 2.1.0-RC.1 and quick-search 0.3.0-RC.1 proposed in qqq#921; qqq-all BOM pins 0.2.1-SNAPSHOT. | qqq#921, qbit-bom#4, qqq-all#10 |
+| Index trust | Real-time indexing runs inside the host transaction before commit (B1); failures are swallowed with no record (B2); basepull pages without order-by (B4) and loses rows modified during a run (B5); basepull and reconcile are never scheduled (B6); full reindex wipes first (B10); no unique key on `quickSearchIndex.tableName` (B19). | audit B1, B2, B4, B5, B6, B10, B19 |
+| Search quality | Query terms are edge-ngrammed because no search analyzer is set, and multi-match defaults to OR (B7); shared dynamic `fieldValues` object breaks on mixed field types and non-lenient boosts (B8); record labels null for annotation tables and PVS ids indexed instead of labels (B9); unbounded paging and term length (B17). | audit B7, B8, B9, B17 |
+| Security | Search has no permission or session checks (B3); no permission rules on processes, tables or app (B16); credentials allowed over plain HTTP and no TLS trust configuration (B11); password stored on serializable metadata (B12). | audit B3, B11, B12, B16 |
+| Platform | opensearch-java 2.10.0 is the end-of-life 2.x line (B13); IAM SigV4 and Serverless unsupported; no timeouts (B15); integration tests pin OpenSearch 2.11.0 only (B22). | audit section 3 |
+| QBit contract | Plain producer, no `QBitMetaData` (qqq-all hand-registers with a hard-coded version), no `sourceQBitName`, static context, `withCustomizer` throws on occupied slots, best-effort startup (B14). | `QuickSearchQBitProducer`, qqq-all `DemoQBits.java:161` |
+| Core overlap | QQQ 4.1 `RecordSearchAction`, `QTableMetaData.searchFields` and `POST /qqq/v1/search` back the Next UI with permissions and no provider SPI; Quick Search has no UI surface and cannot yet satisfy `tableNames` or `limitPerTable`. | qqq commit `7dd36c8ad`; audit section 5 |
+| Docs | README shows a `0.2.0-SNAPSHOT` dependency, promises a scheduled basepull, admin-UI per-table reindex, "field label" prefixes, enforced coverage and qbit-crm/wms integrations, none of which hold. | `README.md` |
+
+Doing nothing leaves qqq-all and Voyage shipping a search feature whose basepull never runs, whose index can contain rolled-back and lost writes, whose results leak across tables regardless of permissions, and whose reindex blanks search.
+
+## Vision / Desired Future State
+
+A host adds the dependency, configures the QBit like any other 4.1 QBit with secrets by reference, points it at a self-managed or AWS-managed OpenSearch cluster over TLS, and gets: real-time indexing that reflects committed data and survives slot collisions, a scheduled catch-up and reconcile with a replayed failed-event log, a reindex that keeps serving results, a search API that returns only what the session may read, and a Next UI search box backed by OpenSearch when the QBit is present.
+
+## Scope
+
+### In Scope
+
+- Released 4.1 build parent; CI matrix on the 4.1 GA line plus the next snapshot; Docker-backed integration tests on OpenSearch 2.19.x and 3.x; JaCoCo gate actually enforced.
+- Standard QBit contract: `QBitMetaDataProducer`, self-registered `QBitMetaData` with the real artifact version, namespace support, `sourceQBitName`, fail-fast validation including field existence, explicit permission rules, instance-scoped runtime state, composable customizers.
+- Index trust: post-commit indexing or a documented contract, failed-event capture and replay, keyset paging everywhere, start-anchored basepull watermark with overlap, scheduling through `QScheduleMetaData`, blackout-free full reindex, unique key on the index table, orphan and drift detection, external document versioning.
+- Search quality and safety: search analyzer, AND semantics, typed-safe `fieldValues` mapping, display values and labels, permission-aware results, bounded paging and term length, wrapped OpenSearch exceptions, `tableNames` and `limitPerTable` inputs, `tableLabel` and typed `recordId` outputs.
+- Connection security: opensearch-java 3.x, `opensearchUrl`, TLS trust block, `authMode` (`NONE`, `BASIC`, and `AWS_SIGV4` per D6), secrets by `${env.}` reference, redaction, timeouts and pool sizing, transport customizer SPI, client lifecycle.
+- Operability and docs: process input screens, admin table status fields, run-history purge, health method, truthful README with a supported-platforms statement and least-privilege roles, configuration reference, migration notes from 0.x, CHANGELOG.
+
+### Out of Scope
+
+- Asynchronous publisher implementations (RabbitMQ, Kafka, ESB). The `IndexEventPublisher` SPI stays and gets `occurredAt` and a version so an async implementation can be correct, but it is marked experimental in 1.0.
+- Amazon OpenSearch Serverless (needs SigV4 `aoss`, and lacks `_delete_by_query` and `_refresh`). Legacy Elasticsearch endpoints and OpenSearch 1.x.
+- Bearer, JWT, API-key and mTLS as first-class `authMode` values (available through the transport customizer SPI).
+- Record-level security beyond the `QueryAction` post-filter (D3). Index-time security keys.
+- Multi-index layouts, composite primary keys, synonyms, fuzzy matching, phrase queries, language analyzers, sort options.
+- `com.kingsrook` to `io.qrun` groupId migration (ADR-0007). qbit-crm / qbit-wms companion integrations.
+- Model-assisted table routing or re-ranking (TypeSafe Jev); candidates in `docs/reference/typesafe-jev-candidates.md` for 1.x.
+
+## Success Criteria
+
+- `com.kingsrook.qbits:qbit-quick-search:1.0.0` on Maven Central, parent importing the released qqq 4.1 BOM, Apache-2.0 metadata, opensearch-java 3.x.
+- CI runs the Testcontainers suite on OpenSearch 2.19.x and 3.x latest minor with no silent skips, on 4.1 GA and the next snapshot; JaCoCo thresholds enforced.
+- qqq-all `full` profile and Voyage pin 1.0.0; qqq-all's smoke finds a seeded customer through the Next UI search box (or through the API if D1's SPI slips).
+- A session without read permission on a table gets zero results from that table; a session with a record security lock sees only its rows. Both covered by tests.
+- A rolled-back insert produces no document (or, under the documented alternative, is removed by the next scheduled reconcile). Covered by an H2 transaction test.
+- Full reindex of a seeded table returns non-empty results at every point during the run. Covered by an integration test.
+- Basepull and reconcile fire on schedule in a host with a configured scheduler; a row modified during a basepull run is indexed by the next run. Covered by tests.
+- "stella" does not match "stone"; a table with an integer and a date searchable field searches without error. Covered by integration tests.
+- Basic auth over TLS against a cluster with a private CA works with the `tls` block; credentials over plain HTTP are rejected unless explicitly allowed. Covered by a secured Testcontainers test.
+- Every blocker B1 to B25 in the 2026-10-04 audit is closed or explicitly deferred with a reason in this doc.
+
+## Key Stakeholders
+
+| Role | Person | Responsibility |
+|---|---|---|
+| Sponsor / Product Owner | James Maes | Decisions D1 to D11, release approval |
+| Engineering Lead | James Maes (with Claude Code) | Build, review, release |
+| Downstream consumers | qqq-all (QRun-IO/qqq-all#10); Voyage (pins 0.2.0 from GitHub Packages, `OrderAppMetaDataProvider.produceQuickSearchQBit`) | Pin and verify 1.0.0 |
+
+## Initiative-level Constraints
+
+- Parent `qbit-build-parent` is the only source of the qqq version (ADR-0007). No child BOM imports outside the opt-in snapshot profile. httpclient5 and Jackson stay BOM-managed.
+- Apache-2.0 everywhere; Central artifacts are immutable, so metadata fixes ship as new versions.
+- 3-space indentation, flower-box Javadoc, fluent `withX()` setters, wrapper types (CODE_STYLE.md).
+- JaCoCo gates stay at 0.70 instruction / 0.90 class, and must actually run.
+- Source compatibility for the fluent config setters and `QuickSearchAction` input/output used by qqq-all and Voyage (`withOpensearchHost/Port/UseSsl/Username/Password`, `withSearchableTable`, `withSearchableEntityClasses`, `QuickSearchQBitProducer().withConfig().produce(qInstance)`). New fields are additive; deprecated fields keep working through 1.x.
+- No secrets in metadata objects, logs, or the frontend `environmentValues` (never name an OpenSearch secret `QQQ_ENV_*`).
+- Any mapping change (search analyzer, `fieldValues` templates) lands before the first 1.0 index is created, with an index-version marker so an existing 0.x index is detected and rebuilt rather than reused.
+
+## Assumptions
+
+- QQQ 4.1.0 GA and `qbit-build-parent` 2.1.0 publish before the 1.0 release; work proceeds on `4.1.0-RC.1` via the snapshot profile.
+- OpenSearch 2.19.x and 3.x are the supported server lines; opensearch-java 3.10.0 is the client (its only compile break here is `.refresh(true)` to `.refresh(Refresh.True)`).
+- Hosts that want scheduling already configure a QQQ scheduler; the QBit does not provide one.
+- QQQ 4.1 provides `RecordChangeListenerInterface`, `QBackendTransaction.addAfterCommitCallback` and `QRuntimeServiceInterface`, none of which exist in 4.0.0 (verified by tag). 1.0 uses them and therefore requires qqq 4.1 (D10).
+
+## Risks
+
+| Risk | Mitigation |
+|---|---|
+| Core `RecordSearchAction` has no SPI, so Next UI integration needs a qqq core change | Raise the SPI as a qqq issue early (D1); 1.0 can ship API-only, UI integration in 1.1 |
+| Adopting 4.1-only APIs drops 4.0 hosts | Voyage and qqq-all both move to 4.1; 0.3.0 remains the last 4.0-compatible line (D10) |
+| After-commit publishing depends on the backend transaction honouring the callback contract | RDBMS does at the 4.1 tag; verify Mongo and other backends in the integration matrix; flush immediately when no transaction is passed; scheduled reconcile is the backstop |
+| Removing the static context changes how custom publishers and process steps obtain the client | Keep static accessors as deprecated delegates to the instance-scoped holder for one minor version |
+| Permission post-filtering reduces page sizes unpredictably | Over-fetch (limit times two, capped) and filter down; document that `totalHits` is an upper bound |
+| Alias-swap reindex needs index-create and alias privileges on the cluster | Fall back to in-place reconcile when the configured user lacks the privilege; document required roles |
+| opensearch-java 3.x bump changes transport behaviour against 2.x servers (HTTP/2 negotiation, URL encoding) | CI matrix on 2.19.6 and 3.9.0; keep the content-compression workaround until proven unnecessary |
+| SigV4 pulls AWS SDK v2 artifacts into a QBit | Declare them `optional`; load the AWS transport reflectively or in a separate class only when `authMode=AWS_SIGV4` |
+
+## Dependencies
+
+- qqq 4.1.0 GA (qqq#798, #921), `qbit-build-parent` 2.1.0 (qbit-bom#4), qqq-orb 0.6.8 (PR #8).
+- CircleCI executor with Docker for the integration suite (orb capability to confirm).
+- qqq core change for a record-search provider SPI (new issue, D1).
+- Optional: `software.amazon.awssdk:aws-crt-client` and `auth` for SigV4 (D6).
+
+## Proposed Epics
+
+Estimates are engineer-days with AI-augmented throughput. E1 is foundational and gets broken into stories first. Blocker IDs reference `docs/audits/2026-10-04/README.md`.
+
+### Epic 1: 4.1 platform and QBit contract (foundational, ~7 days)
+Purpose: make the QBit a standard 4.1 citizen, following the ordered migration plan in audit report 01. `QBitMetaDataProducer<QuickSearchQBitConfig>` with `getQBitMetaData()` (groupId `com.kingsrook.qbits`, artifactId `quick-search` which qqq-all asserts, version from a Maven-filtered resource, namespace, config); tables, processes and app as package-scanned component producers or `@QMetaDataProducingEntity`; registration and bootstrap in `postProduceActions`; `withConfig()` and a `produce(QInstance)` override that adds to the instance kept as deprecated shims so qqq-all and Voyage compile unchanged; `isEnabled()` false when the config is null; `getDefaultBackendNameForTables()` and a settable `tableMetaDataCustomizer`; namespace as default `tableNamePrefix`. Explicit `QPermissionRules` with config override (B16). Null-safe list-based validation covering backend, field existence, timestamp field and OpenSearch naming rules, plus a `QInstanceValidatorPluginInterface` for late checks (B24). `QuickSearchRuntimeService implements QRuntimeServiceInterface` owning connect, `ensureIndexExists` and close, with `startupMode` per D11 (B14); config resolved from `SourceQBitAware.getSourceQBitConfig()` in steps and the listener, `QuickSearchQBitContext` kept as a deprecated facade for one release. Unique key on `quickSearchIndex.tableName` (B19); split `enableScheduledProcesses` into basepull and maintenance flags with the old name as a deprecated alias (B25). Build: JaCoCo `prepare-agent` with an explicit version, drop the JUnit 5.10.0 pins for the parent's `junit-bom` 6.0.1, parent 2.1.x re-pin when released; CI matrix (GA plus next snapshot), Docker-backed integration tests on 2.19.x and 3.x (B22).
+
+### Epic 2: Index trustworthiness (~7 days)
+Purpose: finish the convergence story. Replace the three customizers with one `QuickSearchRecordChangeListener implements RecordChangeListenerInterface` (`appliesTo` only discovered tables; UPDATE keeps the full-record re-read through `event.getTransaction()`), registered with `qInstance.withRecordChangeListener`, publishing through `QBackendTransaction.addAfterCommitCallback` when a transaction is present and immediately otherwise (B1, D7); the listener cannot fail the write, so record failures in a `quickSearchFailedEvent` table that the scheduled run replays, with bounded retry on bulk 429/5xx and failure counts on `quickSearchIndex` (B2, D8); keyset paging in basepull and full reindex reusing the reconcile loop (B4); start-anchored watermark with configurable overlap and greater-than-or-equal (B5); `QScheduleMetaData` on basepull and reconcile with `schedulerName` and per-table interval (B6); full reindex into a fresh index with alias swap and in-place fallback, `conflicts(Proceed)`, error message on FAILED runs (B10); aggregate per-table failures into the step result (B20); honour `enabled` in every path (B21); orphan sweep for removed or renamed tables, config and mapping drift detection with a `NEEDS_REINDEX` status and an index-version marker (B23); external document versioning from `modifyDate`; run-history purge; `IndexEvent.occurredAt` and version.
+
+### Epic 3: Search quality and permission-aware search (~5 days)
+Purpose: make results correct and safe. Standard+lowercase `search_analyzer`, `operator=AND` or `minimum_should_match`, boosts that apply to prefix matches (B7); `dynamic_templates` for `fieldValues.*`, `date_detection: false`, stringified values, `lenient(true)` (B8); display values and PVS labels at index time, `QValueFormatter.formatRecordLabel` fallback, per-field `maxLength` (B9); table-level READ permission filter mirroring core, `terms` filter on `sourceTable`, `QueryAction` post-filter by primary key for record locks per D3, hidden and masked fields excluded at index time (B3); bounded `limit`, window and term length, `track_total_hits` with a lower-bound flag (B17); `OpenSearchException` wrapped in `QException` (B18); disabled tables excluded from search (B21).
+
+### Epic 4: Search surface and UI integration (~4 days, partly in qqq core)
+Purpose: make search usable from the UI. Input parity with core: `tableNames`, `limitPerTable`, `inputSource`; output parity: `tableLabel`, typed `recordId`; default the field list from `QTableMetaData.searchFields` when present; propose and implement a provider SPI in qqq core so `RecordSearchAction` delegates to a registered provider (D1); ship `QuickSearchRecordSearchProvider`; keep `QuickSearchAction` as the richer backend API (scores, highlights, pagination).
+
+### Epic 5: Connection security and authentication (~5 days)
+Purpose: run against real clusters safely. opensearch-java 3.10.0 bump with the `Refresh.True` fix and content-compression workaround kept (B13); `opensearchUrl` and optional host list, https default; `tls` block (CA path, trust store, key store for mTLS, hostname verification, guarded `insecureSkipVerify`) (B11); `authMode` enum `NONE`, `BASIC`, `AWS_SIGV4` with `awsRegion`, `awsServiceName`, `awsAssumeRoleArn` and optional AWS SDK v2 dependencies per D6; `transportCustomizer` `QCodeReference` SPI for bearer, API key and mTLS; `${env.}` interpretation of secrets via `QMetaDataVariableInterpreter` at client construction, `@JsonIgnore` secret getters, redacting `toString()`, `transient` publisher (B12); `allowPlaintextCredentials` gate; `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `User-Agent`, byte-capped bulk batches (B15); validation rules and the deprecation path for the old setters; a secured Testcontainers test (`withSecurityEnabled`).
+
+### Epic 6: Operations, docs and release (~4 days)
+Purpose: ship 1.0 cleanly. Process input screens with a table PVS and summary screen (B25); admin table fields `lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`, read-only tracking fields, PVS from run to index, sections and labels; health method surfaced in the admin table; README rewrite (truthful feature list, dependency snippet, supported-platforms statement, least-privilege roles for the security plugin and AWS FGAC, required cluster privileges for alias swap, `QQQ_ENV_*` warning, SPI stability statement); configuration reference; 0.x to 1.0 migration notes including the forced reindex; CHANGELOG and release notes; 0.3.0 compatibility release on parent 2.1.0 first, then 1.0.0; qqq-all and Voyage pin bumps.
+
+## Decisions for the sponsor
+
+Each has a recommendation. Resolve one at a time; answers get folded into tickets as facts.
+
+| ID | Question | Recommendation |
+|---|---|---|
+| D1 | Integrate with core `RecordSearchAction` via a new qqq SPI, or keep Quick Search API-only? | Propose the SPI in qqq now; ship 1.0 API-only if it slips, UI in 1.1 |
+| D2 | Version plan: does the qqq#921 chain ship 0.3.0 (compat only) and 1.0 follows, or does 4.1 wait for 1.0? | 0.3.0 on the RC chain, 1.0 after E1 to E6; do not gate 4.1 on 1.0 |
+| D3 | Permission depth for 1.0: table-level read permission only, or record-level via `QueryAction` post-filter? | Table-level plus `QueryAction` post-filter by primary key (reuses locks); no indexed security keys in 1.0 |
+| D4 | Scheduling default: schedule automatically when a scheduler exists, or require `schedulerName`? | Require `schedulerName` when basepull is enabled; validation error otherwise so the README promise is enforced |
+| D5 | Full reindex strategy: alias swap (needs cluster privileges) or in-place reconcile semantics only? | Alias swap with documented privileges, in-place fallback when `createIndex` is denied |
+| D6 | AWS IAM SigV4 in 1.0 (optional AWS SDK v2 dependencies, `authMode=AWS_SIGV4`), or declare unsupported and ship basic-auth FGAC only? | Include it: the transport ships inside opensearch-java, the SDK artifacts are optional, and IAM is the default posture for AWS domains; Serverless stays out |
+| D7 | Pre-commit indexing (B1): use 4.1's `RecordChangeListenerInterface` plus `QBackendTransaction.addAfterCommitCallback`, or keep customizers and document pre-commit semantics with scheduled reconcile as the corrector? | Use the 4.1 listener and after-commit callback (reference: `qqq-esb` `EsbRecordChangeListener`); flush immediately when no transaction is passed; keep reconcile scheduled as the backstop; this makes 1.0 4.1-only (D10) |
+| D8 | Real-time failure contract (B2): surface as QQQ record warnings plus a failed-event table (host write succeeds), or fail the host write when OpenSearch is down? | Warnings plus failed-event table; never fail the host write for a search index |
+| D9 | Supported platforms statement for 1.0: self-managed 2.19.x and 3.x plus AWS managed domains (FGAC basic auth, and IAM if D6); Serverless, Elasticsearch and OpenSearch 1.x unsupported? | Yes, as stated; revisit Serverless in 1.x once `_delete_by_query` is removed from the design |
+| D10 | Does 1.0 require qqq 4.1 (adopting `RecordChangeListenerInterface`, after-commit callbacks and `QRuntimeServiceInterface`), or keep a 4.0 fallback path with customizers and `MultiCustomizer`? | 4.1-only; 0.3.0 is the last 4.0 line; delete the customizers in 1.0 rather than carrying two real-time paths |
+| D11 | `startupMode` default: `FAIL_FAST` (host refuses to boot when OpenSearch is unreachable) or `DEGRADED` (current behaviour, which qqq-all's `FullQBitsTest` relies on with OpenSearch at `127.0.0.1:1`)? | `FAIL_FAST` by default for production safety; qqq-all's full profile sets `DEGRADED` explicitly in its follow-up PR, and `DEGRADED` surfaces a health notice |
+
+## Definition of Done
+
+All six epics complete; success criteria met; 1.0.0 published on Central and pinned by qqq-all and Voyage; README, configuration reference, migration notes and CHANGELOG accurate; every blocker in `docs/audits/2026-10-04/README.md` closed or deferred with a reason recorded here and in the second-brain project note.

docs/reference/typesafe-jev-candidates.md

@@ -0,0 +1,36 @@
+# TypeSafe (Jev) candidates for Quick Search
+
+Captured 2026-10-04. Not in 1.0 scope (see `docs/initiatives/quick-search-1.0.md`); a possible 1.x epic or companion extension. Nothing here is implemented.
+
+Jev is TypeSafe's System One model: it returns typed answers (Choice, Score, Noul probability) over a `state` payload via `POST https://api.typesafe.ai/v1/systemone`. There is no Java SDK; the HTTP API is a single JSON POST, and the project already has httpclient5 and Jackson. Docs: https://docs.typesafe.ai/llms.txt
+
+## How the project judges things today
+
+| Concern | Current mechanism | Code |
+|---|---|---|
+| Ranking | BM25 over edge-ngram `searchableText` plus per-field `should` boosts with hand-set weights | `QuickSearchOpenSearchClient.search`, `buildFieldBoostQueries` |
+| Routing | Optional `tableName` filter only; no logic decides which table a query is about | `QuickSearchInput`, `QuickSearchAction` |
+| Query classification | Lowercase and whitespace collapse | `IndexingUtils.normalizeSearchText` |
+| Index event routing | Deterministic INDEX / DELETE; no semantic step, no Jev fit | `IndexEventAction`, customizers, `ReconcileIndexStep` |
+
+## Candidates, strongest first
+
+1. **Table routing (one Choice).** State: the query plus a catalog of indexed tables (label, searchable field names). Question: which kind of record is this query looking for, one option per table plus "no specific table". Use the probability distribution as per-table boosts on `sourceTable`, or feed it to core `RecordSearchAction.tableNames` / `limitPerTable` once the Epic 4 provider exists. Confidence-gate: narrow only when confidence is high, otherwise search everything. Tiny state, cheap, fast. New capability, not a simplification.
+2. **Re-ranking the OpenSearch shortlist (one Noul per candidate).** Over-fetch about three pages, ask "is this record what the query is asking for" with the query and each candidate's `recordLabel` and `fieldValues` in state (all candidates in one request as parallel questions), sort by probability, cut to `limit`. Pattern: https://docs.typesafe.ai/cookbooks/rerank_typesafe.md. Costs: a network hop on the search hot path (not for per-keystroke typeahead), and record data leaving the cluster (needs enterprise zero-data-retention terms for CRM or health-adjacent data). Only as an opt-in reranker behind a flag, explicit searches, first page.
+3. **Highlight selection.** Choice over highlight fragment ids ("which snippet best shows why this matched") instead of joining all fragments with "...". Small win, same egress concern as item 2.
+4. **Offline weight suggestion.** Dev-time scoring of sample records per field for how identifying each value is, to suggest `weight`. Not runtime.
+
+## Rejected
+
+- Query-shape classification (email, phone, order number, name, free text) to switch exact keyword vs ngram matching. Regexes do this deterministically; Jev's own jaggedness notes say to keep literal pattern work in code.
+
+## Integration notes
+
+- Put any client behind an interface (`SearchRouter`, `ResultReranker`) so the core QBit stays OpenSearch-only and the integration is optional. Dependency or build changes need owner approval.
+- Jev 1.13 limits: 80 requests per second, 100K tokens per second, 32k tokens of state per question. Fine for routing; tight for reranking under load.
+- Pin a versioned model id (`jev-1.13.0`) once confidence thresholds are tuned; aliases move.
+- API key stays server-side in host configuration, never in the QBit's metadata.
+
+## Recommendation
+
+Prototype item 1 only, as a confidence-gated boost, and measure against the seeded qqq-all customer data before considering item 2.

mvnw

@@ -0,0 +1,295 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.3.4
+#
+# Optional ENV vars
+# -----------------
+#   JAVA_HOME - location of a JDK home dir, required when download maven via java source
+#   MVNW_REPOURL - repo url base for downloading maven distribution
+#   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+#   MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
+# ----------------------------------------------------------------------------
+
+set -euf
+[ "${MVNW_VERBOSE-}" != debug ] || set -x
+
+# OS specific support.
+native_path() { printf %s\\n "$1"; }
+case "$(uname)" in
+CYGWIN* | MINGW*)
+  [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
+  native_path() { cygpath --path --windows "$1"; }
+  ;;
+esac
+
+# set JAVACMD and JAVACCMD
+set_java_home() {
+  # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
+  if [ -n "${JAVA_HOME-}" ]; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+      JAVACCMD="$JAVA_HOME/jre/sh/javac"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+      JAVACCMD="$JAVA_HOME/bin/javac"
+
+      if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
+        echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
+        echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
+        return 1
+      fi
+    fi
+  else
+    JAVACMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v java
+    )" || :
+    JAVACCMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v javac
+    )" || :
+
+    if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
+      echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
+      return 1
+    fi
+  fi
+}
+
+# hash string like Java String::hashCode
+hash_string() {
+  str="${1:-}" h=0
+  while [ -n "$str" ]; do
+    char="${str%"${str#?}"}"
+    h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
+    str="${str#?}"
+  done
+  printf %x\\n $h
+}
+
+verbose() { :; }
+[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
+
+die() {
+  printf %s\\n "$1" >&2
+  exit 1
+}
+
+trim() {
+  # MWRAPPER-139:
+  #   Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
+  #   Needed for removing poorly interpreted newline sequences when running in more
+  #   exotic environments such as mingw bash on Windows.
+  printf "%s" "${1}" | tr -d '[:space:]'
+}
+
+scriptDir="$(dirname "$0")"
+scriptName="$(basename "$0")"
+
+# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
+while IFS="=" read -r key value; do
+  case "${key-}" in
+  distributionUrl) distributionUrl=$(trim "${value-}") ;;
+  distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
+  esac
+done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
+[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+
+case "${distributionUrl##*/}" in
+maven-mvnd-*bin.*)
+  MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
+  case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
+  *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
+  :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
+  :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
+  :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
+  *)
+    echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
+    distributionPlatform=linux-amd64
+    ;;
+  esac
+  distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
+  ;;
+maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
+*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
+esac
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
+distributionUrlName="${distributionUrl##*/}"
+distributionUrlNameMain="${distributionUrlName%.*}"
+distributionUrlNameMain="${distributionUrlNameMain%-bin}"
+MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
+MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
+
+exec_maven() {
+  unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
+  exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
+}
+
+if [ -d "$MAVEN_HOME" ]; then
+  verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  exec_maven "$@"
+fi
+
+case "${distributionUrl-}" in
+*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
+*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
+esac
+
+# prepare tmp dir
+if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
+  clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
+  trap clean HUP INT TERM EXIT
+else
+  die "cannot create temp dir"
+fi
+
+mkdir -p -- "${MAVEN_HOME%/*}"
+
+# Download and Install Apache Maven
+verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+verbose "Downloading from: $distributionUrl"
+verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+# select .zip or .tar.gz
+if ! command -v unzip >/dev/null; then
+  distributionUrl="${distributionUrl%.zip}.tar.gz"
+  distributionUrlName="${distributionUrl##*/}"
+fi
+
+# verbose opt
+__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
+[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
+
+# normalize http auth
+case "${MVNW_PASSWORD:+has-password}" in
+'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+esac
+
+if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
+  verbose "Found wget ... using wget"
+  wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
+elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
+  verbose "Found curl ... using curl"
+  curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
+elif set_java_home; then
+  verbose "Falling back to use Java to download"
+  javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
+  targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
+  cat >"$javaSource" <<-END
+	public class Downloader extends java.net.Authenticator
+	{
+	  protected java.net.PasswordAuthentication getPasswordAuthentication()
+	  {
+	    return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
+	  }
+	  public static void main( String[] args ) throws Exception
+	  {
+	    setDefault( new Downloader() );
+	    java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
+	  }
+	}
+	END
+  # For Cygwin/MinGW, switch paths to Windows format before running javac and java
+  verbose " - Compiling Downloader.java ..."
+  "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
+  verbose " - Running Downloader.java ..."
+  "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
+fi
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+if [ -n "${distributionSha256Sum-}" ]; then
+  distributionSha256Result=false
+  if [ "$MVN_CMD" = mvnd.sh ]; then
+    echo "Checksum validation is not supported for maven-mvnd." >&2
+    echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  elif command -v sha256sum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  elif command -v shasum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  else
+    echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
+    echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  fi
+  if [ $distributionSha256Result = false ]; then
+    echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
+    echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
+    exit 1
+  fi
+fi
+
+# unzip and move
+if command -v unzip >/dev/null; then
+  unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
+else
+  tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
+fi
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+actualDistributionDir=""
+
+# First try the expected directory name (for regular distributions)
+if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
+  if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
+    actualDistributionDir="$distributionUrlNameMain"
+  fi
+fi
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if [ -z "$actualDistributionDir" ]; then
+  # enable globbing to iterate over items
+  set +f
+  for dir in "$TMP_DOWNLOAD_DIR"/*; do
+    if [ -d "$dir" ]; then
+      if [ -f "$dir/bin/$MVN_CMD" ]; then
+        actualDistributionDir="$(basename "$dir")"
+        break
+      fi
+    fi
+  done
+  set -f
+fi
+
+if [ -z "$actualDistributionDir" ]; then
+  verbose "Contents of $TMP_DOWNLOAD_DIR:"
+  verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
+  die "Could not find Maven distribution directory in extracted archive"
+fi
+
+verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
+mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
+
+clean || :
+exec_maven "$@"

mvnw.cmd

@@ -0,0 +1,189 @@
+<# : batch portion
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Apache Maven Wrapper startup batch script, version 3.3.4
+@REM
+@REM Optional ENV vars
+@REM   MVNW_REPOURL - repo url base for downloading maven distribution
+@REM   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+@REM   MVNW_VERBOSE - true: enable verbose log; others: silence the output
+@REM ----------------------------------------------------------------------------
+
+@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
+@SET __MVNW_CMD__=
+@SET __MVNW_ERROR__=
+@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
+@SET PSModulePath=
+@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
+  IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
+)
+@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
+@SET __MVNW_PSMODULEP_SAVE=
+@SET __MVNW_ARG0_NAME__=
+@SET MVNW_USERNAME=
+@SET MVNW_PASSWORD=
+@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
+@echo Cannot start maven from wrapper >&2 && exit /b 1
+@GOTO :EOF
+: end batch / begin powershell #>
+
+$ErrorActionPreference = "Stop"
+if ($env:MVNW_VERBOSE -eq "true") {
+  $VerbosePreference = "Continue"
+}
+
+# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
+$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
+if (!$distributionUrl) {
+  Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+}
+
+switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
+  "maven-mvnd-*" {
+    $USE_MVND = $true
+    $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
+    $MVN_CMD = "mvnd.cmd"
+    break
+  }
+  default {
+    $USE_MVND = $false
+    $MVN_CMD = $script -replace '^mvnw','mvn'
+    break
+  }
+}
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+if ($env:MVNW_REPOURL) {
+  $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
+  $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
+}
+$distributionUrlName = $distributionUrl -replace '^.*/',''
+$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
+
+$MAVEN_M2_PATH = "$HOME/.m2"
+if ($env:MAVEN_USER_HOME) {
+  $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
+}
+
+if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
+    New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
+}
+
+$MAVEN_WRAPPER_DISTS = $null
+if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
+  $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
+} else {
+  $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
+}
+
+$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
+$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
+$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
+
+if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
+  Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+  exit $?
+}
+
+if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
+  Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
+}
+
+# prepare tmp dir
+$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
+$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
+$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
+trap {
+  if ($TMP_DOWNLOAD_DIR.Exists) {
+    try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+    catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+  }
+}
+
+New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
+
+# Download and Install Apache Maven
+Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+Write-Verbose "Downloading from: $distributionUrl"
+Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+$webclient = New-Object System.Net.WebClient
+if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
+  $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
+}
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
+if ($distributionSha256Sum) {
+  if ($USE_MVND) {
+    Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
+  }
+  Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
+  if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
+    Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
+  }
+}
+
+# unzip and move
+Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+$actualDistributionDir = ""
+
+# First try the expected directory name (for regular distributions)
+$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
+$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
+if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
+  $actualDistributionDir = $distributionUrlNameMain
+}
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if (!$actualDistributionDir) {
+  Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
+    $testPath = Join-Path $_.FullName "bin/$MVN_CMD"
+    if (Test-Path -Path $testPath -PathType Leaf) {
+      $actualDistributionDir = $_.Name
+    }
+  }
+}
+
+if (!$actualDistributionDir) {
+  Write-Error "Could not find Maven distribution directory in extracted archive"
+}
+
+Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
+try {
+  Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
+} catch {
+  if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
+    Write-Error "fail to move MAVEN_HOME"
+  }
+} finally {
+  try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+  catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+}
+
+Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"

pom.xml

@@ -18,16 +18,19 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
-      <revision>0.2.0</revision>
+      <revision>1.1.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
-      <opensearch.version>2.10.0</opensearch.version>
+      <opensearch.version>3.10.0</opensearch.version>
+      <jacoco.version>0.8.15</jacoco.version>
+      <opensearch.test.image>opensearchproject/opensearch:2.19.6</opensearch.test.image>
+      <aws.sdk.version>2.41.10</aws.sdk.version>
    </properties>
 
    <parent>
       <groupId>com.kingsrook</groupId>
       <artifactId>qbit-build-parent</artifactId>
-      <version>1.6.0</version>
+      <version>2.0.0</version>
    </parent>
 
    <groupId>com.kingsrook.qbits</groupId>
@@ -66,25 +69,6 @@
       <tag>HEAD</tag>
    </scm>
 
-   <dependencyManagement>
-      <dependencies>
-         <dependency>
-            <groupId>org.junit</groupId>
-            <artifactId>junit-bom</artifactId>
-            <version>5.10.0</version>
-            <type>pom</type>
-            <scope>import</scope>
-         </dependency>
-         <dependency>
-            <groupId>com.kingsrook.qqq</groupId>
-            <artifactId>qqq-bom-pom</artifactId>
-            <version>0.40.0</version>
-            <type>pom</type>
-            <scope>import</scope>
-         </dependency>
-      </dependencies>
-   </dependencyManagement>
-
    <dependencies>
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
@@ -97,16 +81,32 @@
          <version>${opensearch.version}</version>
       </dependency>
 
+      <!-- httpclient5 and jackson versions are managed by the qqq BOM, so they -->
+      <!-- stay binary-compatible with the httpcore5 and jackson it brings.       -->
       <dependency>
          <groupId>org.apache.httpcomponents.client5</groupId>
          <artifactId>httpclient5</artifactId>
-         <version>5.3</version>
       </dependency>
 
       <dependency>
          <groupId>com.fasterxml.jackson.datatype</groupId>
          <artifactId>jackson-datatype-jsr310</artifactId>
-         <version>2.21.0</version>
+      </dependency>
+
+      <!-- Only needed for authMode AWS_SIGV4 (Amazon OpenSearch Service with IAM). -->
+      <!-- Optional: hosts that use it declare them; the core modules already come  -->
+      <!-- through qqq-backend-core at the same version.                            -->
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>apache-client</artifactId>
+         <version>${aws.sdk.version}</version>
+         <optional>true</optional>
+      </dependency>
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>sts</artifactId>
+         <version>${aws.sdk.version}</version>
+         <optional>true</optional>
       </dependency>
 
       <dependency>
@@ -125,14 +125,13 @@
       <dependency>
          <groupId>org.junit.jupiter</groupId>
          <artifactId>junit-jupiter</artifactId>
-         <version>5.10.0</version>
          <scope>test</scope>
       </dependency>
 
       <dependency>
          <groupId>org.assertj</groupId>
          <artifactId>assertj-core</artifactId>
-         <version>3.27.3</version>
+         <version>3.27.7</version>
          <scope>test</scope>
       </dependency>
 
@@ -147,18 +146,34 @@
       <dependency>
          <groupId>org.testcontainers</groupId>
          <artifactId>testcontainers</artifactId>
-         <version>1.21.0</version>
+         <version>1.21.4</version>
          <scope>test</scope>
       </dependency>
       <dependency>
          <groupId>org.testcontainers</groupId>
          <artifactId>junit-jupiter</artifactId>
-         <version>1.21.0</version>
+         <version>1.21.4</version>
          <scope>test</scope>
       </dependency>
    </dependencies>
 
    <build>
+      <resources>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>true</filtering>
+            <includes>
+               <include>quick-search.properties</include>
+            </includes>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+            <excludes>
+               <exclude>quick-search.properties</exclude>
+            </excludes>
+         </resource>
+      </resources>
       <plugins>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
@@ -177,6 +192,9 @@
             <artifactId>maven-failsafe-plugin</artifactId>
             <version>3.1.2</version>
             <configuration>
+               <systemPropertyVariables>
+                  <opensearch.test.image>${opensearch.test.image}</opensearch.test.image>
+               </systemPropertyVariables>
                <includes>
                   <include>**/*IntegrationTest.java</include>
                   <include>**/*IT.java</include>
@@ -194,7 +212,21 @@
          <plugin>
             <groupId>org.jacoco</groupId>
             <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.version}</version>
             <executions>
+               <execution>
+                  <id>prepare-agent</id>
+                  <goals>
+                     <goal>prepare-agent</goal>
+                  </goals>
+               </execution>
+               <execution>
+                  <id>report</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>report</goal>
+                  </goals>
+               </execution>
                <execution>
                   <id>check</id>
                   <goals>
@@ -224,4 +256,49 @@
          </plugin>
       </plugins>
    </build>
+
+   <profiles>
+      <!-- 1.0 requires qqq 4.1 APIs (record-change listeners, after-commit        -->
+      <!-- callbacks, runtime services). Until qbit-build-parent 2.1.0 imports the -->
+      <!-- released 4.1 BOM, this profile is active by default and imports          -->
+      <!-- qqq-bom-pom 4.1.0-RC.1 ahead of the parent's 4.0.0 BOM. Override with    -->
+      <!-- -Dqqq.snapshot.version=... ; deactivate with -P!qqq-snapshot. Remove the -->
+      <!-- activation when the parent moves to 4.1 (ADR-0007).                      -->
+      <!-- 1.1: the record-search provider needs the core SPI                       -->
+      <!-- (RecordSearchProviderInterface), first released in qqq 4.1.0-RC.2.       -->
+      <!-- The record developer panel provider needs the developer panel         -->
+      <!-- contract (DeveloperPanelProviderInterface), first in qqq 4.1.0-RC.4.   -->
+      <profile>
+         <id>qqq-snapshot</id>
+         <activation>
+            <activeByDefault>true</activeByDefault>
+         </activation>
+         <properties>
+            <qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
+         </properties>
+         <dependencyManagement>
+            <dependencies>
+               <dependency>
+                  <groupId>com.kingsrook.qqq</groupId>
+                  <artifactId>qqq-bom-pom</artifactId>
+                  <version>${qqq.snapshot.version}</version>
+                  <type>pom</type>
+                  <scope>import</scope>
+               </dependency>
+            </dependencies>
+         </dependencyManagement>
+         <repositories>
+            <repository>
+               <id>sonatype-snapshots</id>
+               <url>https://central.sonatype.com/repository/maven-snapshots/</url>
+               <releases>
+                  <enabled>false</enabled>
+               </releases>
+               <snapshots>
+                  <enabled>true</enabled>
+               </snapshots>
+            </repository>
+         </repositories>
+      </profile>
+   </profiles>
 </project>

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchAuthMode.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+/*******************************************************************************
+ ** How the QBit authenticates to OpenSearch.
+ **
+ ** NONE      - no credentials (local development, security plugin disabled).
+ ** BASIC     - HTTP basic authentication (security plugin internal users, or
+ **             Amazon OpenSearch Service fine-grained access control with an
+ **             internal-database master user).
+ ** AWS_SIGV4 - AWS Signature Version 4 request signing with the default AWS
+ **             credential chain (Amazon OpenSearch Service domains that use
+ **             IAM). Requires the optional AWS SDK v2 dependencies.
+ **
+ ** Bearer tokens, API keys and client certificates are added through an
+ ** {@link com.kingsrook.qbits.quicksearch.opensearch.OpenSearchTransportCustomizer}.
+ *******************************************************************************/
+public enum QuickSearchAuthMode
+{
+   NONE,
+   BASIC,
+   AWS_SIGV4
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContext.java

@@ -13,44 +13,58 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.kingsrook.qbits.quicksearch;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
 
 
 /*******************************************************************************
- ** Static holder for Quick Search QBit runtime state.
+ ** Deprecated static facade over {@link QuickSearchRuntime}.
  **
- ** Provides access to the active configuration, the OpenSearch client, the
- ** index-event publisher, and the list of tables discovered at startup.
+ ** Until 1.0 the QBit kept its live state in these static fields. The state now
+ ** lives on the produced QBit's runtime, resolved from QContext. The getters
+ ** here return an explicitly set static value when one exists (tests and
+ ** legacy hosts), otherwise they delegate to {@link QuickSearchRuntime#get()}.
  **
- ** Call clear() in test @AfterEach / @BeforeEach to reset all state.
+ ** @deprecated resolve state with {@link QuickSearchRuntime#get()}; this
+ ** facade will be removed in 1.1.
  *******************************************************************************/
+@Deprecated
 public class QuickSearchQBitContext
 {
-   private static volatile QuickSearchQBitConfig             config;
-   private static volatile List<QuickSearchableTableConfig>  discoveredTables;
-   private static volatile QuickSearchOpenSearchClient       client;
-   private static volatile IndexEventPublisher               publisher;
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchQBitContext.class);
+
+   private static volatile QuickSearchQBitConfig            config;
+   private static volatile List<QuickSearchableTableConfig> discoveredTables;
+   private static volatile QuickSearchOpenSearchClient      client;
+   private static volatile IndexEventPublisher              publisher;
 
 
 
-   /***************************************************************************
-    ** Getter for config
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the config: the static override, else the resolved runtime's config.
+    *******************************************************************************/
    public static QuickSearchQBitConfig getConfig()
    {
-      return config;
+      if(config != null)
+      {
+         return (config);
+      }
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime == null ? null : runtime.getConfig());
    }
 
 
 
-   /***************************************************************************
-    ** Setter for config
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static config override.
+    *******************************************************************************/
    public static void setConfig(QuickSearchQBitConfig config)
    {
       QuickSearchQBitContext.config = config;
@@ -58,19 +72,24 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Getter for discoveredTables
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the discovered tables: the static override, else the runtime's.
+    *******************************************************************************/
    public static List<QuickSearchableTableConfig> getDiscoveredTables()
    {
-      return discoveredTables;
+      if(discoveredTables != null)
+      {
+         return (discoveredTables);
+      }
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime == null ? null : runtime.getDiscoveredTables());
    }
 
 
 
-   /***************************************************************************
-    ** Setter for discoveredTables
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static discovered-tables override.
+    *******************************************************************************/
    public static void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
    {
       QuickSearchQBitContext.discoveredTables = discoveredTables;
@@ -78,44 +97,62 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Find a table config by table name from the discovered tables list.
-    **
-    ** Returns null if discoveredTables is null or no match is found.
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Find a table config by name, or null.
+    *******************************************************************************/
    public static QuickSearchableTableConfig getTableConfig(String tableName)
    {
-      if(discoveredTables == null)
+      List<QuickSearchableTableConfig> tables = getDiscoveredTables();
+      if(tables == null || tableName == null)
       {
-         return null;
+         return (null);
       }
 
-      for(QuickSearchableTableConfig tableConfig : discoveredTables)
+      for(QuickSearchableTableConfig tableConfig : tables)
       {
          if(tableName.equals(tableConfig.getTableName()))
          {
-            return tableConfig;
+            return (tableConfig);
          }
       }
-
-      return null;
+      return (null);
    }
 
 
 
-   /***************************************************************************
-    ** Getter for client
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the client: the static override, else the runtime's (built lazily).
+    ** Returns null when no runtime is registered or the client cannot be built.
+    *******************************************************************************/
    public static QuickSearchOpenSearchClient getClient()
    {
-      return client;
+      if(client != null)
+      {
+         return (client);
+      }
+
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (null);
+      }
+
+      try
+      {
+         return (runtime.getClient());
+      }
+      catch(QException e)
+      {
+         LOG.warn("Could not build OpenSearch client", e);
+         return (null);
+      }
    }
 
 
 
-   /***************************************************************************
-    ** Setter for client
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static client override.
+    *******************************************************************************/
    public static void setClient(QuickSearchOpenSearchClient client)
    {
       QuickSearchQBitContext.client = client;
@@ -123,19 +160,38 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Getter for publisher
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Get the publisher: the static override, else the runtime's.
+    *******************************************************************************/
    public static IndexEventPublisher getPublisher()
    {
-      return publisher;
+      if(publisher != null)
+      {
+         return (publisher);
+      }
+
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (null);
+      }
+
+      try
+      {
+         return (runtime.getPublisher());
+      }
+      catch(QException e)
+      {
+         LOG.warn("Could not build index event publisher", e);
+         return (null);
+      }
    }
 
 
 
-   /***************************************************************************
-    ** Setter for publisher
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Set a static publisher override.
+    *******************************************************************************/
    public static void setPublisher(IndexEventPublisher publisher)
    {
       QuickSearchQBitContext.publisher = publisher;
@@ -143,18 +199,15 @@ public class QuickSearchQBitContext
 
 
 
-   /***************************************************************************
-    ** Reset all static state to null.
-    **
-    ** Call in test @BeforeEach or @AfterEach to prevent state leakage
-    ** between tests.
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Clear the static overrides (tests).
+    *******************************************************************************/
    public static void clear()
    {
-      config = null;
+      config           = null;
       discoveredTables = null;
-      client = null;
-      publisher = null;
+      client           = null;
+      publisher        = null;
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java

@@ -13,265 +13,397 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.kingsrook.qbits.quicksearch;
 
 
+import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import java.util.Properties;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
 import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
 import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
-import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostDeleteCustomizer;
-import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostInsertCustomizer;
-import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostUpdateCustomizer;
-import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
-import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
-import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
-import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep;
-import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
+import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** Central orchestrator for the Quick Search QBit.
+ ** Entry point of the Quick Search QBit, a QQQ 4.1 QBitMetaDataProducer.
  **
- ** Validates config, discovers annotated tables, produces QQQ metadata
- ** (tables, processes, app), initializes the OpenSearch client, and registers
- ** real-time customizers on source tables.
+ ** The framework's default produce() registers QBitMetaData, validates the
+ ** config, runs the component producers in this package (operational tables,
+ ** processes, admin app), applies the config's table customizer and default
+ ** backend, stamps sourceQBitName, then calls {@link #postProduceActions}, which
+ ** discovers searchable tables, builds the runtime, registers the record-change
+ ** listener and runtime service, registers the core record-search provider
+ ** (serveCoreRecordSearch) and gives indexed tables core searchFields, and (in
+ ** FAIL_FAST mode) prepares the index.
  **
- ** Entry point: {@link #produce(QInstance)}.
+ ** Host usage, unchanged from 0.x:
+ **   new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
+ ** That overload adds everything to the instance itself; hosts that discover
+ ** this producer through MetaDataProducerHelper get the standard contract.
  *******************************************************************************/
-public class QuickSearchQBitProducer
+public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>
 {
    private static final QLogger LOG = QLogger.getLogger(QuickSearchQBitProducer.class);
 
-   public static final String BASEPULL_PROCESS_NAME      = "quickSearchBasepullIndex";
-   public static final String FULL_REINDEX_PROCESS_NAME  = "quickSearchFullReindex";
-   public static final String APP_NAME                   = "quickSearchAdmin";
+   public static final String GROUP_ID    = "com.kingsrook.qbits";
+   public static final String ARTIFACT_ID = "quick-search";
 
-   private QuickSearchQBitConfig config;
+   public static final String BASEPULL_PROCESS_NAME     = QuickSearchProcessMetaDataHelper.BASEPULL_PROCESS_NAME;
+   public static final String FULL_REINDEX_PROCESS_NAME = QuickSearchProcessMetaDataHelper.FULL_REINDEX_PROCESS_NAME;
+   public static final String RECONCILE_PROCESS_NAME    = QuickSearchProcessMetaDataHelper.RECONCILE_PROCESS_NAME;
+   public static final String APP_NAME                  = QuickSearchAdminAppMetaDataProducer.APP_NAME;
+
+   private static final String VERSION = loadVersion();
+
+   private QuickSearchQBitConfig qBitConfig;
+   private String                namespace;
 
 
 
-   /***************************************************************************
-    ** Fluent setter for config.
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Fluent setter for the config (0.x name).
+    *******************************************************************************/
    public QuickSearchQBitProducer withConfig(QuickSearchQBitConfig config)
    {
-      this.config = config;
+      this.qBitConfig = config;
       return (this);
    }
 
 
 
-   /***************************************************************************
-    ** Getter for config.
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Getter for the config (0.x name).
+    *******************************************************************************/
    public QuickSearchQBitConfig getConfig()
    {
-      return (this.config);
+      return (this.qBitConfig);
    }
 
 
 
-   /***************************************************************************
-    ** Produce all QQQ metadata for the Quick Search QBit and register it
-    ** into the given QInstance.
-    **
-    ** Steps:
-    ** 1. Validate configuration
-    ** 2. Discover @QuickSearchable annotated entity classes
-    ** 3. Create and initialize the OpenSearch client
-    ** 4. Create the index event publisher
-    ** 5. Store everything in QuickSearchQBitContext
-    ** 6. Produce operational tables
-    ** 7. Produce processes
-    ** 8. Register customizers on source tables (if real-time indexing enabled)
-    ** 9. Produce an admin app
-    **
-    ** @param qInstance the QQQ instance to register metadata into
-    ** @throws QException if validation fails or an unrecoverable error occurs
-    ***************************************************************************/
-   public void produce(QInstance qInstance) throws QException
+   /*******************************************************************************
+    ** Fluent setter for the config.
+    *******************************************************************************/
+   public QuickSearchQBitProducer withQBitConfig(QuickSearchQBitConfig config)
    {
-      ////////////////////////////////////////////////////
-      // 1. Validate config                             //
-      ////////////////////////////////////////////////////
-      List<String> errors = new ArrayList<>();
-      config.validate(qInstance, errors);
-      if(!errors.isEmpty())
+      this.qBitConfig = config;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for the config.
+    *******************************************************************************/
+   public void setQBitConfig(QuickSearchQBitConfig config)
+   {
+      this.qBitConfig = config;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QuickSearchQBitConfig getQBitConfig()
+   {
+      return (this.qBitConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Namespace for this QBit instance; also the default tableNamePrefix.
+    *******************************************************************************/
+   public QuickSearchQBitProducer withNamespace(String namespace)
+   {
+      this.namespace = namespace;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getNamespace()
+   {
+      return (this.namespace);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QBitMetaData getQBitMetaData()
+   {
+      return (new QBitMetaData()
+         .withGroupId(GROUP_ID)
+         .withArtifactId(ARTIFACT_ID)
+         .withVersion(VERSION)
+         .withNamespace(getNamespace())
+         .withConfig(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** A producer found by package scanning has no config; skip it.
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (qBitConfig != null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Produce and add everything to the instance. Returns an empty output so a
+    ** host that also calls addSelfToInstance on the result adds nothing twice.
+    **
+    ** Validation errors are reported as one QException listing every error.
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      if(qBitConfig == null)
       {
-         throw new QException("QuickSearchQBit configuration validation failed: " + String.join("; ", errors));
+         throw (new QException("QuickSearchQBit configuration validation failed: no config was supplied (call withConfig)"));
       }
 
-      ////////////////////////////////////////////////////
-      // 2. Discover searchable tables from annotations //
-      ////////////////////////////////////////////////////
+      if(StringUtils.hasContent(namespace) && !StringUtils.hasContent(qBitConfig.getTableNamePrefix()))
+      {
+         qBitConfig.setTableNamePrefix(namespace);
+      }
+
+      List<String> errors = new ArrayList<>();
+      qBitConfig.validate(qInstance, errors);
+      if(!errors.isEmpty())
+      {
+         throw (new QException("QuickSearchQBit configuration validation failed: " + String.join("; ", errors)));
+      }
+
+      MetaDataProducerMultiOutput output = QBitMetaDataProducer.super.produce(qInstance);
+      output.addSelfToInstance(qInstance);
+
+      return (new MetaDataProducerMultiOutput());
+   }
+
+
+
+   /*******************************************************************************
+    ** Discover tables, build the runtime, register the listener and runtime
+    ** service, and prepare the index in FAIL_FAST mode.
+    *******************************************************************************/
+   @Override
+   public void postProduceActions(MetaDataProducerMultiOutput output, QInstance qInstance) throws QException
+   {
       List<QuickSearchableTableConfig> annotationTables = discoverSearchableTables(qInstance);
+      List<QuickSearchableTableConfig> configTables     = convertConfigDrivenTables(qInstance);
 
-      ////////////////////////////////////////////////////
-      // 2b. Convert config-driven searchable tables    //
-      ////////////////////////////////////////////////////
-      List<QuickSearchableTableConfig> configTables = convertConfigDrivenTables(qInstance);
-
-      ////////////////////////////////////////////////////
-      // 2c. Detect duplicate table names across both   //
-      //     annotation-based and config-driven sources //
-      ////////////////////////////////////////////////////
       for(QuickSearchableTableConfig annotationTable : annotationTables)
       {
          for(QuickSearchableTableConfig configTable : configTables)
          {
             if(annotationTable.getTableName().equals(configTable.getTableName()))
             {
-               throw new QException("Duplicate table name found in both annotation-based and config-driven searchable tables: " + annotationTable.getTableName());
+               throw (new QException("Duplicate table name found in both annotation-based and config-driven searchable tables: " + annotationTable.getTableName()));
             }
          }
       }
 
-      ////////////////////////////////////////////////////
-      // 2d. Merge both lists into discoveredTables     //
-      ////////////////////////////////////////////////////
       List<QuickSearchableTableConfig> discoveredTables = new ArrayList<>();
       discoveredTables.addAll(annotationTables);
       discoveredTables.addAll(configTables);
 
-      ////////////////////////////////////////////////////
-      // 3. Create OpenSearch client                    //
-      ////////////////////////////////////////////////////
-      QuickSearchOpenSearchClient client = null;
-      try
+      QuickSearchRuntime previous = qBitConfig.getRuntime();
+      if(previous != null)
       {
-         client = new QuickSearchOpenSearchClient(config);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to create OpenSearch client during produce; continuing without client", e);
+         previous.close();
       }
 
-      ////////////////////////////////////////////////////
-      // 4. Initialize index (best-effort)              //
-      ////////////////////////////////////////////////////
-      if(client != null)
+      QuickSearchRuntime runtime = new QuickSearchRuntime(qBitConfig);
+      runtime.setDiscoveredTables(discoveredTables);
+      qBitConfig.setRuntime(runtime);
+
+      if(Boolean.TRUE.equals(qBitConfig.getEnableRealTimeIndexing()))
       {
-         try
-         {
-            client.ensureIndexExists();
-         }
-         catch(Exception e)
-         {
-            LOG.warn("Failed to ensure OpenSearch index exists during produce; continuing", e);
-         }
+         qInstance.withRecordChangeListener(new QCodeReference(QuickSearchRecordChangeListener.class));
       }
 
-      ////////////////////////////////////////////////////
-      // 5. Create publisher                            //
-      ////////////////////////////////////////////////////
-      IndexEventPublisher publisher = null;
-      if(config.getIndexEventPublisher() != null)
+      qInstance.withRuntimeService(new QCodeReference(QuickSearchRuntimeService.class));
+
+      addRecordDiagnosticActions(output, qInstance, discoveredTables);
+
+      if(Boolean.TRUE.equals(qBitConfig.getServeCoreRecordSearch()))
       {
-         publisher = config.getIndexEventPublisher();
-      }
-      else if(client != null)
-      {
-         publisher = new SynchronousIndexEventPublisher(client, discoveredTables, config.getBulkBatchSize());
+         registerRecordSearchProvider(qInstance);
+         defaultCoreSearchFields(qInstance, discoveredTables);
       }
 
-      ////////////////////////////////////////////////////
-      // 6. Store in context                            //
-      ////////////////////////////////////////////////////
-      QuickSearchQBitContext.setConfig(config);
-      QuickSearchQBitContext.setClient(client);
-      QuickSearchQBitContext.setPublisher(publisher);
-      QuickSearchQBitContext.setDiscoveredTables(discoveredTables);
-
-      ////////////////////////////////////////////////////
-      // 7. Produce operational tables                  //
-      ////////////////////////////////////////////////////
-      String indexTableName    = config.applyPrefix(QuickSearchIndex.TABLE_NAME);
-      String indexRunTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME);
-
-      QTableMetaData indexTable = buildQuickSearchIndexTable(indexTableName, config.getBackendName());
-      QTableMetaData indexRunTable = buildQuickSearchIndexRunTable(indexRunTableName, config.getBackendName());
-
-      qInstance.addTable(indexTable);
-      qInstance.addTable(indexRunTable);
-
-      ////////////////////////////////////////////////////
-      // 8. Produce processes                           //
-      ////////////////////////////////////////////////////
-      String basepullProcessName   = config.applyPrefix(BASEPULL_PROCESS_NAME);
-      String fullReindexProcessName = config.applyPrefix(FULL_REINDEX_PROCESS_NAME);
-
-      QProcessMetaData basepullProcess = buildBasepullProcess(basepullProcessName);
-      QProcessMetaData fullReindexProcess = buildFullReindexProcess(fullReindexProcessName);
-
-      if(Boolean.TRUE.equals(config.getEnableScheduledProcesses()))
+      if(!StringUtils.hasContent(qBitConfig.getSchedulerName()) && Boolean.TRUE.equals(qBitConfig.getEnableBasepullProcess()))
       {
-         qInstance.addProcess(basepullProcess);
-         qInstance.addProcess(fullReindexProcess);
+         LOG.warn("Quick Search has no schedulerName; the basepull and reconcile processes are registered but will not run on a schedule");
       }
 
-      ////////////////////////////////////////////////////
-      // 9. Register customizers on source tables       //
-      ////////////////////////////////////////////////////
-      if(Boolean.TRUE.equals(config.getEnableRealTimeIndexing()))
+      if(qBitConfig.getStartupMode() == QuickSearchStartupMode.FAIL_FAST)
       {
-         registerCustomizers(qInstance, discoveredTables);
+         runtime.start();
       }
 
-      ////////////////////////////////////////////////////
-      // 10. Produce admin app                          //
-      ////////////////////////////////////////////////////
-      String appName = config.applyPrefix(APP_NAME);
-      QAppMetaData app = new QAppMetaData()
-         .withName(appName)
-         .withLabel("Quick Search Admin")
-         .withChild(indexTable)
-         .withChild(indexRunTable);
-
-      if(Boolean.TRUE.equals(config.getEnableScheduledProcesses()))
-      {
-         app.withChild(basepullProcess);
-         app.withChild(fullReindexProcess);
-      }
-
-      qInstance.addApp(app);
+      LOG.info("Quick Search QBit produced", logPair("version", VERSION), logPair("tables", discoveredTables.size()),
+         logPair("indexName", qBitConfig.getOpensearchIndexName()), logPair("startupMode", qBitConfig.getStartupMode()));
    }
 
 
 
-   /***************************************************************************
-    ** Discover searchable table configurations by scanning annotated entity
-    ** classes from the config.
-    **
-    ** For each class, extracts @QuickSearchable annotation attributes and
-    ** scans fields for @QuickSearchField annotations. Falls back to the
-    ** annotation's fields() array if no @QuickSearchField-annotated fields
-    ** are found.
-    **
-    ** @param qInstance the QInstance to look up table primary keys
-    ** @return list of discovered table configurations
-    ***************************************************************************/
-   List<QuickSearchableTableConfig> discoverSearchableTables(QInstance qInstance)
+   /*******************************************************************************
+    ** Give every indexed table in the instance the Inspect Search Document and
+    ** Reindex in Search record actions (when enableDiagnosticTools and
+    ** enableRecordDiagnosticActions are on). A process the host already
+    ** defined with the same name is kept.
+    *******************************************************************************/
+   void addRecordDiagnosticActions(MetaDataProducerMultiOutput output, QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
    {
-      List<QuickSearchableTableConfig> tables = new ArrayList<>();
-      List<Class<?>> entityClasses = config.getSearchableEntityClasses();
+      if(!Boolean.TRUE.equals(qBitConfig.getEnableDiagnosticTools()) || !Boolean.TRUE.equals(qBitConfig.getEnableRecordDiagnosticActions()))
+      {
+         return;
+      }
+
+      for(QuickSearchableTableConfig tableConfig : tableConfigs)
+      {
+         if(qInstance.getTable(tableConfig.getTableName()) == null)
+         {
+            continue;
+         }
+         for(QProcessMetaData process : List.of(
+            QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(qBitConfig, tableConfig.getTableName()),
+            QuickSearchDiagnosticsProcessMetaDataHelper.buildReindexRecordsProcess(qBitConfig, tableConfig.getTableName())))
+         {
+            if(qInstance.getProcess(process.getName()) == null && output.get(QProcessMetaData.class, process.getName()) == null)
+            {
+               process.setSourceQBitName(getQBitMetaData().getName());
+               output.add(process);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Register QuickSearchRecordSearchProvider as the instance's core record
+    ** search provider. An instance holds one provider; a different one the host
+    ** already registered is kept (with a warning).
+    *******************************************************************************/
+   static void registerRecordSearchProvider(QInstance qInstance)
+   {
+      QCodeReference existing = qInstance.getRecordSearchProvider();
+      if(existing != null && !QuickSearchRecordSearchProvider.class.getName().equals(existing.getName()))
+      {
+         LOG.warn("The instance already has a record search provider; Quick Search does not replace it, so core search does not use OpenSearch",
+            logPair("provider", existing.getName()));
+         return;
+      }
+
+      qInstance.withRecordSearchProvider(new QCodeReference(QuickSearchRecordSearchProvider.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** Core record search only searches tables with searchFields, so give each
+    ** indexed table that has none the searchable fields core accepts (visible
+    ** string or integer fields; never password fields). Fields the host set are
+    ** never replaced. A table that is not in the instance yet, uses backend
+    ** variants, or has no such field is skipped with a warning.
+    *******************************************************************************/
+   static void defaultCoreSearchFields(QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      for(QuickSearchableTableConfig tableConfig : tableConfigs)
+      {
+         QTableMetaData table = qInstance.getTable(tableConfig.getTableName());
+         if(table == null)
+         {
+            LOG.warn("Indexed table is not in the instance when Quick Search is produced, so it gets no core searchFields; set them on the table for core search to use it",
+               logPair("tableName", tableConfig.getTableName()));
+            continue;
+         }
+
+         if(table.getSearchFields() != null)
+         {
+            continue;
+         }
+
+         QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
+         if(backend != null && Boolean.TRUE.equals(backend.getUsesVariants()))
+         {
+            LOG.warn("Indexed table uses backend variants, which core record search does not support; it gets no core searchFields", logPair("tableName", table.getName()));
+            continue;
+         }
+
+         List<String> searchFields = new ArrayList<>();
+         for(String fieldName : CollectionUtils.nonNullList(tableConfig.getSearchableFields()))
+         {
+            QFieldMetaData field = table.getFields() == null ? null : table.getFields().get(fieldName);
+            if(field != null && !Boolean.TRUE.equals(field.getIsHidden()) && field.getType() != null
+               && (field.getType().isStringLike() || field.getType().isIntegral()) && !field.getType().needsMasked()
+               && !searchFields.contains(fieldName))
+            {
+               searchFields.add(fieldName);
+            }
+         }
+
+         if(searchFields.isEmpty())
+         {
+            LOG.warn("Indexed table has no string or integer searchable field, so it gets no core searchFields; set them on the table for core search to use it",
+               logPair("tableName", table.getName()));
+            continue;
+         }
+
+         table.setSearchFields(searchFields);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Discover searchable tables from @QuickSearchable entity classes. Fails when
+    ** a configured field does not exist on a table that is in the instance.
+    *******************************************************************************/
+   List<QuickSearchableTableConfig> discoverSearchableTables(QInstance qInstance) throws QException
+   {
+      List<QuickSearchableTableConfig> tables        = new ArrayList<>();
+      List<Class<?>>                   entityClasses = qBitConfig.getSearchableEntityClasses();
 
       if(entityClasses == null || entityClasses.isEmpty())
       {
@@ -283,81 +415,65 @@ public class QuickSearchQBitProducer
          QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class);
          if(annotation == null)
          {
-            LOG.warn("Entity class does not have @QuickSearchable annotation; skipping",
-               "className", entityClass.getName());
+            LOG.warn("Entity class does not have @QuickSearchable annotation; skipping", logPair("className", entityClass.getName()));
             continue;
          }
 
-         String tableName                  = annotation.tableName();
-         Integer basepullIntervalMinutes   = annotation.basepullIntervalMinutes();
-         String basepullTimestampField     = annotation.basepullTimestampField();
-         Boolean enabledByDefault          = annotation.enabledByDefault();
+         String tableName = annotation.tableName();
 
-         //////////////////////////////////////////////////////////
-         // Scan fields for @QuickSearchField annotations        //
-         //////////////////////////////////////////////////////////
-         List<String> searchableFields         = new ArrayList<>();
-         Map<String, Integer> fieldWeights     = new LinkedHashMap<>();
+         List<String>         searchableFields   = new ArrayList<>();
+         Map<String, Integer> fieldWeights       = new LinkedHashMap<>();
          Map<String, Boolean> fieldIncludeLabels = new LinkedHashMap<>();
 
          List<Field> allFields = new ArrayList<>();
-         Class<?> current = entityClass;
+         Class<?>    current   = entityClass;
          while(current != null && current != Object.class)
          {
-            allFields.addAll(java.util.Arrays.asList(current.getDeclaredFields()));
+            allFields.addAll(Arrays.asList(current.getDeclaredFields()));
             current = current.getSuperclass();
          }
+
          for(Field field : allFields)
          {
             QuickSearchField fieldAnnotation = field.getAnnotation(QuickSearchField.class);
             if(fieldAnnotation != null)
             {
-               String fieldName = field.getName();
-               searchableFields.add(fieldName);
-               fieldWeights.put(fieldName, fieldAnnotation.weight());
-               fieldIncludeLabels.put(fieldName, fieldAnnotation.includeLabel());
+               searchableFields.add(field.getName());
+               fieldWeights.put(field.getName(), fieldAnnotation.weight());
+               fieldIncludeLabels.put(field.getName(), fieldAnnotation.includeLabel());
             }
          }
 
-         //////////////////////////////////////////////////////////
-         // Fall back to @QuickSearchable.fields() if no         //
-         // @QuickSearchField-annotated fields were found        //
-         //////////////////////////////////////////////////////////
-         if(searchableFields.isEmpty())
+         if(searchableFields.isEmpty() && annotation.fields() != null)
          {
-            String[] annotationFields = annotation.fields();
-            if(annotationFields != null)
+            for(String fieldName : annotation.fields())
             {
-               for(String fieldName : annotationFields)
-               {
-                  searchableFields.add(fieldName);
-                  fieldWeights.put(fieldName, 1);
-                  fieldIncludeLabels.put(fieldName, false);
-               }
+               searchableFields.add(fieldName);
+               fieldWeights.put(fieldName, 1);
+               fieldIncludeLabels.put(fieldName, false);
             }
          }
 
-         //////////////////////////////////////////////////////////
-         // Get primary key field from QInstance table metadata   //
-         //////////////////////////////////////////////////////////
-         String primaryKeyField = "id";
-         QTableMetaData table = qInstance.getTable(tableName);
+         String         primaryKeyField        = "id";
+         String         basepullTimestampField = StringUtils.hasContent(annotation.basepullTimestampField()) ? annotation.basepullTimestampField() : null;
+         QTableMetaData table                  = qInstance.getTable(tableName);
          if(table != null)
          {
-            primaryKeyField = table.getPrimaryKeyField();
+            primaryKeyField        = table.getPrimaryKeyField();
+            basepullTimestampField = resolveTimestampField(table, searchableFields, basepullTimestampField);
+            removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
          }
 
-         QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
+         tables.add(new QuickSearchableTableConfig()
             .withTableName(tableName)
             .withPrimaryKeyField(primaryKeyField)
             .withSearchableFields(searchableFields)
             .withFieldWeights(fieldWeights)
             .withFieldIncludeLabels(fieldIncludeLabels)
-            .withBasepullIntervalMinutes(basepullIntervalMinutes)
+            .withBasepullIntervalMinutes(annotation.basepullIntervalMinutes())
             .withBasepullTimestampField(basepullTimestampField)
-            .withEnabledByDefault(enabledByDefault);
-
-         tables.add(tableConfig);
+            .withEnabledByDefault(annotation.enabledByDefault())
+            .withMaxFieldLength(qBitConfig.getMaxFieldLength()));
       }
 
       return (tables);
@@ -365,24 +481,13 @@ public class QuickSearchQBitProducer
 
 
 
-   /***************************************************************************
-    ** Convert config-driven SearchableTableConfig entries into
-    ** QuickSearchableTableConfig objects.
-    **
-    ** Maps each SearchableFieldConfig into the searchableFields list,
-    ** fieldWeights map, and fieldIncludeLabels map. Resolves the primary
-    ** key field from the QInstance table metadata (falls back to "id").
-    ** Applies the default basepullIntervalMinutes from QBit config when
-    ** the table-level value is null. Transfers recordLabelFormat and
-    ** recordLabelFields.
-    **
-    ** @param qInstance the QInstance to look up table primary keys
-    ** @return list of converted table configurations, empty if none configured
-    ***************************************************************************/
+   /*******************************************************************************
+    ** Convert config-driven tables, resolving primary keys and defaults.
+    *******************************************************************************/
    private List<QuickSearchableTableConfig> convertConfigDrivenTables(QInstance qInstance)
    {
-      List<QuickSearchableTableConfig> tables = new ArrayList<>();
-      List<SearchableTableConfig> searchableTables = config.getSearchableTables();
+      List<QuickSearchableTableConfig> tables           = new ArrayList<>();
+      List<SearchableTableConfig>      searchableTables = qBitConfig.getSearchableTables();
 
       if(searchableTables == null || searchableTables.isEmpty())
       {
@@ -391,53 +496,49 @@ public class QuickSearchQBitProducer
 
       for(SearchableTableConfig stc : searchableTables)
       {
-         List<String> searchableFields           = new ArrayList<>();
-         Map<String, Integer> fieldWeights        = new LinkedHashMap<>();
-         Map<String, Boolean> fieldIncludeLabels  = new LinkedHashMap<>();
+         List<String>         searchableFields   = new ArrayList<>();
+         Map<String, Integer> fieldWeights       = new LinkedHashMap<>();
+         Map<String, Boolean> fieldIncludeLabels = new LinkedHashMap<>();
 
-         if(stc.getFields() != null)
+         for(SearchableFieldConfig fieldConfig : stc.getFields() == null ? List.<SearchableFieldConfig>of() : stc.getFields())
          {
-            for(SearchableFieldConfig fieldConfig : stc.getFields())
-            {
-               String fieldName = fieldConfig.getFieldName();
-               searchableFields.add(fieldName);
-               fieldWeights.put(fieldName, fieldConfig.getWeight() != null ? fieldConfig.getWeight() : 1);
-               fieldIncludeLabels.put(fieldName, fieldConfig.getIncludeLabel() != null ? fieldConfig.getIncludeLabel() : false);
-            }
+            searchableFields.add(fieldConfig.getFieldName());
+            fieldWeights.put(fieldConfig.getFieldName(), fieldConfig.getWeight() != null ? fieldConfig.getWeight() : 1);
+            fieldIncludeLabels.put(fieldConfig.getFieldName(), fieldConfig.getIncludeLabel() != null ? fieldConfig.getIncludeLabel() : false);
          }
 
-         //////////////////////////////////////////////////////////
-         // Resolve primary key from QInstance table metadata     //
-         //////////////////////////////////////////////////////////
-         String primaryKeyField = "id";
-         QTableMetaData table = qInstance.getTable(stc.getTableName());
+         String         primaryKeyField        = "id";
+         String         basepullTimestampField = stc.getBasepullTimestampField();
+         QTableMetaData table                  = qInstance.getTable(stc.getTableName());
          if(table != null)
          {
             primaryKeyField = table.getPrimaryKeyField();
+            if(basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField))
+            {
+               warnIncrementalBasepullDisabled(table, basepullTimestampField);
+               basepullTimestampField = null;
+            }
+            removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
          }
 
-         //////////////////////////////////////////////////////////
-         // Apply default basepullIntervalMinutes from config    //
-         //////////////////////////////////////////////////////////
          Integer basepullInterval = stc.getBasepullIntervalMinutes();
          if(basepullInterval == null)
          {
-            basepullInterval = config.getDefaultBasepullIntervalMinutes();
+            basepullInterval = qBitConfig.getDefaultBasepullIntervalMinutes();
          }
 
-         QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
+         tables.add(new QuickSearchableTableConfig()
             .withTableName(stc.getTableName())
             .withPrimaryKeyField(primaryKeyField)
             .withSearchableFields(searchableFields)
             .withFieldWeights(fieldWeights)
             .withFieldIncludeLabels(fieldIncludeLabels)
             .withBasepullIntervalMinutes(basepullInterval)
-            .withBasepullTimestampField(stc.getBasepullTimestampField())
+            .withBasepullTimestampField(basepullTimestampField)
             .withEnabledByDefault(stc.getEnabledByDefault())
             .withRecordLabelFormat(stc.getRecordLabelFormat())
-            .withRecordLabelFields(stc.getRecordLabelFields());
-
-         tables.add(tableConfig);
+            .withRecordLabelFields(stc.getRecordLabelFields())
+            .withMaxFieldLength(qBitConfig.getMaxFieldLength()));
       }
 
       return (tables);
@@ -445,115 +546,116 @@ public class QuickSearchQBitProducer
 
 
 
-   /***************************************************************************
-    ** Build QTableMetaData for the quickSearchIndex operational table.
-    ***************************************************************************/
-   private QTableMetaData buildQuickSearchIndexTable(String tableName, String backendName)
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   /*******************************************************************************
+    ** Fail on missing searchable fields. A missing timestamp field disables
+    ** incremental basepull for the table (reconcile still covers it): the
+    ** default "modifyDate" is downgraded with a warning, an explicit other name
+    ** is an error.
+    *******************************************************************************/
+   private static String resolveTimestampField(QTableMetaData table, List<String> searchableFields, String basepullTimestampField) throws QException
    {
-      return new QTableMetaData()
-         .withName(tableName)
-         .withLabel("Quick Search Index")
-         .withBackendName(backendName)
-         .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("tableName", QFieldType.STRING))
-         .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
-         .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING))
-         .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.STRING))
-         .withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("recordCount", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("status", QFieldType.STRING))
-         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME));
-   }
-
-
-
-   /***************************************************************************
-    ** Build QTableMetaData for the quickSearchIndexRun operational table.
-    ***************************************************************************/
-   private QTableMetaData buildQuickSearchIndexRunTable(String tableName, String backendName)
-   {
-      return new QTableMetaData()
-         .withName(tableName)
-         .withLabel("Quick Search Index Run")
-         .withBackendName(backendName)
-         .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
-         .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("runType", QFieldType.STRING))
-         .withField(new QFieldMetaData("status", QFieldType.STRING))
-         .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("errorCount", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("errorMessage", QFieldType.STRING))
-         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME));
-   }
-
-
-
-   /***************************************************************************
-    ** Build the basepull indexing process metadata.
-    ***************************************************************************/
-   private QProcessMetaData buildBasepullProcess(String processName)
-   {
-      return new QProcessMetaData()
-         .withName(processName)
-         .withLabel("Quick Search Basepull Index")
-         .withStep(new QBackendStepMetaData()
-            .withName("basepull")
-            .withCode(new QCodeReference(BasepullIndexStep.class)));
-   }
-
-
-
-   /***************************************************************************
-    ** Build the full reindex process metadata with optional tableName input.
-    ***************************************************************************/
-   private QProcessMetaData buildFullReindexProcess(String processName)
-   {
-      return new QProcessMetaData()
-         .withName(processName)
-         .withLabel("Quick Search Full Reindex")
-         .withStep(new QBackendStepMetaData()
-            .withName("fullReindex")
-            .withCode(new QCodeReference(FullReindexStep.class)));
-   }
-
-
-
-   /***************************************************************************
-    ** Register post-insert, post-update, and post-delete customizers on each
-    ** discovered source table.
-    ***************************************************************************/
-   private void registerCustomizers(QInstance qInstance, List<QuickSearchableTableConfig> discoveredTables)
-   {
-      for(QuickSearchableTableConfig tableConfig : discoveredTables)
+      List<String> missing = new ArrayList<>();
+      for(String fieldName : searchableFields)
       {
-         String tableName = tableConfig.getTableName();
-         QTableMetaData table = qInstance.getTable(tableName);
-
-         if(table == null)
+         if(!table.getFields().containsKey(fieldName))
          {
-            LOG.warn("Source table not found in QInstance; skipping customizer registration",
-               "tableName", tableName);
-            continue;
+            missing.add(fieldName);
          }
+      }
+      if(!missing.isEmpty())
+      {
+         throw (new QException("Searchable field(s) " + missing + " do not exist on table [" + table.getName() + "]"));
+      }
 
-         table.withCustomizer(TableCustomizers.POST_INSERT_RECORD,
-            new QCodeReference(QuickSearchPostInsertCustomizer.class));
+      if(basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField))
+      {
+         if(QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD.equals(basepullTimestampField))
+         {
+            warnIncrementalBasepullDisabled(table, basepullTimestampField);
+            return (null);
+         }
+         throw (new QException("basepullTimestampField [" + basepullTimestampField + "] does not exist on table [" + table.getName() + "]"));
+      }
+      return (basepullTimestampField);
+   }
 
-         table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD,
-            new QCodeReference(QuickSearchPostUpdateCustomizer.class));
 
-         table.withCustomizer(TableCustomizers.POST_DELETE_RECORD,
-            new QCodeReference(QuickSearchPostDeleteCustomizer.class));
+
+   /*******************************************************************************
+    ** The startup warning, once per table, for annotated and config-driven
+    ** tables alike, when the basepull timestamp field is missing. A field
+    ** explicitly set to null is an opt-out and is not warned about.
+    *******************************************************************************/
+   private static void warnIncrementalBasepullDisabled(QTableMetaData table, String basepullTimestampField)
+   {
+      LOG.warn("Table has no basepull timestamp field; incremental basepull is disabled for it (reconcile still covers it)",
+         logPair("tableName", table.getName()), logPair("timestampField", basepullTimestampField));
+   }
+
+
+
+   /*******************************************************************************
+    ** Hidden and password fields are never indexed: their values would leak
+    ** through searchable text and highlights.
+    *******************************************************************************/
+   private static void removeHiddenFields(QTableMetaData table, List<String> searchableFields, Map<String, Integer> fieldWeights, Map<String, Boolean> fieldIncludeLabels)
+   {
+      List<String> hidden = new ArrayList<>();
+      for(String fieldName : searchableFields)
+      {
+         QFieldMetaData field = table.getFields().get(fieldName);
+         if(field != null && (Boolean.TRUE.equals(field.getIsHidden()) || (field.getType() != null && field.getType().needsMasked())))
+         {
+            hidden.add(fieldName);
+         }
+      }
+      if(!hidden.isEmpty())
+      {
+         LOG.warn("Hidden and password fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
+         searchableFields.removeAll(hidden);
+         hidden.forEach(fieldWeights::remove);
+         hidden.forEach(fieldIncludeLabels::remove);
       }
    }
 
+
+
+   /*******************************************************************************
+    ** The artifact version, from the Maven-filtered quick-search.properties.
+    *******************************************************************************/
+   static String loadVersion()
+   {
+      try(InputStream inputStream = QuickSearchQBitProducer.class.getClassLoader().getResourceAsStream("quick-search.properties"))
+      {
+         if(inputStream != null)
+         {
+            Properties properties = new Properties();
+            properties.load(inputStream);
+            String version = properties.getProperty("version");
+            if(StringUtils.hasContent(version) && !version.contains("${"))
+            {
+               return (version);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not read quick-search.properties", e);
+      }
+      return ("unknown");
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for the artifact version.
+    *******************************************************************************/
+   public static String getVersion()
+   {
+      return (VERSION);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntime.java

@@ -0,0 +1,606 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticEventRecorder;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsRateLimiter;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
+import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Live state of one produced Quick Search QBit: its discovered tables, the
+ ** lazily built OpenSearch client and publisher, and a short-lived cache of
+ ** which tables are enabled.
+ **
+ ** One runtime exists per produced QBit. It hangs off the QBit's config
+ ** (QuickSearchQBitConfig.getRuntime()), which the host's QInstance holds in
+ ** QBitMetaData, so steps, listeners and actions resolve it from QContext with
+ ** {@link #get()} rather than from JVM-global statics. Two QInstances in one
+ ** JVM therefore no longer share state.
+ *******************************************************************************/
+public class QuickSearchRuntime
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRuntime.class);
+
+   private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
+
+   public static final long RECORD_SEARCH_BACKOFF_MILLIS = 30_000L;
+
+   private final QuickSearchQBitConfig config;
+
+   private List<QuickSearchableTableConfig>        discoveredTables = new ArrayList<>();
+   private Map<String, QuickSearchableTableConfig> tablesByName     = new LinkedHashMap<>();
+
+   private volatile QuickSearchOpenSearchClient client;
+   private volatile IndexEventPublisher         publisher;
+   private volatile boolean                     indexReady                     = false;
+   private volatile boolean                     closed                         = false;
+   private volatile long                        recordSearchBackoffUntilMillis = 0L;
+
+   private final Map<String, CachedFlag> enabledCache = new ConcurrentHashMap<>();
+
+   private volatile DiagnosticEventRecorder diagnosticEventRecorder;
+   private final DiagnosticsRateLimiter     diagnosticsRateLimiter = new DiagnosticsRateLimiter();
+   private final AtomicBoolean              validationRunning      = new AtomicBoolean(false);
+   private final AtomicLong                 recordSearchFailures   = new AtomicLong();
+   private final AtomicLong                 backoffDeclinedClaims  = new AtomicLong();
+   private final Instant                    startedAt              = Instant.now();
+   private volatile Instant                 lastRecordSearchFailureTime;
+   private volatile String                  lastRecordSearchFailureMessage;
+
+
+
+   /*******************************************************************************
+    ** A cached enabled flag with the time it was read.
+    *******************************************************************************/
+   private record CachedFlag(boolean enabled, long readAtMillis)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchRuntime(QuickSearchQBitConfig config)
+   {
+      this.config = config;
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve the runtime of the Quick Search QBit registered in the current
+    ** QContext's QInstance. Returns null when there is no QContext, no QInstance,
+    ** no Quick Search QBit, or the QBit has not been produced yet.
+    *******************************************************************************/
+   public static QuickSearchRuntime get()
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(qInstance == null || qInstance.getQBits() == null)
+      {
+         return (null);
+      }
+
+      for(QBitMetaData qBitMetaData : qInstance.getQBits().values())
+      {
+         if(qBitMetaData.getConfig() instanceof QuickSearchQBitConfig quickSearchConfig && quickSearchConfig.getRuntime() != null)
+         {
+            return (quickSearchConfig.getRuntime());
+         }
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Like {@link #get()}, but throws when no runtime can be resolved.
+    *******************************************************************************/
+   public static QuickSearchRuntime getOrThrow() throws QException
+   {
+      QuickSearchRuntime runtime = get();
+      if(runtime == null)
+      {
+         throw (new QException("Quick Search QBit is not registered in the current QInstance (or has not been produced)"));
+      }
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for config
+    *******************************************************************************/
+   public QuickSearchQBitConfig getConfig()
+   {
+      return (this.config);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for discoveredTables (never null)
+    *******************************************************************************/
+   public List<QuickSearchableTableConfig> getDiscoveredTables()
+   {
+      return (Collections.unmodifiableList(this.discoveredTables));
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace the discovered tables.
+    *******************************************************************************/
+   public void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
+   {
+      this.discoveredTables = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
+      this.tablesByName     = new LinkedHashMap<>();
+      for(QuickSearchableTableConfig tableConfig : this.discoveredTables)
+      {
+         this.tablesByName.put(tableConfig.getTableName(), tableConfig);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Find the config for one table, or null.
+    *******************************************************************************/
+   public QuickSearchableTableConfig getTableConfig(String tableName)
+   {
+      return (tableName == null ? null : tablesByName.get(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Return the OpenSearch client, building it on first use. Building the
+    ** client does not contact the cluster; {@link #ensureIndexReady()} does.
+    *******************************************************************************/
+   public QuickSearchOpenSearchClient getClient() throws QException
+   {
+      QuickSearchOpenSearchClient existing = client;
+      if(existing != null)
+      {
+         return (existing);
+      }
+
+      synchronized(this)
+      {
+         if(client == null)
+         {
+            client = new QuickSearchOpenSearchClient(config);
+         }
+         return (client);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Make sure the index (or alias) exists with the current mapping. Runs the
+    ** check once per runtime; a failure leaves it unready so the next call
+    ** retries.
+    *******************************************************************************/
+   public void ensureIndexReady() throws QException
+   {
+      if(indexReady)
+      {
+         return;
+      }
+
+      synchronized(this)
+      {
+         if(!indexReady)
+         {
+            getClient().ensureIndexExists();
+            indexReady = true;
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether {@link #ensureIndexReady()} has succeeded.
+    *******************************************************************************/
+   public boolean isIndexReady()
+   {
+      return (indexReady);
+   }
+
+
+
+   /*******************************************************************************
+    ** Return the publisher: the host-supplied one from config, else a
+    ** SynchronousIndexEventPublisher over this runtime's client.
+    *******************************************************************************/
+   public IndexEventPublisher getPublisher() throws QException
+   {
+      IndexEventPublisher existing = publisher;
+      if(existing != null)
+      {
+         return (existing);
+      }
+
+      synchronized(this)
+      {
+         if(publisher == null)
+         {
+            if(config.getIndexEventPublisher() != null)
+            {
+               publisher = config.getIndexEventPublisher();
+            }
+            else
+            {
+               publisher = new SynchronousIndexEventPublisher(getClient(), discoveredTables, config.getBulkBatchSize());
+            }
+         }
+         return (publisher);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table is enabled for indexing and search, per its
+    ** quickSearchIndex row. A table with no row yet is enabled. Results are
+    ** cached for one minute; a failed lookup is treated as enabled so indexing
+    ** does not stop because the operational table is unreachable.
+    *******************************************************************************/
+   public boolean isTableEnabled(String tableName)
+   {
+      if(tableName == null || !tablesByName.containsKey(tableName))
+      {
+         return (false);
+      }
+
+      long       now    = System.currentTimeMillis();
+      CachedFlag cached = enabledCache.get(tableName);
+      if(cached != null && now - cached.readAtMillis() < ENABLED_CACHE_TTL_MILLIS)
+      {
+         return (cached.enabled());
+      }
+
+      boolean enabled = readEnabledFlag(tableName);
+      enabledCache.put(tableName, new CachedFlag(enabled, now));
+      return (enabled);
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget cached enabled flags (after an operator toggles a table).
+    *******************************************************************************/
+   public void invalidateEnabledCache()
+   {
+      enabledCache.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Record that a core record search through OpenSearch failed. For the next
+    ** RECORD_SEARCH_BACKOFF_MILLIS the record-search provider claims no tables,
+    ** so core searches them itself at once instead of waiting for another
+    ** timeout on every search while OpenSearch is down.
+    *******************************************************************************/
+   public void markRecordSearchFailed()
+   {
+      markRecordSearchFailed(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** As markRecordSearchFailed(), also counting the failure and keeping its
+    ** message and time for the diagnostics.
+    *******************************************************************************/
+   public void markRecordSearchFailed(String message)
+   {
+      recordSearchBackoffUntilMillis = System.currentTimeMillis() + RECORD_SEARCH_BACKOFF_MILLIS;
+      recordSearchFailures.incrementAndGet();
+      lastRecordSearchFailureTime    = Instant.now();
+      lastRecordSearchFailureMessage = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Milliseconds left in the current record-search backoff (0 when none).
+    *******************************************************************************/
+   public long getRecordSearchBackoffRemainingMillis()
+   {
+      return (Math.max(0L, recordSearchBackoffUntilMillis - System.currentTimeMillis()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Count a table the provider declined because of a backoff (core searched
+    ** it with its own search instead).
+    *******************************************************************************/
+   public void countBackoffDeclinedClaim()
+   {
+      backoffDeclinedClaims.incrementAndGet();
+   }
+
+
+
+   /*******************************************************************************
+    ** Record searches through OpenSearch that failed since this runtime started.
+    *******************************************************************************/
+   public long getRecordSearchFailureCount()
+   {
+      return (recordSearchFailures.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Table searches core ran itself because the provider was backing off,
+    ** since this runtime started.
+    *******************************************************************************/
+   public long getBackoffDeclinedClaimCount()
+   {
+      return (backoffDeclinedClaims.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** When the last record search through OpenSearch failed (null when none).
+    *******************************************************************************/
+   public Instant getLastRecordSearchFailureTime()
+   {
+      return (lastRecordSearchFailureTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** The last record-search failure's message (null when none).
+    *******************************************************************************/
+   public String getLastRecordSearchFailureMessage()
+   {
+      return (lastRecordSearchFailureMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** When this runtime was created.
+    *******************************************************************************/
+   public Instant getStartedAt()
+   {
+      return (startedAt);
+   }
+
+
+
+   /*******************************************************************************
+    ** The diagnostic event recorder, built on first use.
+    *******************************************************************************/
+   public DiagnosticEventRecorder getDiagnosticEventRecorder()
+   {
+      DiagnosticEventRecorder existing = diagnosticEventRecorder;
+      if(existing != null)
+      {
+         return (existing);
+      }
+      synchronized(this)
+      {
+         if(diagnosticEventRecorder == null)
+         {
+            diagnosticEventRecorder = new DiagnosticEventRecorder(config);
+         }
+         return (diagnosticEventRecorder);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The per-user rate limiter of the diagnostic tools.
+    *******************************************************************************/
+   public DiagnosticsRateLimiter getDiagnosticsRateLimiter()
+   {
+      return (diagnosticsRateLimiter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Claim the single validation slot of this JVM; false when a validation is
+    ** already running. Release with endValidation().
+    *******************************************************************************/
+   public boolean beginValidation()
+   {
+      return (validationRunning.compareAndSet(false, true));
+   }
+
+
+
+   /*******************************************************************************
+    ** Release the validation slot.
+    *******************************************************************************/
+   public void endValidation()
+   {
+      validationRunning.set(false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether core record search is backing off after a recent failure.
+    *******************************************************************************/
+   public boolean isRecordSearchBackingOff()
+   {
+      return (System.currentTimeMillis() < recordSearchBackoffUntilMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** End a record-search backoff early (an operator or test knows OpenSearch is
+    ** back).
+    *******************************************************************************/
+   public void clearRecordSearchBackoff()
+   {
+      recordSearchBackoffUntilMillis = 0L;
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the enabled flag from the quickSearchIndex row.
+    *******************************************************************************/
+   private boolean readEnabledFlag(String tableName)
+   {
+      try
+      {
+         if(QContext.getQInstance() == null || QContext.getQInstance().getTable(config.getQuickSearchIndexTableName()) == null)
+         {
+            return (true);
+         }
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(config.getQuickSearchIndexTableName());
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+         List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+
+         if(rows == null || rows.isEmpty())
+         {
+            return (true);
+         }
+
+         Boolean enabled = rows.get(0).getValueBoolean("enabled");
+         return (enabled == null || enabled);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not read enabled flag for table; treating as enabled", e, logPair("tableName", tableName));
+         return (true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Start the runtime per the configured startup mode: FAIL_FAST throws when
+    ** the index cannot be prepared; DEGRADED logs and continues. A LinkageError
+    ** (an optional library such as the AWS SDK missing while the client is
+    ** built) follows the same modes, with a message naming the missing class,
+    ** so a DEGRADED host still boots.
+    *******************************************************************************/
+   public void start() throws QException
+   {
+      try
+      {
+         ensureIndexReady();
+      }
+      catch(LinkageError e)
+      {
+         String message = "Quick Search could not load a class to build its OpenSearch client [" + e.getMessage()
+            + "]; authMode AWS_SIGV4 needs the optional AWS SDK v2 dependencies (apache-client, sts) on the classpath";
+         if(config.getStartupMode() == QuickSearchStartupMode.DEGRADED)
+         {
+            LOG.warn(message + "; Quick Search is running degraded", e, logPair("indexName", config.getOpensearchIndexName()));
+            return;
+         }
+
+         throw (new QException(message, e));
+      }
+      catch(Exception e)
+      {
+         if(config.getStartupMode() == QuickSearchStartupMode.DEGRADED)
+         {
+            LOG.warn("OpenSearch is not reachable; Quick Search is running degraded and will retry on first use", e,
+               logPair("indexName", config.getOpensearchIndexName()));
+            return;
+         }
+
+         throw (new QException("Quick Search could not prepare its OpenSearch index [" + config.getOpensearchIndexName()
+            + "]; set startupMode=DEGRADED to boot without it: " + e.getMessage(), e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Close the publisher and client. Safe to call more than once.
+    *******************************************************************************/
+   public synchronized void close()
+   {
+      if(closed)
+      {
+         return;
+      }
+      closed = true;
+
+      if(publisher != null)
+      {
+         try
+         {
+            publisher.close();
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error closing Quick Search publisher", e);
+         }
+      }
+
+      if(diagnosticEventRecorder != null)
+      {
+         diagnosticEventRecorder.close();
+      }
+
+      if(client != null)
+      {
+         client.close();
+      }
+
+      indexReady = false;
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchRuntimeService.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+
+
+/*******************************************************************************
+ ** Lifecycle hook for hosts that start through QApplicationLauncher: prepares
+ ** the OpenSearch index on start (per startupMode) and closes the transport on
+ ** stop. Hosts without a launcher get the same start behaviour from produce()
+ ** in FAIL_FAST mode and lazily on first use in DEGRADED mode.
+ *******************************************************************************/
+public class QuickSearchRuntimeService implements QRuntimeServiceInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRuntimeService.class);
+
+   private QuickSearchRuntime runtime;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getName()
+   {
+      return ("quickSearch");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void start(QInstance qInstance) throws QException
+   {
+      runtime = findRuntime(qInstance);
+      if(runtime == null)
+      {
+         LOG.warn("Quick Search runtime service started, but no produced Quick Search QBit was found in the QInstance");
+         return;
+      }
+      runtime.start();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void stop()
+   {
+      if(runtime != null)
+      {
+         runtime.close();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Find the runtime on the instance's Quick Search QBit (not through
+    ** QContext, which may not be set when the launcher starts services).
+    *******************************************************************************/
+   static QuickSearchRuntime findRuntime(QInstance qInstance)
+   {
+      if(qInstance == null || qInstance.getQBits() == null)
+      {
+         return (null);
+      }
+      for(QBitMetaData qBitMetaData : qInstance.getQBits().values())
+      {
+         if(qBitMetaData.getConfig() instanceof QuickSearchQBitConfig config && config.getRuntime() != null)
+         {
+            return (config.getRuntime());
+         }
+      }
+      return (null);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchStartupMode.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+/*******************************************************************************
+ ** What happens when OpenSearch cannot be reached while the QBit starts.
+ **
+ ** FAIL_FAST - produce() and the runtime service throw, so the host does not
+ **             boot with search silently broken. The default.
+ ** DEGRADED  - the failure is logged; the host boots; indexing and search
+ **             retry the connection on first use and report through the
+ **             health check. Intended for demo profiles that may run without
+ **             an OpenSearch cluster.
+ *******************************************************************************/
+public enum QuickSearchStartupMode
+{
+   FAIL_FAST,
+   DEGRADED
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchTlsConfig.java

@@ -0,0 +1,376 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch;
+
+
+import java.io.Serializable;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+
+/*******************************************************************************
+ ** TLS trust and client-certificate settings for the OpenSearch connection.
+ **
+ ** By default the JVM trust store and hostname verification are used. Supply
+ ** caCertificatePath (a PEM bundle) or trustStorePath to trust a private CA;
+ ** supply keyStorePath for client-certificate (mTLS) authentication.
+ ** insecureSkipVerify disables certificate and hostname checks and is refused
+ ** for non-loopback hosts unless allowInsecureInProduction is also set.
+ ** hostnameVerification=false keeps certificate checks but skips the host name
+ ** check; it is allowed for any host and logs a warning.
+ **
+ ** Path and password values may be ${env.X} / ${prop.X} references; they are
+ ** interpreted when the transport is built, never stored resolved. A path that
+ ** resolves to an empty value is treated as missing.
+ *******************************************************************************/
+public class QuickSearchTlsConfig implements Serializable
+{
+   private String  caCertificatePath;
+   private String  trustStorePath;
+   private String  trustStorePassword;
+   private String  trustStoreType;
+   private String  keyStorePath;
+   private String  keyStorePassword;
+   private String  keyStoreType;
+   private Boolean hostnameVerification      = true;
+   private Boolean insecureSkipVerify        = false;
+   private Boolean allowInsecureInProduction = false;
+
+
+
+   /*******************************************************************************
+    ** Whether any custom trust or key material is configured.
+    *******************************************************************************/
+   @JsonIgnore
+   public boolean hasCustomTrustOrKeyMaterial()
+   {
+      return (caCertificatePath != null || trustStorePath != null || keyStorePath != null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for caCertificatePath
+    *******************************************************************************/
+   public String getCaCertificatePath()
+   {
+      return (this.caCertificatePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for caCertificatePath
+    *******************************************************************************/
+   public void setCaCertificatePath(String caCertificatePath)
+   {
+      this.caCertificatePath = caCertificatePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for caCertificatePath
+    *******************************************************************************/
+   public QuickSearchTlsConfig withCaCertificatePath(String caCertificatePath)
+   {
+      this.caCertificatePath = caCertificatePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trustStorePath
+    *******************************************************************************/
+   public String getTrustStorePath()
+   {
+      return (this.trustStorePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for trustStorePath
+    *******************************************************************************/
+   public void setTrustStorePath(String trustStorePath)
+   {
+      this.trustStorePath = trustStorePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trustStorePath
+    *******************************************************************************/
+   public QuickSearchTlsConfig withTrustStorePath(String trustStorePath)
+   {
+      this.trustStorePath = trustStorePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trustStorePassword. Hidden from JSON so the secret (or its
+    ** reference) never leaves the JVM.
+    *******************************************************************************/
+   @JsonIgnore
+   public String getTrustStorePassword()
+   {
+      return (this.trustStorePassword);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for trustStorePassword
+    *******************************************************************************/
+   public void setTrustStorePassword(String trustStorePassword)
+   {
+      this.trustStorePassword = trustStorePassword;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trustStorePassword
+    *******************************************************************************/
+   public QuickSearchTlsConfig withTrustStorePassword(String trustStorePassword)
+   {
+      this.trustStorePassword = trustStorePassword;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trustStoreType (JKS or PKCS12; defaults to the JVM default)
+    *******************************************************************************/
+   public String getTrustStoreType()
+   {
+      return (this.trustStoreType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for trustStoreType
+    *******************************************************************************/
+   public void setTrustStoreType(String trustStoreType)
+   {
+      this.trustStoreType = trustStoreType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trustStoreType
+    *******************************************************************************/
+   public QuickSearchTlsConfig withTrustStoreType(String trustStoreType)
+   {
+      this.trustStoreType = trustStoreType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyStorePath
+    *******************************************************************************/
+   public String getKeyStorePath()
+   {
+      return (this.keyStorePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyStorePath
+    *******************************************************************************/
+   public void setKeyStorePath(String keyStorePath)
+   {
+      this.keyStorePath = keyStorePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyStorePath
+    *******************************************************************************/
+   public QuickSearchTlsConfig withKeyStorePath(String keyStorePath)
+   {
+      this.keyStorePath = keyStorePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyStorePassword. Hidden from JSON.
+    *******************************************************************************/
+   @JsonIgnore
+   public String getKeyStorePassword()
+   {
+      return (this.keyStorePassword);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyStorePassword
+    *******************************************************************************/
+   public void setKeyStorePassword(String keyStorePassword)
+   {
+      this.keyStorePassword = keyStorePassword;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyStorePassword
+    *******************************************************************************/
+   public QuickSearchTlsConfig withKeyStorePassword(String keyStorePassword)
+   {
+      this.keyStorePassword = keyStorePassword;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keyStoreType
+    *******************************************************************************/
+   public String getKeyStoreType()
+   {
+      return (this.keyStoreType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for keyStoreType
+    *******************************************************************************/
+   public void setKeyStoreType(String keyStoreType)
+   {
+      this.keyStoreType = keyStoreType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for keyStoreType
+    *******************************************************************************/
+   public QuickSearchTlsConfig withKeyStoreType(String keyStoreType)
+   {
+      this.keyStoreType = keyStoreType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hostnameVerification
+    *******************************************************************************/
+   public Boolean getHostnameVerification()
+   {
+      return (this.hostnameVerification);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hostnameVerification
+    *******************************************************************************/
+   public void setHostnameVerification(Boolean hostnameVerification)
+   {
+      this.hostnameVerification = hostnameVerification;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hostnameVerification
+    *******************************************************************************/
+   public QuickSearchTlsConfig withHostnameVerification(Boolean hostnameVerification)
+   {
+      this.hostnameVerification = hostnameVerification;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for insecureSkipVerify
+    *******************************************************************************/
+   public Boolean getInsecureSkipVerify()
+   {
+      return (this.insecureSkipVerify);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for insecureSkipVerify
+    *******************************************************************************/
+   public void setInsecureSkipVerify(Boolean insecureSkipVerify)
+   {
+      this.insecureSkipVerify = insecureSkipVerify;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for insecureSkipVerify
+    *******************************************************************************/
+   public QuickSearchTlsConfig withInsecureSkipVerify(Boolean insecureSkipVerify)
+   {
+      this.insecureSkipVerify = insecureSkipVerify;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for allowInsecureInProduction
+    *******************************************************************************/
+   public Boolean getAllowInsecureInProduction()
+   {
+      return (this.allowInsecureInProduction);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowInsecureInProduction
+    *******************************************************************************/
+   public void setAllowInsecureInProduction(Boolean allowInsecureInProduction)
+   {
+      this.allowInsecureInProduction = allowInsecureInProduction;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowInsecureInProduction
+    *******************************************************************************/
+   public QuickSearchTlsConfig withAllowInsecureInProduction(Boolean allowInsecureInProduction)
+   {
+      this.allowInsecureInProduction = allowInsecureInProduction;
+      return (this);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java

@@ -39,6 +39,7 @@ public class QuickSearchableTableConfig
    private Boolean             enabledByDefault;
    private String              recordLabelFormat;
    private List<String>        recordLabelFields;
+   private Integer             maxFieldLength;
 
 
 
@@ -350,4 +351,35 @@ public class QuickSearchableTableConfig
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxFieldLength
+    *******************************************************************************/
+   public Integer getMaxFieldLength()
+   {
+      return (this.maxFieldLength);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxFieldLength
+    *******************************************************************************/
+   public void setMaxFieldLength(Integer maxFieldLength)
+   {
+      this.maxFieldLength = maxFieldLength;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxFieldLength
+    *******************************************************************************/
+   public QuickSearchableTableConfig withMaxFieldLength(Integer maxFieldLength)
+   {
+      this.maxFieldLength = maxFieldLength;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java

@@ -17,116 +17,542 @@
 package com.kingsrook.qbits.quicksearch.actions;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
 import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import org.opensearch.client.opensearch._types.OpenSearchException;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.opensearch.client.opensearch.core.search.Hit;
+import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** Action that executes a quick-search query against OpenSearch and returns
- ** normalized, paginated results.
+ ** Permission-aware, bounded search over the Quick Search index.
  **
- ** Validates and normalizes the input, delegates to the
- ** QuickSearchOpenSearchClient, maps each hit to a QuickSearchResult, and
- ** computes pagination metadata (totalHits, hasMore).
+ ** Only tables the session may read (table READ permission), that are
+ ** configured and enabled, are searched. With applyRecordSecurityLocks (the
+ ** default) hits are re-read through QueryAction by primary key, so record
+ ** security locks apply and no label or snippet is returned for a record the
+ ** session cannot query.
+ **
+ ** limit is capped at maxSearchLimit and offset + limit at OpenSearch's result
+ ** window. tableNames restricts the search; limitPerTable searches each table
+ ** separately and returns up to that many hits per table.
+ **
+ ** offset counts results the caller can see. Without locks it is passed
+ ** straight to OpenSearch. With locks, hits are read from position 0 in
+ ** batches that double in size, filtered, the first offset accessible results
+ ** are skipped and limit are kept, so consecutive pages neither overlap nor
+ ** skip an accessible hit. No request reads past MAX_RESULT_WINDOW (10,000
+ ** raw hits), so accessible results beyond it are not reachable.
+ **
+ ** With locks, each scan has a hard budget of raw hits, read in batches of
+ ** at most MAX_LOCKED_BATCH_SIZE, and of OpenSearch requests, each followed
+ ** by one QueryAction per table in the batch. A single-list search gets
+ ** MAX_RESULT_WINDOW hits and MAX_LOCKED_ROUND_TRIPS requests. In per-table
+ ** mode each of the T tables gets its own equal share,
+ ** max(MAX_RESULT_WINDOW / T, min(MAX_RESULT_WINDOW, 2 x limitPerTable)) hits
+ ** and max(2, MAX_LOCKED_ROUND_TRIPS / T) requests, so a fully locked table
+ ** cannot starve another and one call reads at most
+ ** max(MAX_RESULT_WINDOW, T x 2 x limitPerTable) raw hits over at most
+ ** max(MAX_LOCKED_ROUND_TRIPS, 2 x T) requests; offset never raises it.
+ ** A budget ends a scan like the result window does: hasMore is false and
+ ** the total is a lower bound, so in per-table mode paging within a table
+ ** stops past max(MAX_RESULT_WINDOW / T, 2 x limitPerTable) raw hits. Hits
+ ** are deduplicated by table and record id within a call; batches are
+ ** separate from/size requests without a tiebreak sort, so a hit that moves
+ ** between batches during a refresh can still be missed.
  *******************************************************************************/
 public class QuickSearchAction
 {
-   private static final QLogger LOG             = QLogger.getLogger(QuickSearchAction.class);
-   private static final int     DEFAULT_LIMIT   = 25;
-   private static final int     DEFAULT_OFFSET  = 0;
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchAction.class);
+
+   public static final int DEFAULT_LIMIT       = 25;
+   public static final int MIN_TERM_LENGTH     = 2;
+   public static final int MAX_TERM_LENGTH     = 100;
+   public static final int MAX_LIMIT_PER_TABLE = 100;
+
+   public static final int MAX_LOCKED_ROUND_TRIPS = 32;
+   public static final int MAX_LOCKED_BATCH_SIZE  = 1_000;
 
 
 
    /*******************************************************************************
-    ** Execute a quick-search for the given input.
     **
-    ** Returns an empty output (totalHits=0, hasMore=false, empty results list)
-    ** when the search term is null or blank.  Otherwise normalizes the term,
-    ** applies default pagination values, calls the OpenSearch client, and maps
-    ** hits to QuickSearchResult objects.
-    **
-    ** @param input the search parameters; must not be null
-    ** @return populated QuickSearchOutput
-    ** @throws QException if the OpenSearch call fails
     *******************************************************************************/
    public QuickSearchOutput execute(QuickSearchInput input) throws QException
    {
       String searchTerm = input.getSearchTerm();
-
       if(searchTerm == null || searchTerm.isBlank())
       {
-         LOG.debug("QuickSearch called with blank/null search term - returning empty output");
-         return (new QuickSearchOutput()
-            .withResults(Collections.emptyList())
-            .withTotalHits(0L)
-            .withHasMore(false));
+         return (emptyOutput());
       }
 
       String normalizedTerm = IndexingUtils.normalizeSearchText(searchTerm);
+      if(normalizedTerm.length() < MIN_TERM_LENGTH)
+      {
+         return (emptyOutput());
+      }
+      if(normalizedTerm.length() > MAX_TERM_LENGTH)
+      {
+         throw (new QException("Search term is too long (maximum " + MAX_TERM_LENGTH + " characters)"));
+      }
 
-      int limit  = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : input.getLimit();
-      int offset = (input.getOffset() == null || input.getOffset() < 0) ? DEFAULT_OFFSET : input.getOffset();
-
-      QuickSearchOpenSearchClient       client       = QuickSearchQBitContext.getClient();
-      List<QuickSearchableTableConfig>  tableConfigs = QuickSearchQBitContext.getDiscoveredTables();
+      QuickSearchRuntime               runtime      = QuickSearchRuntime.get();
+      QuickSearchQBitConfig            config       = runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig();
+      QuickSearchOpenSearchClient      client       = runtime != null ? runtime.getClient() : QuickSearchQBitContext.getClient();
+      List<QuickSearchableTableConfig> tableConfigs = runtime != null ? runtime.getDiscoveredTables() : QuickSearchQBitContext.getDiscoveredTables();
 
       if(client == null)
       {
          throw new QException("QuickSearch client is not initialized. The QBit may not have connected to OpenSearch during startup.");
       }
+      if(tableConfigs == null)
+      {
+         tableConfigs = Collections.emptyList();
+      }
 
-      LOG.debug("Executing quick search", "term", normalizedTerm, "tableName", input.getTableName(), "limit", limit, "offset", offset);
+      int maxLimit = (config == null || config.getMaxSearchLimit() == null) ? MAX_LIMIT_PER_TABLE : config.getMaxSearchLimit();
+      int limit    = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : Math.min(input.getLimit(), maxLimit);
+      int offset   = (input.getOffset() == null || input.getOffset() < 0) ? 0 : input.getOffset();
 
-      SearchResponse<OpenSearchDocument> response = client.search(normalizedTerm, input.getTableName(), limit, offset, tableConfigs);
+      if(offset >= QuickSearchOpenSearchClient.MAX_RESULT_WINDOW)
+      {
+         return (emptyOutput());
+      }
 
-      List<QuickSearchResult> results = new ArrayList<>();
+      List<String> allowedTables = resolveAllowedTables(input, runtime, tableConfigs);
+      if(allowedTables.isEmpty())
+      {
+         return (emptyOutput());
+      }
+
+      boolean applyLocks = config == null || Boolean.TRUE.equals(config.getApplyRecordSecurityLocks());
+
+      LOG.debug("Executing quick search", logPair("termLength", normalizedTerm.length()), logPair("tables", allowedTables.size()), logPair("limit", limit), logPair("offset", offset));
+
+      if(input.getLimitPerTable() != null && input.getLimitPerTable() > 0)
+      {
+         return (searchPerTable(client, normalizedTerm, allowedTables, offset, Math.min(input.getLimitPerTable(), maxLimit), tableConfigs, applyLocks));
+      }
+
+      Page page = searchPage(client, normalizedTerm, allowedTables, offset, limit, tableConfigs, applyLocks, new ScanBudget(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW, MAX_LOCKED_ROUND_TRIPS));
+      return (page.toOutput());
+   }
+
+
+
+   /*******************************************************************************
+    ** Search each allowed table separately, skipping offset results and
+    ** returning up to limitPerTable hits in each table.
+    *******************************************************************************/
+   private QuickSearchOutput searchPerTable(QuickSearchOpenSearchClient client, String term, List<String> allowedTables, int offset, int limitPerTable, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks) throws QException
+   {
+      List<QuickSearchResult> results    = new ArrayList<>();
+      long                    totalHits  = 0;
+      boolean                 hasMore    = false;
+      boolean                 lowerBound = false;
+
+      int window     = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
+      int tableHits  = Math.max(window / allowedTables.size(), Math.min(window, 2 * limitPerTable));
+      int tableTrips = Math.max(2, MAX_LOCKED_ROUND_TRIPS / allowedTables.size());
+
+      for(String tableName : allowedTables)
+      {
+         Page page = searchPage(client, term, List.of(tableName), offset, limitPerTable, tableConfigs, applyLocks, new ScanBudget(tableHits, tableTrips));
+         results.addAll(page.results());
+         totalHits += page.totalHits();
+         hasMore |= page.hasMore();
+         lowerBound |= page.lowerBound();
+      }
+
+      return (new QuickSearchOutput().withResults(results).withTotalHits(totalHits).withTotalHitsIsLowerBound(lowerBound).withHasMore(hasMore));
+   }
+
+
+
+   /*******************************************************************************
+    ** One page over the given tables: skip offset results, keep up to limit.
+    ** Without locks offset goes straight to OpenSearch; with locks offset and
+    ** limit count accessible results within one scan budget (see the class
+    ** comment). Every request keeps from + size within MAX_RESULT_WINDOW.
+    *******************************************************************************/
+   private Page searchPage(QuickSearchOpenSearchClient client, String term, List<String> tables, int offset, int limit, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks, ScanBudget budget) throws QException
+   {
+      int window   = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
+      int pageSize = Math.min(limit, window - offset);
+      if(pageSize <= 0)
+      {
+         return (new Page(Collections.emptyList(), 0L, false, false));
+      }
+
+      if(!applyLocks)
+      {
+         SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, pageSize, offset, tableConfigs, null);
+         List<QuickSearchResult>            results  = toResults(response);
+
+         long    totalHits  = response.hits().total() != null ? response.hits().total().value() : results.size();
+         boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
+         return (new Page(results, totalHits, lowerBound, (offset + results.size()) < totalHits));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // read candidates from position 0 until offset + pageSize accessible     //
+      // results are known, the hits run out, the result window is reached, or  //
+      // the scan budget is spent. each batch over-reads 2x what is still       //
+      // needed and at least doubles the last one, up to MAX_LOCKED_BATCH_SIZE, //
+      // and the last round trip takes up to MAX_LOCKED_BATCH_SIZE of the rest. //
+      ////////////////////////////////////////////////////////////////////////////
+      int                     target     = offset + pageSize;
+      List<QuickSearchResult> accessible = new ArrayList<>();
+      int                     position   = 0;
+      int                     batchSize  = 0;
+      boolean                 exhausted  = false;
+      boolean                 rawFloor   = false;
+      Set<String>             seen       = new HashSet<>();
+      String                  preference = "quick-search-" + UUID.randomUUID();
+
+      while(accessible.size() < target && !exhausted && position < window && budget.hasRemaining())
+      {
+         int ceiling = Math.min(MAX_LOCKED_BATCH_SIZE, Math.min(window - position, budget.remainingHits));
+         batchSize = budget.remainingRoundTrips == 1 ? ceiling : Math.min(ceiling, Math.max(2 * (target - accessible.size()), 2 * batchSize));
+
+         SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, batchSize, position, tableConfigs, preference);
+         int                                rawHits  = response.hits().hits().size();
+         long                               rawTotal = response.hits().total() != null ? response.hits().total().value() : position + rawHits;
+         rawFloor = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
+
+         List<QuickSearchResult> candidates = new ArrayList<>();
+         for(QuickSearchResult result : toResults(response))
+         {
+            if(seen.add(result.getTableName() + ":" + result.getRecordId()))
+            {
+               candidates.add(result);
+            }
+         }
+         accessible.addAll(filterByRecordAccess(candidates, tableConfigs));
+         position += rawHits;
+         budget.spend(rawHits);
+         exhausted = rawHits < batchSize || position >= rawTotal;
+      }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // the raw total would reveal how many matching rows the session may not  //
+      // see; report only what was verified accessible. hasMore says only that  //
+      // more candidates exist, so a caller can page on to accessible rows.     //
+      ////////////////////////////////////////////////////////////////////////////
+      boolean truncated = accessible.size() > target;
+      boolean hasMore   = truncated || (!exhausted && position < window && budget.hasRemaining());
+
+      List<QuickSearchResult> results = accessible.size() > offset
+         ? new ArrayList<>(accessible.subList(offset, Math.min(target, accessible.size())))
+         : new ArrayList<>();
+
+      return (new Page(results, (long) accessible.size(), !exhausted || rawFloor, hasMore));
+   }
+
+
+
+   /*******************************************************************************
+    ** Requested tables (tableNames, else tableName, else all configured) that
+    ** are configured, enabled, and readable by the session.
+    *******************************************************************************/
+   private List<String> resolveAllowedTables(QuickSearchInput input, QuickSearchRuntime runtime, List<QuickSearchableTableConfig> tableConfigs)
+   {
+      Set<String> configured = new HashSet<>();
+      for(QuickSearchableTableConfig tableConfig : tableConfigs)
+      {
+         configured.add(tableConfig.getTableName());
+      }
+
+      List<String> requested = new ArrayList<>();
+      if(!CollectionUtils.nullSafeIsEmpty(input.getTableNames()))
+      {
+         requested.addAll(input.getTableNames());
+      }
+      else if(StringUtils.hasContent(input.getTableName()))
+      {
+         requested.add(input.getTableName());
+      }
+      else
+      {
+         requested.addAll(configured);
+      }
+
+      boolean      checkPermissions = QContext.getQInstance() != null && QContext.getQSession() != null;
+      List<String> allowed          = new ArrayList<>();
+      for(String tableName : new LinkedHashSet<>(requested))
+      {
+         if(!configured.contains(tableName))
+         {
+            continue;
+         }
+         if(runtime != null && !runtime.isTableEnabled(tableName))
+         {
+            continue;
+         }
+         if(checkPermissions && !PermissionsHelper.hasTablePermission(input, tableName, TablePermissionSubType.READ))
+         {
+            continue;
+         }
+         allowed.add(tableName);
+      }
+
+      return (allowed);
+   }
+
+
+
+   /*******************************************************************************
+    ** Keep only hits whose record the session can query (record security
+    ** locks). Tables not in the QInstance are left as they are.
+    *******************************************************************************/
+   private List<QuickSearchResult> filterByRecordAccess(List<QuickSearchResult> results, List<QuickSearchableTableConfig> tableConfigs) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(qInstance == null || QContext.getQSession() == null || results.isEmpty())
+      {
+         return (results);
+      }
+
+      Map<String, List<QuickSearchResult>> byTable = new LinkedHashMap<>();
+      for(QuickSearchResult result : results)
+      {
+         byTable.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
+      }
+
+      Set<QuickSearchResult> accessible = new HashSet<>();
+      for(Map.Entry<String, List<QuickSearchResult>> entry : byTable.entrySet())
+      {
+         QTableMetaData table = qInstance.getTable(entry.getKey());
+         if(table == null)
+         {
+            accessible.addAll(entry.getValue());
+            continue;
+         }
+
+         List<Serializable> ids = new ArrayList<>();
+         for(QuickSearchResult result : entry.getValue())
+         {
+            ids.add(result.getRecordId());
+         }
+
+         Set<String> found = new HashSet<>();
+         for(int start = 0; start < ids.size(); start += MAX_LOCKED_BATCH_SIZE)
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table.getName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_LOCKED_BATCH_SIZE)))));
+
+            for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+            {
+               found.add(record.getValueString(table.getPrimaryKeyField()));
+            }
+         }
+
+         for(QuickSearchResult result : entry.getValue())
+         {
+            if(found.contains(result.getRecordId()))
+            {
+               accessible.add(result);
+            }
+         }
+      }
+
+      List<QuickSearchResult> filtered = new ArrayList<>();
+      for(QuickSearchResult result : results)
+      {
+         if(accessible.contains(result))
+         {
+            filtered.add(result);
+         }
+      }
+      return (filtered);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QuickSearchResult> toResults(SearchResponse<OpenSearchDocument> response)
+   {
+      QInstance               qInstance = QContext.getQInstance();
+      List<QuickSearchResult> results   = new ArrayList<>();
 
       for(Hit<OpenSearchDocument> hit : response.hits().hits())
       {
          OpenSearchDocument source = hit.source();
+         if(source == null)
+         {
+            continue;
+         }
 
-         String highlightSnippet = null;
-         Map<String, List<String>> highlight = hit.highlight();
-
+         String                    highlightSnippet = null;
+         Map<String, List<String>> highlight        = hit.highlight();
          if(highlight != null)
          {
             List<String> fragments = highlight.get("searchableText");
-
             if(fragments != null && !fragments.isEmpty())
             {
                highlightSnippet = String.join("...", fragments);
             }
          }
 
-         Double rawScore = hit.score();
-         Float  score    = rawScore != null ? rawScore.floatValue() : null;
+         QTableMetaData table      = qInstance == null ? null : qInstance.getTable(source.getSourceTable());
+         String         tableLabel = table != null && StringUtils.hasContent(table.getLabel()) ? table.getLabel() : source.getSourceTable();
+         Double         rawScore   = hit.score();
 
          results.add(new QuickSearchResult()
             .withTableName(source.getSourceTable())
+            .withTableLabel(tableLabel)
             .withRecordId(source.getRecordId())
             .withRecordLabel(source.getRecordLabel())
-            .withScore(score)
+            .withScore(rawScore != null ? rawScore.floatValue() : null)
             .withHighlightSnippet(highlightSnippet));
       }
 
-      long    totalHits = response.hits().total() != null ? response.hits().total().value() : results.size();
-      boolean hasMore   = (offset + results.size()) < totalHits;
+      return (results);
+   }
 
-      return (new QuickSearchOutput()
-         .withResults(results)
-         .withTotalHits(totalHits)
-         .withHasMore(hasMore));
+
+
+   /*******************************************************************************
+    ** Run one search, surfacing an OpenSearch client error as a QException so
+    ** callers never see an unchecked OpenSearchException.
+    *******************************************************************************/
+   private static SearchResponse<OpenSearchDocument> runSearch(QuickSearchOpenSearchClient client, String term, List<String> tables, int fetchSize, int offset, List<QuickSearchableTableConfig> tableConfigs, String preference) throws QException
+   {
+      long start = System.currentTimeMillis();
+      try
+      {
+         return (client.search(term, tables, fetchSize, offset, tableConfigs, preference));
+      }
+      catch(OpenSearchException e)
+      {
+         QException wrapped = new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e);
+         recordFailure(tables, term, start, wrapped);
+         throw (wrapped);
+      }
+      catch(QException e)
+      {
+         recordFailure(tables, term, start, e);
+         throw (e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a failed search as a diagnostic event (the term's length only,
+    ** never the term).
+    *******************************************************************************/
+   private static void recordFailure(List<String> tables, String term, long start, Exception e)
+   {
+      DiagnosticsSupport.recordSearchFailure(QuickSearchRuntime.get(), QuickSearchDiagnosticEvent.SOURCE_QUICK_SEARCH_ACTION, tables, term,
+         System.currentTimeMillis() - start, e, "term length " + (term == null ? 0 : term.length()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Work budget for one locked scan: raw hits fetched and OpenSearch round
+    ** trips.
+    *******************************************************************************/
+   private static final class ScanBudget
+   {
+      private Integer remainingHits;
+      private Integer remainingRoundTrips;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       *******************************************************************************/
+      ScanBudget(Integer hits, Integer roundTrips)
+      {
+         this.remainingHits       = hits;
+         this.remainingRoundTrips = roundTrips;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      boolean hasRemaining()
+      {
+         return (remainingHits > 0 && remainingRoundTrips > 0);
+      }
+
+
+
+      /*******************************************************************************
+       ** Record one round trip that fetched the given number of raw hits.
+       *******************************************************************************/
+      void spend(int rawHits)
+      {
+         remainingHits -= rawHits;
+         remainingRoundTrips--;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** One page of results with its total and paging flags.
+    *******************************************************************************/
+   private record Page(List<QuickSearchResult> results, Long totalHits, Boolean lowerBound, Boolean hasMore)
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      QuickSearchOutput toOutput()
+      {
+         return (new QuickSearchOutput().withResults(results).withTotalHits(totalHits).withTotalHitsIsLowerBound(lowerBound).withHasMore(hasMore));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QuickSearchOutput emptyOutput()
+   {
+      return (new QuickSearchOutput().withResults(Collections.emptyList()).withTotalHits(0L).withTotalHitsIsLowerBound(false).withHasMore(false));
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchInput.java

@@ -17,18 +17,24 @@
 package com.kingsrook.qbits.quicksearch.actions;
 
 
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+
+
 /*******************************************************************************
  ** Input DTO for the QuickSearch action.
  **
  ** Carries the search term plus optional filtering and pagination parameters.
  *******************************************************************************/
-public class QuickSearchInput
+public class QuickSearchInput extends AbstractActionInput
 {
 
    private String  searchTerm;
    private String  tableName;
    private Integer limit;
    private Integer offset;
+   private List<String> tableNames;
+   private Integer limitPerTable;
 
 
    /*******************************************************************************
@@ -116,7 +122,14 @@ public class QuickSearchInput
 
 
    /*******************************************************************************
-    ** Getter for offset.
+    ** Getter for offset: the number of results to skip, counting only results
+    ** the session can see (with record security locks on, hits filtered out
+    ** by a lock are not counted). With limitPerTable it applies within each
+    ** table. Results are reachable only within the first 10,000 raw hits; in
+    ** per-table mode with locks each table gets an equal share of that scan
+    ** budget, so paging within a table stops past
+    ** max(10,000 / T, 2 x limitPerTable) raw hits (hasMore turns false and
+    ** totalHitsIsLowerBound stays true).
     *******************************************************************************/
    public Integer getOffset()
    {
@@ -142,4 +155,65 @@ public class QuickSearchInput
       return this;
    }
 
+
+   /*******************************************************************************
+    ** Getter for tableNames (restrict the search to these tables)
+    *******************************************************************************/
+   public List<String> getTableNames()
+   {
+      return tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableNames
+    *******************************************************************************/
+   public void setTableNames(List<String> tableNames)
+   {
+      this.tableNames = tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableNames
+    *******************************************************************************/
+   public QuickSearchInput withTableNames(List<String> tableNames)
+   {
+      this.tableNames = tableNames;
+      return this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for limitPerTable (search each table separately, up to this many)
+    *******************************************************************************/
+   public Integer getLimitPerTable()
+   {
+      return limitPerTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for limitPerTable
+    *******************************************************************************/
+   public void setLimitPerTable(Integer limitPerTable)
+   {
+      this.limitPerTable = limitPerTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for limitPerTable
+    *******************************************************************************/
+   public QuickSearchInput withLimitPerTable(Integer limitPerTable)
+   {
+      this.limitPerTable = limitPerTable;
+      return this;
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchOutput.java

@@ -31,6 +31,7 @@ public class QuickSearchOutput
 
    private List<QuickSearchResult> results;
    private Long                    totalHits;
+   private Boolean totalHitsIsLowerBound = false;
    private Boolean                 hasMore;
 
 
@@ -117,4 +118,36 @@ public class QuickSearchOutput
       return this;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for totalHitsIsLowerBound: true when totalHits is a floor (hits
+    ** beyond the result window, or hits removed by permission filtering)
+    *******************************************************************************/
+   public Boolean getTotalHitsIsLowerBound()
+   {
+      return totalHitsIsLowerBound;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for totalHitsIsLowerBound
+    *******************************************************************************/
+   public void setTotalHitsIsLowerBound(Boolean totalHitsIsLowerBound)
+   {
+      this.totalHitsIsLowerBound = totalHitsIsLowerBound;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for totalHitsIsLowerBound
+    *******************************************************************************/
+   public QuickSearchOutput withTotalHitsIsLowerBound(Boolean totalHitsIsLowerBound)
+   {
+      this.totalHitsIsLowerBound = totalHitsIsLowerBound;
+      return this;
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchRecordSearchProvider.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.actions;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Serves QQQ core record search (RecordSearchAction, POST /qqq/v1/search,
+ ** the Next UI global search) from the Quick Search index, for the tables
+ ** this QBit indexes. Registered on the QInstance by QuickSearchQBitProducer
+ ** when serveCoreRecordSearch is on (the default).
+ **
+ ** Core creates a new instance for every search, so this class holds no
+ ** state: it resolves the QBit's QuickSearchRuntime (client, tables, config)
+ ** from QContext.
+ **
+ ** A table is claimed when it is indexed by this QBit and enabled, and core
+ ** record search is not backing off after a recent failure. search() sends
+ ** one _msearch request with a top-N search per table and returns record ids
+ ** only, best first; core re-reads them by primary key through QueryAction,
+ ** so table permissions, record security locks and record labels are core's
+ ** job, and ids of deleted records drop out. Each table gets
+ ** IDS_PER_LIMIT_MULTIPLIER x limitPerTable ids (at most
+ ** RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE) so dropouts do not leave
+ ** the table short.
+ **
+ ** The request is bounded by recordSearchTimeoutMillis. Any failure throws,
+ ** and core then searches the claimed tables with its own LIKE search; the
+ ** failure also starts a short backoff (QuickSearchRuntime
+ ** RECORD_SEARCH_BACKOFF_MILLIS) during which no tables are claimed.
+ *******************************************************************************/
+public class QuickSearchRecordSearchProvider implements RecordSearchProviderInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordSearchProvider.class);
+
+   public static final int IDS_PER_LIMIT_MULTIPLIER = 4;
+
+
+
+   /*******************************************************************************
+    ** Claim tables this QBit indexes and that are enabled, unless core record
+    ** search is off or backing off. Cheap: an in-memory lookup plus the
+    ** runtime's cached enabled flag.
+    *******************************************************************************/
+   @Override
+   public boolean claimsTable(QTableMetaData table)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || table == null || !Boolean.TRUE.equals(runtime.getConfig().getServeCoreRecordSearch()))
+      {
+         return (false);
+      }
+
+      if(runtime.getTableConfig(table.getName()) == null)
+      {
+         return (false);
+      }
+
+      if(runtime.isRecordSearchBackingOff())
+      {
+         runtime.countBackoffDeclinedClaim();
+         return (false);
+      }
+
+      return (runtime.isTableEnabled(table.getName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Ids of the best matches in each requested table, in relevance order.
+    *******************************************************************************/
+   @Override
+   public RecordSearchOutput search(RecordSearchInput input) throws QException
+   {
+      QuickSearchRuntime    runtime = QuickSearchRuntime.getOrThrow();
+      QuickSearchQBitConfig config  = runtime.getConfig();
+
+      List<RecordSearchResult> results = new ArrayList<>();
+      String                   term    = IndexingUtils.normalizeSearchText(input.getSearchTerm());
+      if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         return (new RecordSearchOutput().withResults(results));
+      }
+
+      List<String> tableNames = new ArrayList<>();
+      for(String tableName : new LinkedHashSet<>(CollectionUtils.nonNullList(input.getTableNames())))
+      {
+         if(runtime.getTableConfig(tableName) != null)
+         {
+            tableNames.add(tableName);
+         }
+      }
+      if(tableNames.isEmpty())
+      {
+         return (new RecordSearchOutput().withResults(results));
+      }
+
+      int limitPerTable = input.getLimitPerTable() == null || input.getLimitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : input.getLimitPerTable();
+      int idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * IDS_PER_LIMIT_MULTIPLIER);
+      int timeoutMillis = config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis();
+
+      Map<String, List<String>> idsByTable;
+      long                      start = System.currentTimeMillis();
+      try
+      {
+         idsByTable = runtime.getClient().searchRecordIdsPerTable(term, tableNames, idsPerTable, runtime.getDiscoveredTables(), timeoutMillis);
+      }
+      catch(QException | RuntimeException e)
+      {
+         runtime.markRecordSearchFailed(e.getMessage());
+
+         //////////////////////////////////////////////////////////////////////
+         // the user's term is not recorded (it may hold personal data); its //
+         // length is, so a length-related failure is still visible          //
+         //////////////////////////////////////////////////////////////////////
+         DiagnosticsSupport.recordSearchFailure(runtime, QuickSearchDiagnosticEvent.SOURCE_CORE_RECORD_SEARCH, tableNames, term, System.currentTimeMillis() - start, e,
+            "term length " + term.length() + ", " + idsPerTable + " ids per table, timeout " + timeoutMillis + " ms; core searched these tables itself");
+         LOG.info("Quick Search record search failed; core searches these tables itself, and Quick Search claims none for a while", logPair("tables", tableNames),
+            logPair("backoffMillis", QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS), logPair("error", e.getMessage()));
+         throw (e instanceof QException qe ? qe : new QException("Quick Search record search failed: " + e.getMessage(), e));
+      }
+
+      for(String tableName : tableNames)
+      {
+         for(String recordId : CollectionUtils.nonNullList(idsByTable.get(tableName)))
+         {
+            results.add(new RecordSearchResult().withTableName(tableName).withRecordId(recordId));
+         }
+      }
+
+      return (new RecordSearchOutput().withResults(results));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java

@@ -25,6 +25,7 @@ public class QuickSearchResult
 {
 
    private String tableName;
+   private String tableLabel;
    private String recordId;
    private String recordLabel;
    private Float  score;
@@ -170,4 +171,35 @@ public class QuickSearchResult
       return this;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for tableLabel
+    *******************************************************************************/
+   public String getTableLabel()
+   {
+      return tableLabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableLabel
+    *******************************************************************************/
+   public void setTableLabel(String tableLabel)
+   {
+      this.tableLabel = tableLabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableLabel
+    *******************************************************************************/
+   public QuickSearchResult withTableLabel(String tableLabel)
+   {
+      this.tableLabel = tableLabel;
+      return this;
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizer.java

@@ -1,106 +0,0 @@
-/*
- * Copyright 2024 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.quicksearch.customizers;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
-import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Post-delete table customizer that removes deleted records from the Quick
- ** Search OpenSearch index.
- **
- ** QQQ instantiates this class via reflection using the no-arg constructor.
- ** The publisher and table configuration are obtained from the static
- ** QuickSearchQBitContext at call time.
- *******************************************************************************/
-public class QuickSearchPostDeleteCustomizer implements TableCustomizerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(QuickSearchPostDeleteCustomizer.class);
-
-
-
-   /*******************************************************************************
-    ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
-    *******************************************************************************/
-   public QuickSearchPostDeleteCustomizer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Post-delete hook: builds DELETE events for each deleted record and publishes
-    ** them via the configured IndexEventPublisher.
-    **
-    ** If the publisher is null (QBit not yet initialized) or if publishing fails,
-    ** a warning is logged and the original records are returned without throwing,
-    ** so as never to block the underlying QQQ delete operation.
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
-   {
-      if(records == null || records.isEmpty())
-      {
-         return records;
-      }
-
-      IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
-      if(publisher == null)
-      {
-         LOG.warn("QuickSearch publisher is null; skipping post-delete indexing", logPair("tableName", deleteInput.getTableName()));
-         return records;
-      }
-
-      try
-      {
-         String tableName = deleteInput.getTableName();
-         QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
-         String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
-
-         List<IndexEvent> events = new ArrayList<>();
-         for(QRecord record : records)
-         {
-            events.add(new IndexEvent()
-               .withAction(IndexEventAction.DELETE)
-               .withTableName(tableName)
-               .withRecordId(record.getValueString(primaryKeyField)));
-         }
-
-         publisher.publishDeleteEvents(events);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to publish post-delete index events", e, logPair("tableName", deleteInput.getTableName()));
-      }
-
-      return records;
-   }
-
-}

src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizer.java

@@ -1,107 +0,0 @@
-/*
- * Copyright 2024 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.quicksearch.customizers;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
-import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Post-insert table customizer that indexes newly inserted records in the
- ** Quick Search OpenSearch index.
- **
- ** QQQ instantiates this class via reflection using the no-arg constructor.
- ** The publisher and table configuration are obtained from the static
- ** QuickSearchQBitContext at call time.
- *******************************************************************************/
-public class QuickSearchPostInsertCustomizer implements TableCustomizerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(QuickSearchPostInsertCustomizer.class);
-
-
-
-   /*******************************************************************************
-    ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
-    *******************************************************************************/
-   public QuickSearchPostInsertCustomizer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Post-insert hook: builds INDEX events for each inserted record and publishes
-    ** them via the configured IndexEventPublisher.
-    **
-    ** If the publisher is null (QBit not yet initialized) or if publishing fails,
-    ** a warning is logged and the original records are returned without throwing,
-    ** so as never to block the underlying QQQ insert operation.
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
-   {
-      if(records == null || records.isEmpty())
-      {
-         return records;
-      }
-
-      IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
-      if(publisher == null)
-      {
-         LOG.warn("QuickSearch publisher is null; skipping post-insert indexing", logPair("tableName", insertInput.getTableName()));
-         return records;
-      }
-
-      try
-      {
-         String tableName = insertInput.getTableName();
-         QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
-         String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
-
-         List<IndexEvent> events = new ArrayList<>();
-         for(QRecord record : records)
-         {
-            events.add(new IndexEvent()
-               .withAction(IndexEventAction.INDEX)
-               .withTableName(tableName)
-               .withRecordId(record.getValueString(primaryKeyField))
-               .withRecord(record));
-         }
-
-         publisher.publishIndexEvents(events);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to publish post-insert index events", e, logPair("tableName", insertInput.getTableName()));
-      }
-
-      return records;
-   }
-
-}

src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizer.java

@@ -1,108 +0,0 @@
-/*
- * Copyright 2024 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.quicksearch.customizers;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
-import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
-import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Post-update table customizer that re-indexes updated records in the Quick
- ** Search OpenSearch index.
- **
- ** QQQ instantiates this class via reflection using the no-arg constructor.
- ** The publisher and table configuration are obtained from the static
- ** QuickSearchQBitContext at call time.
- *******************************************************************************/
-public class QuickSearchPostUpdateCustomizer implements TableCustomizerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(QuickSearchPostUpdateCustomizer.class);
-
-
-
-   /*******************************************************************************
-    ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
-    *******************************************************************************/
-   public QuickSearchPostUpdateCustomizer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Post-update hook: builds INDEX events for each updated record and publishes
-    ** them via the configured IndexEventPublisher.
-    **
-    ** If the publisher is null (QBit not yet initialized) or if publishing fails,
-    ** a warning is logged and the original records are returned without throwing,
-    ** so as never to block the underlying QQQ update operation.
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      if(records == null || records.isEmpty())
-      {
-         return records;
-      }
-
-      IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
-      if(publisher == null)
-      {
-         LOG.warn("QuickSearch publisher is null; skipping post-update indexing", logPair("tableName", updateInput.getTableName()));
-         return records;
-      }
-
-      try
-      {
-         String tableName = updateInput.getTableName();
-         QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
-         String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
-
-         List<IndexEvent> events = new ArrayList<>();
-         for(QRecord record : records)
-         {
-            events.add(new IndexEvent()
-               .withAction(IndexEventAction.INDEX)
-               .withTableName(tableName)
-               .withRecordId(record.getValueString(primaryKeyField))
-               .withRecord(record));
-         }
-
-         publisher.publishIndexEvents(events);
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Failed to publish post-update index events", e, logPair("tableName", updateInput.getTableName()));
-      }
-
-      return records;
-   }
-
-}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/AnalyzerTester.java

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+
+
+/*******************************************************************************
+ ** The analyzer tester: run _analyze on the live index with its two
+ ** analyzers. The text is analyzed as it would be indexed (standard
+ ** tokenizer, lowercase, edge n-grams of 2 to 20 characters) and as a query
+ ** (standard tokenizer, lowercase); with a query, each query token is marked
+ ** found or not among the text's indexed tokens, which is how matching works
+ ** (every query token must be found). Read-only.
+ *******************************************************************************/
+public class AnalyzerTester
+{
+
+   /*******************************************************************************
+    ** The tokens of the text (both analyzers) and of the query.
+    *******************************************************************************/
+   public record Result(String text, String query, List<AnalyzedToken> indexTokens, List<AnalyzedToken> textAsQueryTokens, List<AnalyzedToken> queryTokens)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Analyze text (and an optional query) with the index's analyzers.
+    *******************************************************************************/
+   public Result analyze(QuickSearchRuntime runtime, String text, String query) throws QException
+   {
+      QuickSearchDiagnosticsOperations ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      String                           safeText = text == null ? "" : text;
+      return (new Result(safeText, query,
+         ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, safeText),
+         ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, safeText),
+         StringUtils.hasContent(query) ? ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, query) : null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render a result as HTML.
+    *******************************************************************************/
+   public static String render(Result result)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Analyzer tokens");
+      if(result.text().length() > QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH)
+      {
+         html.status(Level.WARN, "Only the first " + QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH + " characters were analyzed.");
+      }
+
+      Set<String> indexed = new LinkedHashSet<>();
+      result.indexTokens().forEach(t -> indexed.add(t.token()));
+
+      if(result.queryTokens() != null)
+      {
+         html.subheading("Does the query match this text?");
+         List<DiagnosticsHtml.Check> checks = new ArrayList<>();
+         boolean                     all    = !result.queryTokens().isEmpty();
+         for(AnalyzedToken token : result.queryTokens())
+         {
+            boolean found = indexed.contains(token.token());
+            all &= found;
+            String why = found ? "found among the indexed tokens"
+               : token.token().length() < 2 ? "one character: never indexed (n-grams start at 2)"
+               : token.token().length() > 20 ? "longer than 20 characters: never indexed (n-grams stop at 20)"
+               : "no word in the text starts with it";
+            checks.add(new DiagnosticsHtml.Check("\"" + token.token() + "\"", found ? Level.OK : Level.FAIL, why));
+         }
+         html.status(all ? Level.OK : Level.FAIL, all ? "Every query token is found: the searchable-text clause matches."
+            : "At least one query token is missing: with AND semantics, a document with only this text does not match.");
+         html.checks(checks);
+      }
+
+      html.subheading("As indexed (" + QuickSearchOpenSearchClient.INDEX_ANALYZER + "): " + result.indexTokens().size() + " token(s)");
+      html.paragraph(join(result.indexTokens()));
+      html.subheading("As a query (" + QuickSearchOpenSearchClient.SEARCH_ANALYZER + "): " + result.textAsQueryTokens().size() + " token(s)");
+      html.paragraph(join(result.textAsQueryTokens()));
+      if(result.queryTokens() != null)
+      {
+         html.subheading("Query tokens: " + result.queryTokens().size());
+         html.paragraph(join(result.queryTokens()));
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Tokens as "token (position)" separated by commas.
+    *******************************************************************************/
+   static String join(List<AnalyzedToken> tokens)
+   {
+      if(tokens == null || tokens.isEmpty())
+      {
+         return ("(none)");
+      }
+      List<String> parts = new ArrayList<>();
+      for(AnalyzedToken token : tokens)
+      {
+         parts.add(token.token() + " @" + token.position());
+      }
+      return (String.join(", ", parts));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticEventRecorder.java

@@ -0,0 +1,259 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.net.InetAddress;
+import java.time.Instant;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Writes quickSearchDiagnosticEvent rows: failed searches, diagnostic tool
+ ** runs (the audit trail) and refused runs.
+ ** Best effort and off the caller's thread: events are queued (at most
+ ** QUEUE_CAPACITY) and inserted by one daemon thread as the system user, so
+ ** a slow or failing database never slows or fails a search. At most
+ ** MAX_EVENTS_PER_MINUTE events are written per JVM per minute; the rest are
+ ** counted as dropped (see getDroppedCount), so a search outage cannot flood
+ ** the table. A no-op when the diagnostic tables are not configured.
+ *******************************************************************************/
+public class DiagnosticEventRecorder implements AutoCloseable
+{
+   private static final QLogger LOG = QLogger.getLogger(DiagnosticEventRecorder.class);
+
+   public static final int MAX_EVENTS_PER_MINUTE = 120;
+   public static final int QUEUE_CAPACITY        = 500;
+
+   private final QuickSearchQBitConfig config;
+   private final ThreadPoolExecutor    executor;
+   private final AtomicLong            droppedCount = new AtomicLong();
+   private final AtomicLong            failedCount  = new AtomicLong();
+   private final String                hostName;
+
+   private long windowStartMillis = 0L;
+   private int  windowCount       = 0;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public DiagnosticEventRecorder(QuickSearchQBitConfig config)
+   {
+      this.config   = config;
+      this.hostName = resolveHostName();
+      this.executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(QUEUE_CAPACITY), runnable ->
+      {
+         Thread thread = new Thread(runnable, "quick-search-diagnostic-events");
+         thread.setDaemon(true);
+         return (thread);
+      }, (runnable, pool) -> droppedCount.incrementAndGet());
+   }
+
+
+
+   /*******************************************************************************
+    ** Queue an event for insert. Fills in host name and create date, bounds
+    ** string lengths, and uses the current session's user when none is set.
+    *******************************************************************************/
+   public void record(QuickSearchDiagnosticEvent event)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      String    tableName = config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
+      if(event == null || qInstance == null || !Boolean.TRUE.equals(config.getEnableDiagnosticTools()) || qInstance.getTable(tableName) == null)
+      {
+         return;
+      }
+
+      if(!admit())
+      {
+         droppedCount.incrementAndGet();
+         return;
+      }
+
+      if(event.getUserName() == null)
+      {
+         event.setUserName(DiagnosticsSupport.currentUserName());
+      }
+
+      event.withHostName(truncate(hostName, 100))
+         .withCreateDate(Instant.now())
+         .withSource(truncate(event.getSource(), 50))
+         .withUserName(truncate(event.getUserName(), 250))
+         .withTableNames(truncate(event.getTableNames(), 1000))
+         .withSearchTerm(truncate(event.getSearchTerm(), 100))
+         .withErrorClass(truncate(event.getErrorClass(), 250))
+         .withErrorMessage(truncate(event.getErrorMessage(), 4000))
+         .withDetail(truncate(event.getDetail(), 4000));
+
+      executor.execute(() -> insert(qInstance, tableName, event));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one event as the system user. Failures are logged and counted.
+    *******************************************************************************/
+   private void insert(QInstance qInstance, String tableName, QuickSearchDiagnosticEvent event)
+   {
+      QSession session = new QSystemUserSession();
+      try
+      {
+         QContext.init(qInstance, session);
+         new InsertAction().execute(new InsertInput(tableName).withRecordEntity(event));
+      }
+      catch(Exception e)
+      {
+         failedCount.incrementAndGet();
+         LOG.warn("Could not record a Quick Search diagnostic event", e, logPair("eventType", event.getEventType()), logPair("source", event.getSource()));
+      }
+      finally
+      {
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether one more event fits in the current minute's budget.
+    *******************************************************************************/
+   private synchronized boolean admit()
+   {
+      long now = System.currentTimeMillis();
+      if(now - windowStartMillis >= 60_000L)
+      {
+         windowStartMillis = now;
+         windowCount       = 0;
+      }
+      if(windowCount >= MAX_EVENTS_PER_MINUTE)
+      {
+         return (false);
+      }
+      windowCount++;
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Wait (up to timeoutMillis) until every queued event has been written.
+    ** For tests and orderly shutdown.
+    *******************************************************************************/
+   public void flush(long timeoutMillis)
+   {
+      try
+      {
+         executor.submit(() ->
+         {
+         }).get(timeoutMillis, TimeUnit.MILLISECONDS);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Diagnostic event flush did not complete", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Events not written because the queue or the per-minute budget was full.
+    *******************************************************************************/
+   public long getDroppedCount()
+   {
+      return (droppedCount.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Events whose insert failed.
+    *******************************************************************************/
+   public long getFailedCount()
+   {
+      return (failedCount.get());
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop the writer after the queued events (up to 2 seconds).
+    *******************************************************************************/
+   @Override
+   public void close()
+   {
+      executor.shutdown();
+      try
+      {
+         if(!executor.awaitTermination(2, TimeUnit.SECONDS))
+         {
+            executor.shutdownNow();
+         }
+      }
+      catch(InterruptedException e)
+      {
+         executor.shutdownNow();
+         Thread.currentThread().interrupt();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** This JVM's host name (the pod name in Kubernetes).
+    *******************************************************************************/
+   static String resolveHostName()
+   {
+      String fromEnv = System.getenv("HOSTNAME");
+      if(fromEnv != null && !fromEnv.isBlank())
+      {
+         return (fromEnv);
+      }
+      try
+      {
+         return (InetAddress.getLocalHost().getHostName());
+      }
+      catch(Exception e)
+      {
+         return ("unknown");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   static String truncate(String value, int max)
+   {
+      return (value == null || value.length() <= max ? value : value.substring(0, max));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticRecordReads.java

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Source-table reads for the diagnostics, in the current session (so table
+ ** permissions and record security locks apply, as for any query the user
+ ** runs). Records are read the way indexing reads them: display values
+ ** generated and possible values translated, so a document built from them
+ ** is exactly the document indexing would write.
+ *******************************************************************************/
+public final class DiagnosticRecordReads
+{
+   public static final int MAX_IDS_PER_QUERY = 1_000;
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private DiagnosticRecordReads()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Records of a table by primary key (as strings), keyed by primary key
+    ** string, read as indexing reads them. Ids that do not exist or that the
+    ** session cannot read are absent. Ids that do not convert to the key's
+    ** type are skipped.
+    *******************************************************************************/
+   public static Map<String, QRecord> loadForIndexing(String tableName, Collection<String> recordIds) throws QException
+   {
+      QTableMetaData       table   = requireTable(tableName);
+      String               pk      = table.getPrimaryKeyField();
+      Map<String, QRecord> records = new LinkedHashMap<>();
+      List<Serializable>   ids     = typedIds(table, recordIds);
+
+      for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
+         queryInput.setShouldGenerateDisplayValues(true);
+         queryInput.setShouldTranslatePossibleValues(true);
+         for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+         {
+            records.put(record.getValueString(pk), record);
+         }
+      }
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Up to limit records of a table ordered by primary key (ascending or
+    ** descending), read as indexing reads them.
+    *******************************************************************************/
+   public static List<QRecord> loadEdge(String tableName, boolean ascending, int limit) throws QException
+   {
+      QTableMetaData table      = requireTable(tableName);
+      QueryInput     queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(table.getPrimaryKeyField(), ascending)).withLimit(limit));
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldTranslatePossibleValues(true);
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The primary keys (as strings) among recordIds that exist and that the
+    ** session can read.
+    *******************************************************************************/
+   public static List<String> existingIds(String tableName, Collection<String> recordIds) throws QException
+   {
+      QTableMetaData     table  = requireTable(tableName);
+      String             pk     = table.getPrimaryKeyField();
+      List<String>       found  = new ArrayList<>();
+      List<Serializable> ids    = typedIds(table, recordIds);
+
+      for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
+         for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
+         {
+            found.add(record.getValueString(pk));
+         }
+      }
+      return (found);
+   }
+
+
+
+   /*******************************************************************************
+    ** Row count of a table in the current session.
+    *******************************************************************************/
+   public static Integer count(String tableName) throws QException
+   {
+      CountInput countInput = new CountInput();
+      countInput.setTableName(tableName);
+      return (new CountAction().execute(countInput).getCount());
+   }
+
+
+
+   /*******************************************************************************
+    ** Ids converted to the primary key's type; ids that do not convert are
+    ** skipped (they cannot exist).
+    *******************************************************************************/
+   static List<Serializable> typedIds(QTableMetaData table, Collection<String> recordIds)
+   {
+      QFieldMetaData     pkField = table.getField(table.getPrimaryKeyField());
+      List<Serializable> ids     = new ArrayList<>();
+      for(String recordId : CollectionUtils.nonNullCollection(recordIds))
+      {
+         try
+         {
+            Serializable id = ValueUtils.getValueAsFieldType(pkField.getType(), recordId);
+            if(id != null && !ids.contains(id))
+            {
+               ids.add(id);
+            }
+         }
+         catch(Exception e)
+         {
+            //////////////////////////////////////////////////////////
+            // e.g. "abc" for an integer key: no such record exists //
+            //////////////////////////////////////////////////////////
+         }
+      }
+      return (ids);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QTableMetaData requireTable(String tableName) throws QException
+   {
+      QTableMetaData table = QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw (new QException("Table [" + tableName + "] is not in the instance"));
+      }
+      return (table);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsHtml.java

@@ -0,0 +1,362 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.List;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+
+/*******************************************************************************
+ ** A small HTML builder for the diagnostic screens (process HTML blocks, the
+ ** validation report field and the diagnostics widget). Every text value is
+ ** escaped. Layout rules: one column of blocks capped at a readable width,
+ ** no fixed widths, and long values (JSON, ids, text) wrap instead of
+ ** scrolling, so a screen reads from 360px to 3200px without an inner or
+ ** horizontal scrollbar. Colours are mid-tone so they read in light and dark
+ ** themes; status is always also spelled out in text.
+ *******************************************************************************/
+public class DiagnosticsHtml
+{
+   public static final int MAX_JSON_CHARS = 60_000;
+
+   private static final ObjectMapper PRETTY = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+
+   private static final String WRAP = "overflow-wrap:anywhere;word-break:break-word;";
+
+   private final StringBuilder html = new StringBuilder();
+
+
+
+   /*******************************************************************************
+    ** Severity of a check or a line, rendered as a coloured label.
+    *******************************************************************************/
+   public enum Level
+   {
+      OK("OK", "#16a34a"),
+      INFO("INFO", "#2563eb"),
+      WARN("WARN", "#d97706"),
+      FAIL("FAIL", "#dc2626");
+
+      private final String label;
+      private final String color;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       *******************************************************************************/
+      Level(String label, String color)
+      {
+         this.label = label;
+         this.color = color;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for label
+       *******************************************************************************/
+      public String getLabel()
+      {
+         return (label);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Escape text for HTML element content and attribute values.
+    *******************************************************************************/
+   public static String escape(Object value)
+   {
+      if(value == null)
+      {
+         return ("");
+      }
+      String        text = String.valueOf(value);
+      StringBuilder out  = new StringBuilder(text.length() + 16);
+      for(int i = 0; i < text.length(); i++)
+      {
+         char c = text.charAt(i);
+         switch(c)
+         {
+            case '<' -> out.append("&lt;");
+            case '>' -> out.append("&gt;");
+            case '&' -> out.append("&amp;");
+            case '"' -> out.append("&quot;");
+            case '\'' -> out.append("&#39;");
+            default -> out.append(c);
+         }
+      }
+      return (out.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Escape text but keep OpenSearch highlight tags (em) as mark elements.
+    *******************************************************************************/
+   public static String highlight(String fragment)
+   {
+      return (escape(fragment).replace("&lt;em&gt;", "<mark>").replace("&lt;/em&gt;", "</mark>"));
+   }
+
+
+
+   /*******************************************************************************
+    ** A coloured status label.
+    *******************************************************************************/
+   public static String badge(Level level)
+   {
+      return ("<strong style=\"color:" + level.color + "\">" + level.label + "</strong>");
+   }
+
+
+
+   /*******************************************************************************
+    ** Open the root block.
+    *******************************************************************************/
+   public DiagnosticsHtml open()
+   {
+      html.append("<div style=\"max-width:72rem;").append(WRAP).append("\">");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Close the root block.
+    *******************************************************************************/
+   public DiagnosticsHtml close()
+   {
+      html.append("</div>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A section heading.
+    *******************************************************************************/
+   public DiagnosticsHtml heading(String text)
+   {
+      html.append("<h3>").append(escape(text)).append("</h3>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A sub-heading.
+    *******************************************************************************/
+   public DiagnosticsHtml subheading(String text)
+   {
+      html.append("<h4>").append(escape(text)).append("</h4>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A paragraph of escaped text.
+    *******************************************************************************/
+   public DiagnosticsHtml paragraph(String text)
+   {
+      html.append("<p>").append(escape(text)).append("</p>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A paragraph starting with a status label.
+    *******************************************************************************/
+   public DiagnosticsHtml status(Level level, String text)
+   {
+      html.append("<p>").append(badge(level)).append(" ").append(escape(text)).append("</p>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A list of checks: each line a status label and text.
+    *******************************************************************************/
+   public DiagnosticsHtml checks(List<Check> checks)
+   {
+      if(checks == null || checks.isEmpty())
+      {
+         return (this);
+      }
+      html.append("<ul>");
+      for(Check check : checks)
+      {
+         html.append("<li>").append(badge(check.level())).append(" <strong>").append(escape(check.name())).append("</strong>");
+         if(check.message() != null)
+         {
+            html.append(": ").append(escape(check.message()));
+         }
+         html.append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A list of label: value lines (values escaped). Null values show as
+    ** "none".
+    *******************************************************************************/
+   public DiagnosticsHtml facts(List<String[]> facts)
+   {
+      html.append("<ul>");
+      for(String[] fact : facts)
+      {
+         html.append("<li><strong>").append(escape(fact[0])).append(":</strong> ")
+            .append(fact.length > 1 && fact[1] != null ? escape(fact[1]) : "<em>none</em>").append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A bulleted list of pre-built HTML items (callers escape their parts).
+    *******************************************************************************/
+   public DiagnosticsHtml rawList(List<String> itemsHtml)
+   {
+      if(itemsHtml == null || itemsHtml.isEmpty())
+      {
+         return (this);
+      }
+      html.append("<ul>");
+      for(String item : itemsHtml)
+      {
+         html.append("<li>").append(item).append("</li>");
+      }
+      html.append("</ul>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A preformatted block that wraps (no horizontal scrolling).
+    *******************************************************************************/
+   public DiagnosticsHtml pre(String text)
+   {
+      html.append("<pre style=\"white-space:pre-wrap;").append(WRAP).append("\">").append(escape(text)).append("</pre>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A collapsed block with a summary line and pretty-printed JSON inside.
+    *******************************************************************************/
+   public DiagnosticsHtml jsonDetails(String summary, String json)
+   {
+      html.append("<details><summary>").append(escape(summary)).append("</summary>");
+      pre(prettyJson(json));
+      html.append("</details>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A collapsed block with a summary line and preformatted text inside.
+    *******************************************************************************/
+   public DiagnosticsHtml textDetails(String summary, String text)
+   {
+      html.append("<details><summary>").append(escape(summary)).append("</summary>");
+      pre(text);
+      html.append("</details>");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Append pre-built HTML (callers escape their parts).
+    *******************************************************************************/
+   public DiagnosticsHtml raw(String rawHtml)
+   {
+      html.append(rawHtml);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Pretty-print JSON (or return the text as-is when it is not JSON),
+    ** truncated to MAX_JSON_CHARS.
+    *******************************************************************************/
+   public static String prettyJson(String json)
+   {
+      if(json == null)
+      {
+         return ("");
+      }
+      String pretty;
+      try
+      {
+         pretty = PRETTY.writeValueAsString(PRETTY.readTree(json));
+      }
+      catch(Exception e)
+      {
+         pretty = json;
+      }
+      return (pretty.length() > MAX_JSON_CHARS ? pretty.substring(0, MAX_JSON_CHARS) + "\n... (truncated at " + MAX_JSON_CHARS + " characters)" : pretty);
+   }
+
+
+
+   /*******************************************************************************
+    ** Pretty-print a JSON node.
+    *******************************************************************************/
+   public static String prettyJson(JsonNode node)
+   {
+      return (node == null ? "" : prettyJson(node.toString()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The built HTML.
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return (html.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** One check: name, level and message.
+    *******************************************************************************/
+   public record Check(String name, Level level, String message)
+   {
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsRateLimiter.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.LongSupplier;
+
+
+/*******************************************************************************
+ ** Sliding one-minute window of diagnostic tool runs per key (user and
+ ** tool), per JVM. tryAcquire admits a run when fewer than the limit ran in
+ ** the last 60 seconds.
+ *******************************************************************************/
+public class DiagnosticsRateLimiter
+{
+   private static final long WINDOW_MILLIS = 60_000L;
+
+   private final Map<String, Deque<Long>> runsByKey = new ConcurrentHashMap<>();
+   private final LongSupplier             clock;
+
+
+
+   /*******************************************************************************
+    ** Constructor (system clock)
+    *******************************************************************************/
+   public DiagnosticsRateLimiter()
+   {
+      this(System::currentTimeMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor with a clock (tests)
+    *******************************************************************************/
+   public DiagnosticsRateLimiter(LongSupplier clock)
+   {
+      this.clock = clock;
+   }
+
+
+
+   /*******************************************************************************
+    ** Admit one run for the key when fewer than perMinute ran in the last
+    ** minute; a null or non-positive limit admits everything.
+    *******************************************************************************/
+   public boolean tryAcquire(String key, Integer perMinute)
+   {
+      if(perMinute == null || perMinute <= 0)
+      {
+         return (true);
+      }
+
+      long        now  = clock.getAsLong();
+      Deque<Long> runs = runsByKey.computeIfAbsent(key, k -> new ArrayDeque<>());
+      synchronized(runs)
+      {
+         while(!runs.isEmpty() && now - runs.peekFirst() >= WINDOW_MILLIS)
+         {
+            runs.pollFirst();
+         }
+         if(runs.size() >= perMinute)
+         {
+            return (false);
+         }
+         runs.addLast(now);
+         return (true);
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/DiagnosticsSupport.java

@@ -0,0 +1,222 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.Collection;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Shared pieces of the diagnostic tools: tool names, the current user, the
+ ** per-user rate limit, and recording tool runs and search failures as
+ ** quickSearchDiagnosticEvent rows (the audit trail).
+ *******************************************************************************/
+public final class DiagnosticsSupport
+{
+   private static final QLogger LOG = QLogger.getLogger(DiagnosticsSupport.class);
+
+   public static final String TOOL_PLAYGROUND      = "SEARCH_PLAYGROUND";
+   public static final String TOOL_INSPECT_RECORD  = "INSPECT_RECORD";
+   public static final String TOOL_REINDEX_RECORDS = "REINDEX_RECORDS";
+   public static final String TOOL_VALIDATE_INDEX  = "VALIDATE_INDEX";
+   public static final String TOOL_ANALYZE_TEXT    = "ANALYZE_TEXT";
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private DiagnosticsSupport()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The session's user id reference (else full name), or null.
+    *******************************************************************************/
+   public static String currentUserName()
+   {
+      QSession session = QContext.getQSession();
+      if(session == null || session.getUser() == null)
+      {
+         return (null);
+      }
+      return (session.getUser().getIdReference() != null ? session.getUser().getIdReference() : session.getUser().getFullName());
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime of the Quick Search QBit, or a user-facing error saying the
+    ** tool is unavailable.
+    *******************************************************************************/
+   public static QuickSearchRuntime requireRuntime() throws QUserFacingException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         throw (new QUserFacingException("Quick Search is not configured in this application, so its diagnostic tools are unavailable."));
+      }
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Admit one run of a tool for the current user, or refuse it (recording
+    ** the refusal) when the user ran diagnosticRunsPerMinute tools in the last
+    ** minute.
+    *******************************************************************************/
+   public static void checkRateLimit(QuickSearchRuntime runtime, String tool) throws QUserFacingException
+   {
+      String user = currentUserName();
+      if(runtime.getDiagnosticsRateLimiter().tryAcquire((user == null ? "anonymous" : user) + "|" + tool, runtime.getConfig().getDiagnosticRunsPerMinute()))
+      {
+         return;
+      }
+
+      runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+         .withEventType(QuickSearchDiagnosticEvent.TYPE_RATE_LIMITED)
+         .withSource(tool)
+         .withOutcome(QuickSearchDiagnosticEvent.OUTCOME_REFUSED));
+      throw (new QUserFacingException("Too many Quick Search diagnostic runs in the last minute (limit " + runtime.getConfig().getDiagnosticRunsPerMinute()
+         + " per tool). Wait a minute and try again."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a tool run (audit): who ran which tool on which tables with which
+    ** term, how long it took, and its outcome.
+    *******************************************************************************/
+   public static void recordToolRun(QuickSearchRuntime runtime, String tool, String eventType, Collection<String> tableNames, String term, long startMillis, Throwable error, String detail)
+   {
+      try
+      {
+         runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+            .withEventType(eventType)
+            .withSource(tool)
+            .withOutcome(error == null ? QuickSearchDiagnosticEvent.OUTCOME_OK : QuickSearchDiagnosticEvent.OUTCOME_ERROR)
+            .withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
+            .withSearchTerm(term)
+            .withDurationMillis((int) Math.min(Integer.MAX_VALUE, System.currentTimeMillis() - startMillis))
+            .withErrorClass(error == null ? null : error.getClass().getName())
+            .withErrorMessage(error == null ? null : error.getMessage())
+            .withDetail(detail));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not queue a diagnostic event", e);
+      }
+
+      LOG.info("Quick Search diagnostic tool run", logPair("tool", tool), logPair("user", currentUserName()), logPair("tables", tableNames),
+         logPair("millis", System.currentTimeMillis() - startMillis), logPair("error", error == null ? null : error.getMessage()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a failed search (OpenSearch error or timeout). userTerm is the
+    ** user's term: never stored, and redacted from the message and detail.
+    *******************************************************************************/
+   public static void recordSearchFailure(QuickSearchRuntime runtime, String source, Collection<String> tableNames, String userTerm, long durationMillis, Throwable error, String detail)
+   {
+      if(runtime == null)
+      {
+         return;
+      }
+      try
+      {
+         Throwable cause = rootCause(error);
+         runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
+            .withEventType(QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED)
+            .withSource(source)
+            .withOutcome(QuickSearchDiagnosticEvent.OUTCOME_ERROR)
+            .withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
+            .withDurationMillis((int) Math.min(Integer.MAX_VALUE, durationMillis))
+            .withErrorClass(cause == null ? null : cause.getClass().getName())
+            .withErrorMessage(redactTerm(error == null ? null : error.getMessage(), userTerm))
+            .withDetail(redactTerm(detail, userTerm)));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Could not queue a search failure event", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The text with the user's search term, and each of its words of three or
+    ** more characters, replaced by a marker (case-insensitive): error messages
+    ** (ours and OpenSearch's) can echo the term, which may hold personal data.
+    *******************************************************************************/
+   public static String redactTerm(String text, String userTerm)
+   {
+      if(text == null || userTerm == null || userTerm.isBlank())
+      {
+         return (text);
+      }
+      String redacted = text.replaceAll("(?i)" + java.util.regex.Pattern.quote(userTerm.trim()), "[search term]");
+      for(String word : userTerm.trim().split("\\s+"))
+      {
+         if(word.length() >= 3)
+         {
+            redacted = redacted.replaceAll("(?i)" + java.util.regex.Pattern.quote(word), "[search term]");
+         }
+      }
+      return (redacted);
+   }
+
+
+
+   /*******************************************************************************
+    ** The innermost cause of a throwable (the one that names the real error
+    ** class: timeout, connection refused, OpenSearch error).
+    *******************************************************************************/
+   public static Throwable rootCause(Throwable error)
+   {
+      Throwable cause = error;
+      int       depth = 0;
+      while(cause != null && cause.getCause() != null && cause.getCause() != cause && depth++ < 20)
+      {
+         cause = cause.getCause();
+      }
+      return (cause);
+   }
+
+
+
+   /*******************************************************************************
+    ** A user-facing error for a tool whose OpenSearch call failed: the tool
+    ** reports the failure honestly instead of showing partial data.
+    *******************************************************************************/
+   public static QUserFacingException unavailable(String what, QException e)
+   {
+      return (new QUserFacingException(what + " is unavailable: " + e.getMessage(), e));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/IndexValidator.java

@@ -0,0 +1,782 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.RawResponse;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Validate the Quick Search index against the database and its expected
+ ** shape, and report every finding with a level (OK, INFO, WARN, FAIL):
+ ** - cluster: reachable, cluster and index health, the alias pointing at one
+ **   physical index, leftover physical indexes from interrupted rebuilds;
+ ** - mapping and settings: mapping version, field types and analyzers of the
+ **   fixed fields, the fieldValues dynamic template and every fieldValues.*
+ **   field mapped as text, the two analyzers and the edge n-gram filter;
+ ** - per table: database rows vs indexed documents, a sample of records
+ **   (the lowest and highest primary keys) checked present and current in
+ **   the index, a random sample of documents checked to still have a record,
+ **   the quickSearchIndex row's status and last run, failed events waiting.
+ ** Database reads run as the system user, so counts compare like with like
+ ** (the index holds every row whatever the record locks). The report holds
+ ** counts, ids and field names, never record values. Read-only, except that
+ ** persist() saves the report as a quickSearchValidationRun row.
+ *******************************************************************************/
+public class IndexValidator
+{
+   private static final QLogger LOG = QLogger.getLogger(IndexValidator.class);
+
+   private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
+
+   public static final String SCOPE_CLUSTER = "cluster";
+   public static final String SCOPE_MAPPING = "mapping";
+
+   private static final int MAX_LISTED_IDS = 25;
+
+
+
+   /*******************************************************************************
+    ** One finding.
+    *******************************************************************************/
+   public record Finding(String scope, String check, Level level, String message)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Per-table numbers.
+    *******************************************************************************/
+   public record TableStats(String tableName, Integer databaseCount, Long indexCount, Integer sampled, List<String> missingIds, Map<String, List<String>> staleFields,
+                            Integer documentsSampled, List<String> extraIds)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A validation report.
+    *******************************************************************************/
+   public record Report(String status, Instant startTime, Instant endTime, Integer sampleSize, List<String> tableNames, List<Finding> findings, List<TableStats> tableStats)
+   {
+      /*******************************************************************************
+       ** Count of findings at a level.
+       *******************************************************************************/
+      public int count(Level level)
+      {
+         return ((int) findings.stream().filter(f -> f.level() == level).count());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate the named tables (empty for every indexed table) with the given
+    ** sample size per table.
+    *******************************************************************************/
+   public Report validate(QuickSearchRuntime runtime, List<String> requestedTables, int sampleSize) throws QException
+   {
+      Instant          start     = Instant.now();
+      List<Finding>    findings  = new ArrayList<>();
+      List<TableStats> stats     = new ArrayList<>();
+      List<String>     tables    = resolveTables(runtime, requestedTables, findings);
+      int              boundedN  = Math.max(1, Math.min(sampleSize, QuickSearchQBitConfig.MAX_VALIDATION_SAMPLE_SIZE));
+      QInstance        qInstance = QContext.getQInstance();
+
+      QContext.withTemporaryContext(new CapturedContext(qInstance, new QSystemUserSession()), () ->
+      {
+         QuickSearchDiagnosticsOperations ops;
+         try
+         {
+            ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+         }
+         catch(QException e)
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Client", Level.FAIL, "The OpenSearch client could not be built: " + e.getMessage()));
+            return;
+         }
+
+         if(!checkCluster(runtime, ops, findings))
+         {
+            return;
+         }
+         checkMapping(ops, findings);
+         checkSettings(ops, findings);
+
+         for(String tableName : tables)
+         {
+            stats.add(checkTable(runtime, ops, runtime.getTableConfig(tableName), boundedN, findings));
+         }
+      });
+
+      String status = findings.stream().anyMatch(f -> f.level() == Level.FAIL) ? QuickSearchValidationRun.STATUS_FAIL
+         : findings.stream().anyMatch(f -> f.level() == Level.WARN) ? QuickSearchValidationRun.STATUS_WARN : QuickSearchValidationRun.STATUS_PASS;
+      return (new Report(status, start, Instant.now(), boundedN, tables, findings, stats));
+   }
+
+
+
+   /*******************************************************************************
+    ** The requested indexed tables (unknown ones reported), else every indexed
+    ** table.
+    *******************************************************************************/
+   private static List<String> resolveTables(QuickSearchRuntime runtime, List<String> requested, List<Finding> findings)
+   {
+      List<String> tables = new ArrayList<>();
+      if(CollectionUtils.nullSafeIsEmpty(requested))
+      {
+         runtime.getDiscoveredTables().forEach(t -> tables.add(t.getTableName()));
+         return (tables);
+      }
+      for(String tableName : requested)
+      {
+         if(runtime.getTableConfig(tableName) == null)
+         {
+            findings.add(new Finding(tableName, "Configured", Level.WARN, "Quick Search does not index this table; skipped."));
+         }
+         else if(!tables.contains(tableName))
+         {
+            tables.add(tableName);
+         }
+      }
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health, alias and physical indexes. Returns false when the index
+    ** cannot be reached (the other checks are then skipped).
+    *******************************************************************************/
+   boolean checkCluster(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse health;
+      try
+      {
+         health = ops.clusterHealth();
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "OpenSearch did not answer: " + e.getMessage()));
+         return (false);
+      }
+
+      if(health.status() == 404)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Index", Level.FAIL, "The index alias [" + ops.getIndexName() + "] does not exist. Run the full reindex to create it."));
+         return (false);
+      }
+      if(!health.isOk())
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "Cluster health returned status " + health.status() + ": " + QuickSearchDiagnosticsOperations.errorReason(health.body())));
+         return (false);
+      }
+
+      findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.OK, "OpenSearch answered (" + health.body().path("cluster_name").asText("cluster") + ", "
+         + health.body().path("number_of_nodes").asInt() + " node(s))."));
+      findings.add(healthFinding("Cluster health", health.body().path("status").asText("unknown")));
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = health.body().path("indices").fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index = indexes.next();
+         findings.add(healthFinding("Index health " + index.getKey(), index.getValue().path("status").asText("unknown")));
+      }
+
+      try
+      {
+         Set<String> physical = runtime.getClient().getPhysicalIndexes();
+         if(physical.isEmpty())
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.WARN, "[" + ops.getIndexName() + "] is a concrete index, not an alias (created before 1.0). The next full reindex replaces it with an alias."));
+         }
+         else if(physical.size() > 1)
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "The alias points at " + physical.size() + " physical indexes " + physical + "; searches and counts see documents twice. Run the full reindex."));
+         }
+         else
+         {
+            findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.OK, "The alias points at " + physical.iterator().next() + "."));
+         }
+
+         RawResponse cat = ops.physicalIndexes();
+         if(cat.isOk() && cat.body() != null)
+         {
+            for(JsonNode row : cat.body())
+            {
+               String name = row.path("index").asText();
+               if(!physical.contains(name))
+               {
+                  findings.add(new Finding(SCOPE_CLUSTER, "Leftover index", Level.WARN, "Physical index " + name + " (" + row.path("docs.count").asText("?") + " docs, "
+                     + row.path("store.size").asText("?") + ") is not behind the alias: left by an interrupted rebuild. Delete it to free space."));
+               }
+            }
+         }
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "Could not read the alias: " + e.getMessage()));
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static Finding healthFinding(String check, String status)
+   {
+      return (switch(status)
+      {
+         case "green" -> new Finding(SCOPE_CLUSTER, check, Level.OK, "green.");
+         case "yellow" -> new Finding(SCOPE_CLUSTER, check, Level.WARN, "yellow: replica shards are unassigned (expected on a single-node cluster when the index asks for replicas).");
+         default -> new Finding(SCOPE_CLUSTER, check, Level.FAIL, status + ": primary shards are unassigned; searches and writes may fail.");
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** The live mapping against the mapping this version creates.
+    *******************************************************************************/
+   void checkMapping(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse response;
+      try
+      {
+         response = ops.mapping();
+         QuickSearchDiagnosticsOperations.requireOk(response, "Mapping read failed");
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_MAPPING, "Mapping", Level.FAIL, e.getMessage()));
+         return;
+      }
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index    = indexes.next();
+         JsonNode                    mappings = index.getValue().path("mappings");
+         String                      prefix   = index.getKey() + ": ";
+
+         int version = mappings.path("_meta").path(QuickSearchOpenSearchClient.META_MAPPING_VERSION).asInt(1);
+         findings.add(version >= QuickSearchOpenSearchClient.MAPPING_VERSION
+            ? new Finding(SCOPE_MAPPING, "Mapping version", Level.OK, prefix + "version " + version + ".")
+            : new Finding(SCOPE_MAPPING, "Mapping version", Level.FAIL, prefix + "version " + version + ", this release writes " + QuickSearchOpenSearchClient.MAPPING_VERSION + ". Run the full reindex."));
+
+         JsonNode properties = mappings.path("properties");
+         List<String> problems = new ArrayList<>();
+         expectType(properties, "sourceTable", "keyword", problems);
+         expectType(properties, "recordId", "keyword", problems);
+         expectType(properties, "indexedAt", "date", problems);
+         expectText(properties.path("searchableText"), "searchableText", problems);
+         expectText(properties.path("recordLabel"), "recordLabel", problems);
+
+         boolean template = false;
+         for(JsonNode entry : mappings.path("dynamic_templates"))
+         {
+            template |= entry.has("fieldValuesAsText");
+         }
+         if(!template)
+         {
+            problems.add("the fieldValuesAsText dynamic template is missing, so new fields may be mapped as numbers or dates");
+         }
+         if(mappings.path("date_detection").asBoolean(true) || mappings.path("numeric_detection").asBoolean(false))
+         {
+            problems.add("date or numeric detection is on");
+         }
+
+         Iterator<Map.Entry<String, JsonNode>> fields = properties.path("fieldValues").path("properties").fields();
+         int                                   count  = 0;
+         while(fields.hasNext())
+         {
+            Map.Entry<String, JsonNode> field = fields.next();
+            expectText(field.getValue(), "fieldValues." + field.getKey(), problems);
+            count++;
+         }
+
+         findings.add(problems.isEmpty()
+            ? new Finding(SCOPE_MAPPING, "Field mappings", Level.OK, prefix + "fixed fields and " + count + " fieldValues field(s) mapped as expected.")
+            : new Finding(SCOPE_MAPPING, "Field mappings", Level.FAIL, prefix + String.join("; ", problems) + ". Run the full reindex to rebuild the mapping."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void expectType(JsonNode properties, String field, String type, List<String> problems)
+   {
+      String actual = properties.path(field).path("type").asText(null);
+      if(!type.equals(actual))
+      {
+         problems.add(field + " is " + (actual == null ? "not mapped" : "mapped as " + actual) + ", expected " + type);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void expectText(JsonNode mapping, String field, List<String> problems)
+   {
+      String type     = mapping.path("type").asText(null);
+      String analyzer = mapping.path("analyzer").asText(null);
+      String search   = mapping.path("search_analyzer").asText(null);
+      if(!"text".equals(type))
+      {
+         problems.add(field + " is " + (type == null ? "not mapped" : "mapped as " + type) + ", expected text");
+      }
+      else if(!QuickSearchOpenSearchClient.INDEX_ANALYZER.equals(analyzer) || !QuickSearchOpenSearchClient.SEARCH_ANALYZER.equals(search))
+      {
+         problems.add(field + " uses analyzer " + analyzer + " / search analyzer " + search + ", expected " + QuickSearchOpenSearchClient.INDEX_ANALYZER + " / " + QuickSearchOpenSearchClient.SEARCH_ANALYZER);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The live analysis settings against the ones this version creates.
+    *******************************************************************************/
+   void checkSettings(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
+   {
+      RawResponse response;
+      try
+      {
+         response = ops.settings();
+         QuickSearchDiagnosticsOperations.requireOk(response, "Settings read failed");
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, e.getMessage()));
+         return;
+      }
+
+      Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
+      while(indexes.hasNext())
+      {
+         Map.Entry<String, JsonNode> index    = indexes.next();
+         JsonNode                    settings = index.getValue().path("settings").path("index");
+         JsonNode                    analysis = settings.path("analysis");
+         List<String>                problems = new ArrayList<>();
+
+         JsonNode indexAnalyzer  = analysis.path("analyzer").path(QuickSearchOpenSearchClient.INDEX_ANALYZER);
+         JsonNode searchAnalyzer = analysis.path("analyzer").path(QuickSearchOpenSearchClient.SEARCH_ANALYZER);
+         JsonNode ngram          = analysis.path("filter").path("edge_ngram_filter");
+
+         if(!"standard".equals(indexAnalyzer.path("tokenizer").asText()) || !List.of("lowercase", "edge_ngram_filter").equals(textList(indexAnalyzer.path("filter"))))
+         {
+            problems.add(QuickSearchOpenSearchClient.INDEX_ANALYZER + " is " + (indexAnalyzer.isMissingNode() ? "missing" : indexAnalyzer.toString()));
+         }
+         if(!"standard".equals(searchAnalyzer.path("tokenizer").asText()) || !List.of("lowercase").equals(textList(searchAnalyzer.path("filter"))))
+         {
+            problems.add(QuickSearchOpenSearchClient.SEARCH_ANALYZER + " is " + (searchAnalyzer.isMissingNode() ? "missing" : searchAnalyzer.toString()));
+         }
+         if(!"edge_ngram".equals(ngram.path("type").asText()) || ngram.path("min_gram").asInt() != 2 || ngram.path("max_gram").asInt() != 20)
+         {
+            problems.add("edge_ngram_filter is " + (ngram.isMissingNode() ? "missing" : ngram.toString()));
+         }
+
+         findings.add(problems.isEmpty()
+            ? new Finding(SCOPE_MAPPING, "Analysis settings", Level.OK, index.getKey() + ": analyzers and edge n-grams (2 to 20) as expected; "
+            + settings.path("number_of_shards").asText("?") + " shard(s), " + settings.path("number_of_replicas").asText("?") + " replica(s).")
+            : new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, index.getKey() + ": " + String.join("; ", problems) + ". Run the full reindex."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static List<String> textList(JsonNode array)
+   {
+      List<String> values = new ArrayList<>();
+      array.forEach(v -> values.add(v.asText()));
+      return (values);
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts, sampled freshness and orphans of one table, plus its index row
+    ** and failed events.
+    *******************************************************************************/
+   TableStats checkTable(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, int sampleSize, List<Finding> findings)
+   {
+      String tableName = tableConfig.getTableName();
+      Integer databaseCount = null;
+      Long    indexCount    = null;
+      try
+      {
+         databaseCount = DiagnosticRecordReads.count(tableName);
+         indexCount    = runtime.getClient().countDocumentsForTable(tableName);
+         long difference = indexCount - databaseCount;
+         long tolerance  = Math.max(5, databaseCount / 100);
+         Level level = difference == 0 ? Level.OK : (Math.abs(difference) <= tolerance ? Level.WARN : Level.FAIL);
+         findings.add(new Finding(tableName, "Counts", level, "database " + databaseCount + ", index " + indexCount
+            + (difference == 0 ? "." : " (" + (difference > 0 ? difference + " more in the index" : (-difference) + " missing from the index") + "). Reconcile fixes counts.")));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Counts", Level.FAIL, e.getMessage()));
+      }
+
+      List<String>              missing     = new ArrayList<>();
+      Map<String, List<String>> staleFields = new LinkedHashMap<>();
+      int                       sampled     = 0;
+      try
+      {
+         Map<String, QRecord> sample = new LinkedHashMap<>();
+         for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, true, (sampleSize + 1) / 2))
+         {
+            sample.put(record.getValueString(tableConfig.getPrimaryKeyField()), record);
+         }
+         for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, false, sampleSize / 2))
+         {
+            sample.putIfAbsent(record.getValueString(tableConfig.getPrimaryKeyField()), record);
+         }
+         sampled = sample.size();
+
+         Map<String, JsonNode> indexed = ops.getDocuments(tableName, sample.keySet());
+         for(Map.Entry<String, QRecord> entry : sample.entrySet())
+         {
+            JsonNode source = indexed.get(entry.getKey());
+            if(source == null)
+            {
+               missing.add(entry.getKey());
+               continue;
+            }
+            List<String> differing = differingFields(tableConfig, IndexingUtils.buildDocument(entry.getValue(), tableConfig), source);
+            if(!differing.isEmpty())
+            {
+               staleFields.put(entry.getKey(), differing);
+            }
+         }
+
+         findings.add(missing.isEmpty() && staleFields.isEmpty()
+            ? new Finding(tableName, "Sampled records", Level.OK, sampled + " record(s) checked: all indexed and current.")
+            : new Finding(tableName, "Sampled records", Level.FAIL, sampled + " record(s) checked: " + missing.size() + " missing" + listIds(missing) + ", "
+            + staleFields.size() + " stale" + listIds(new ArrayList<>(staleFields.keySet())) + ". Reindex those records, or run reconcile for the table."));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Sampled records", Level.FAIL, e.getMessage()));
+      }
+
+      List<String> extra            = new ArrayList<>();
+      int          documentsSampled = 0;
+      try
+      {
+         List<String> documentIds = ops.sampleRecordIds(tableName, sampleSize);
+         documentsSampled = documentIds.size();
+         Set<String> existing = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, documentIds));
+         for(String recordId : documentIds)
+         {
+            if(!existing.contains(recordId))
+            {
+               extra.add(recordId);
+            }
+         }
+         findings.add(extra.isEmpty()
+            ? new Finding(tableName, "Sampled documents", Level.OK, documentsSampled + " random document(s) checked: every one has its record.")
+            : new Finding(tableName, "Sampled documents", Level.FAIL, documentsSampled + " random document(s) checked: " + extra.size() + " have no record (orphans)"
+            + listIds(extra) + ". Reconcile removes them."));
+      }
+      catch(QException e)
+      {
+         findings.add(new Finding(tableName, "Sampled documents", Level.FAIL, e.getMessage()));
+      }
+
+      checkIndexRow(runtime, tableName, findings);
+      checkFailedEvents(runtime, tableName, findings);
+
+      return (new TableStats(tableName, databaseCount, indexCount, sampled, missing, staleFields, documentsSampled, extra));
+   }
+
+
+
+   /*******************************************************************************
+    ** Names of the values that differ between an expected document and an
+    ** indexed source (recordLabel, searchableText, fieldValues.x).
+    *******************************************************************************/
+   static List<String> differingFields(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, JsonNode source)
+   {
+      return (RecordInspector.diff(tableConfig, expected, new QuickSearchDiagnosticsOperations.DocumentView(true, null, null, source)).stream()
+         .filter(d -> !d.isSame())
+         .map(RecordInspector.FieldDiff::name)
+         .toList());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void checkIndexRow(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
+   {
+      try
+      {
+         String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+         if(QContext.getQInstance().getTable(indexTable) == null)
+         {
+            return;
+         }
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(indexTable);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "No Quick Search Index row yet: no basepull, reconcile or reindex has run for this table."));
+            return;
+         }
+
+         QRecord row     = rows.get(0);
+         String  status  = row.getValueString("status");
+         String  lastRun = row.getValueString("lastRunStatus");
+         if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.INFO, "Disabled: Quick Search neither indexes nor searches this table, so differences are expected."));
+         }
+         if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "Status NEEDS_REINDEX: the configured fields changed since the last reindex. Rebuild the table."));
+         }
+         else if(AbstractIndexingStep.STATUS_REBUILDING.equals(status))
+         {
+            findings.add(new Finding(tableName, "Index row", Level.WARN, "Status REBUILDING: a full reindex is running. If it was killed, basepull returns the table to ACTIVE once its run records show no sign of life for fullReindexStaleMinutes."));
+         }
+         if(AbstractIndexingStep.RUN_FAILED.equals(lastRun))
+         {
+            findings.add(new Finding(tableName, "Last run", Level.WARN, "The last indexing run failed: " + row.getValueString("lastErrorMessage")));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read the Quick Search Index row during validation", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void checkFailedEvents(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
+   {
+      try
+      {
+         String failedTable = runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
+         if(QContext.getQInstance().getTable(failedTable) == null)
+         {
+            return;
+         }
+         CountInput countInput = new CountInput();
+         countInput.setTableName(failedTable);
+         countInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+            new QFilterCriteria("status", QCriteriaOperator.IN, List.of(QuickSearchFailedEvent.STATUS_PENDING, QuickSearchFailedEvent.STATUS_EXHAUSTED))));
+         Integer count = new CountAction().execute(countInput).getCount();
+         if(count != null && count > 0)
+         {
+            findings.add(new Finding(tableName, "Failed events", Level.WARN, count + " real-time index event(s) failed and wait for replay (or are exhausted). See Quick Search Failed Event."));
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not count failed events during validation", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Save a report as a quickSearchValidationRun row (as the system user);
+    ** returns its id, or null when the table is not configured or the insert
+    ** fails (logged).
+    *******************************************************************************/
+   public Integer persist(QuickSearchRuntime runtime, Report report, String triggerType, String userName)
+   {
+      String tableName = runtime.getConfig().applyPrefix(QuickSearchValidationRun.TABLE_NAME);
+      if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
+      {
+         return (null);
+      }
+
+      QuickSearchValidationRun run = new QuickSearchValidationRun()
+         .withStatus(report.status())
+         .withTriggerType(triggerType)
+         .withUserName(userName)
+         .withTableNames(DiagnosticEventRecorder.truncate(String.join(",", report.tableNames()), 1000))
+         .withSampleSize(report.sampleSize())
+         .withTablesChecked(report.tableStats().size())
+         .withFailCount(report.count(Level.FAIL))
+         .withWarnCount(report.count(Level.WARN))
+         .withSummary(DiagnosticEventRecorder.truncate(summary(report), 250))
+         .withReportHtml(render(report))
+         .withReportJson(toJson(report))
+         .withStartTime(report.startTime())
+         .withEndTime(report.endTime())
+         .withCreateDate(Instant.now());
+
+      Integer[] id = new Integer[1];
+      try
+      {
+         QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
+         {
+            InsertOutput output = new InsertAction().execute(new InsertInput(tableName).withRecordEntity(run));
+            id[0] = output.getRecords().get(0).getValueInteger("id");
+         });
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not save the Quick Search validation report", e);
+      }
+      return (id[0]);
+   }
+
+
+
+   /*******************************************************************************
+    ** One line: status and counts.
+    *******************************************************************************/
+   public static String summary(Report report)
+   {
+      return (report.status() + ": " + report.tableStats().size() + " table(s), " + report.count(Level.FAIL) + " failed check(s), " + report.count(Level.WARN) + " warning(s).");
+   }
+
+
+
+   /*******************************************************************************
+    ** The report as JSON.
+    *******************************************************************************/
+   public static String toJson(Report report)
+   {
+      try
+      {
+         return (MAPPER.writeValueAsString(report));
+      }
+      catch(Exception e)
+      {
+         return ("{\"error\":\"report could not be serialized\"}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The report as HTML: summary, then the findings grouped by scope.
+    *******************************************************************************/
+   public static String render(Report report)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      Level overall = switch(report.status())
+      {
+         case QuickSearchValidationRun.STATUS_PASS -> Level.OK;
+         case QuickSearchValidationRun.STATUS_WARN -> Level.WARN;
+         default -> Level.FAIL;
+      };
+      html.heading("Index validation");
+      html.status(overall, summary(report));
+      html.paragraph("Ran " + report.startTime() + " to " + report.endTime() + ", " + report.sampleSize() + " sampled record(s) and document(s) per table. Database reads ran as the system user.");
+
+      Map<String, List<DiagnosticsHtml.Check>> byScope = new LinkedHashMap<>();
+      for(Finding finding : report.findings())
+      {
+         byScope.computeIfAbsent(finding.scope(), k -> new ArrayList<>()).add(new DiagnosticsHtml.Check(finding.check(), finding.level(), finding.message()));
+      }
+      for(Map.Entry<String, List<DiagnosticsHtml.Check>> scope : byScope.entrySet())
+      {
+         String heading = switch(scope.getKey())
+         {
+            case SCOPE_CLUSTER -> "Cluster and index";
+            case SCOPE_MAPPING -> "Mapping and analyzers";
+            default -> "Table " + scope.getKey();
+         };
+         html.subheading(heading);
+         html.checks(scope.getValue());
+      }
+
+      for(TableStats stats : report.tableStats())
+      {
+         if(!stats.staleFields().isEmpty())
+         {
+            List<String> items = new ArrayList<>();
+            stats.staleFields().entrySet().stream().limit(MAX_LISTED_IDS).forEach(e ->
+               items.add("#" + DiagnosticsHtml.escape(e.getKey()) + ": " + DiagnosticsHtml.escape(String.join(", ", e.getValue()))));
+            html.subheading("Stale fields in " + stats.tableName());
+            html.rawList(items);
+         }
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String listIds(List<String> ids)
+   {
+      if(ids.isEmpty())
+      {
+         return ("");
+      }
+      return (" (" + String.join(", ", ids.subList(0, Math.min(MAX_LISTED_IDS, ids.size()))) + (ids.size() > MAX_LISTED_IDS ? ", ..." : "") + ")");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/QuickSearchDeveloperPanelProvider.java

@@ -0,0 +1,535 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanel;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelAction;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelBlock;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelContext;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelKeyValue;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelLink;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelProviderInterface;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelTone;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The "Quick Search" panel of a record's developer view (QQQ's
+ ** DeveloperPanelProviderInterface, /app/{table}/{id}/dev).
+ **
+ ** Shown for tables this QBit indexes, to users who may read the Quick Search
+ ** Index table. It shows the table's index row (enabled, status, counts, last
+ ** basepull and full reindex, searchable fields), the latest index run, the
+ ** record's indexed document compared with the database (Inspect Search
+ ** Document's comparison; needs the debug permission), the record's failed
+ ** index events, and the Inspect / Reindex record actions and admin links the
+ ** user may use. Every read checks the user's permissions; an unreachable
+ ** OpenSearch shows a warning instead of failing the panel.
+ *******************************************************************************/
+public class QuickSearchDeveloperPanelProvider implements DeveloperPanelProviderInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchDeveloperPanelProvider.class);
+
+   public static final String ID    = "quickSearch";
+   public static final String TITLE = "Quick Search";
+
+   public static final String STATUS_INDEXED     = "Indexed";
+   public static final String STATUS_STALE       = "Stale";
+   public static final String STATUS_NOT_INDEXED = "Not indexed";
+   public static final String STATUS_DISABLED    = "Disabled";
+   public static final String STATUS_ENABLED     = "Enabled";
+   public static final String STATUS_UNAVAILABLE = "OpenSearch unavailable";
+
+   static final int MAX_FAILED_EVENTS = 20;
+   static final int MAX_VALUE_LENGTH  = 300;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getId()
+   {
+      return (ID);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getTitle()
+   {
+      return (TITLE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Integer getOrder()
+   {
+      return (400);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getIcon()
+   {
+      return ("search");
+   }
+
+
+
+   /*******************************************************************************
+    ** Available for a table this QBit indexes, when the user may read the Quick
+    ** Search Index table (the panel shows admin-level indexing state).
+    *******************************************************************************/
+   @Override
+   public boolean isAvailable(DeveloperPanelContext context)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || context == null || runtime.getTableConfig(context.getTableName()) == null)
+      {
+         return (false);
+      }
+      return (mayRead(runtime.getConfig().getQuickSearchIndexTableName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the panel.
+    *******************************************************************************/
+   @Override
+   public DeveloperPanel build(DeveloperPanelContext context) throws QException
+   {
+      QuickSearchRuntime          runtime     = QuickSearchRuntime.getOrThrow();
+      QuickSearchQBitConfig       config      = runtime.getConfig();
+      String                      tableName   = context.getTableName();
+      QuickSearchableTableConfig  tableConfig = runtime.getTableConfig(tableName);
+      Serializable                primaryKey  = context.getPrimaryKey();
+      String                      recordId    = primaryKey == null ? null : String.valueOf(primaryKey);
+      String                      indexTable  = config.getQuickSearchIndexTableName();
+
+      DeveloperPanel panel = new DeveloperPanel()
+         .withDescription("How Quick Search (OpenSearch) indexes this table, and this record's search document.");
+
+      //////////////////////////////////////////////////////////////
+      // the table's index row (readable: checked by isAvailable) //
+      //////////////////////////////////////////////////////////////
+      QRecord indexRow = null;
+      if(mayRead(indexTable))
+      {
+         indexRow = queryFirst(indexTable, new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+      }
+      boolean enabled = indexRow == null || !Boolean.FALSE.equals(indexRow.getValueBoolean("enabled"));
+      panel.withBlock(indexBlock(config, tableConfig, indexTable, indexRow, enabled));
+
+      //////////////////////
+      // latest index run //
+      //////////////////////
+      String runTable = config.getQuickSearchIndexRunTableName();
+      if(indexRow != null && mayRead(runTable))
+      {
+         QRecord run = queryFirst(runTable, new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexRow.getValue("id")))
+            .withOrderBy(new QFilterOrderBy("id", false)));
+         panel.withBlock(run == null
+            ? DeveloperPanelBlock.text("No index runs are recorded for this table yet.", DeveloperPanelTone.INFO).withLabel("Latest index run")
+            : runBlock(runTable, run));
+      }
+
+      /////////////////////////////////////////
+      // this record's document vs. database //
+      /////////////////////////////////////////
+      String verdict = null;
+      boolean openSearchFailed = false;
+      if(recordId != null)
+      {
+         if(!Boolean.TRUE.equals(config.getEnableDiagnosticTools()))
+         {
+            panel.withBlock(DeveloperPanelBlock.text("Quick Search diagnostic tools are disabled (enableDiagnosticTools), so this record's search document is not compared.", DeveloperPanelTone.INFO)
+               .withLabel("Search document"));
+         }
+         else if(!mayInspect(config, tableName))
+         {
+            panel.withBlock(DeveloperPanelBlock.text("You do not have the Quick Search debug permission, so this record's search document is not shown.", DeveloperPanelTone.INFO)
+               .withLabel("Search document"));
+         }
+         else
+         {
+            try
+            {
+               RecordInspector.Inspection inspection = inspect(runtime, tableName, recordId, new QueryInput(tableName));
+               verdict = inspection.verdict();
+               addInspectionBlocks(panel, inspection);
+            }
+            catch(Exception e)
+            {
+               openSearchFailed = true;
+               LOG.info("Quick Search developer panel could not read the search document", logPair("tableName", tableName), logPair("recordId", recordId), logPair("error", e.getMessage()));
+               panel.withBlock(DeveloperPanelBlock.text("OpenSearch could not be reached, so this record's search document cannot be shown: "
+                  + DiagnosticsSupport.rootCause(e).getMessage(), DeveloperPanelTone.WARNING).withLabel("Search document"));
+            }
+         }
+      }
+
+      /////////////////////////////////
+      // failed index events, if any //
+      /////////////////////////////////
+      String failedTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
+      if(recordId != null && mayRead(failedTable))
+      {
+         DeveloperPanelBlock failedBlock = failedEventsBlock(failedTable, tableName, recordId);
+         if(failedBlock != null)
+         {
+            panel.withBlock(failedBlock);
+         }
+      }
+
+      //////////////////////////////
+      // actions the user may run //
+      //////////////////////////////
+      if(recordId != null)
+      {
+         DeveloperPanelBlock actions = DeveloperPanelBlock.actions("Actions");
+         addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName), tableName, "Inspect search document",
+            "Compare the indexed document with the database, and see why the record does or does not match a term.");
+         addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config, tableName), tableName, "Reindex this record",
+            "Rebuild this record's search document from the database now.");
+         if(!actions.getActions().isEmpty())
+         {
+            panel.withBlock(actions);
+         }
+      }
+
+      ///////////
+      // links //
+      ///////////
+      DeveloperPanelBlock links   = DeveloperPanelBlock.links("Links");
+      String              appName = config.applyPrefix(QuickSearchAdminAppMetaDataProducer.APP_NAME);
+      if(QContext.getQInstance().getApp(appName) != null && PermissionsHelper.hasAppPermission(new QueryInput(indexTable), appName))
+      {
+         links.withLink(new DeveloperPanelLink().withLabel("Quick Search Admin").withHref("/app/" + appName).withDescription("Index status, runs, failed events and diagnostic tools."));
+      }
+      if(indexRow != null)
+      {
+         links.withLink(new DeveloperPanelLink().withLabel("Quick Search Index row for " + tableName).withHref(recordHref(indexTable, indexRow.getValue("id"))));
+      }
+      if(!links.getLinks().isEmpty())
+      {
+         panel.withBlock(links);
+      }
+
+      setStatus(panel, enabled, verdict, openSearchFailed);
+      return (panel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect one record (overridable for tests).
+    *******************************************************************************/
+   protected RecordInspector.Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, AbstractActionInput permissionInput) throws QException
+   {
+      return (new RecordInspector().inspect(runtime, tableName, recordId, null, permissionInput));
+   }
+
+
+
+   /*******************************************************************************
+    ** The table's index row as key/values.
+    *******************************************************************************/
+   private DeveloperPanelBlock indexBlock(QuickSearchQBitConfig config, QuickSearchableTableConfig tableConfig, String indexTable, QRecord indexRow, boolean enabled)
+   {
+      DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Index");
+      if(indexRow == null)
+      {
+         block.withItem("Index row", "none yet (the first indexing run creates it)");
+      }
+      else
+      {
+         block.withItem(new DeveloperPanelKeyValue().withLabel("Index row").withValue("#" + indexRow.getValueString("id")).withHref(recordHref(indexTable, indexRow.getValue("id"))));
+      }
+      block.withItem("Enabled", enabled ? "Yes" : "No");
+      block.withItem("Status", indexRow == null ? null : indexRow.getValueString("status"));
+      block.withItem("Indexed records", indexRow == null ? null : indexRow.getValueString("recordCount"));
+      block.withItem("Last basepull", indexRow == null ? null : indexRow.getValueString("lastBasepullTime"));
+      block.withItem("Last full reindex", indexRow == null ? null : indexRow.getValueString("lastFullReindexTime"));
+      block.withItem("Searchable fields", String.join(", ", CollectionUtils.nonNullList(tableConfig.getSearchableFields())));
+      block.withItem(new DeveloperPanelKeyValue().withLabel("OpenSearch index (alias)").withValue(config.getOpensearchIndexName()).withMonospace(true));
+      block.withItem("Real-time indexing", Boolean.TRUE.equals(config.getEnableRealTimeIndexing()) ? "On" : "Off");
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** The latest index run as key/values.
+    *******************************************************************************/
+   private DeveloperPanelBlock runBlock(String runTable, QRecord run)
+   {
+      DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Latest index run")
+         .withItem(new DeveloperPanelKeyValue().withLabel("Run").withValue("#" + run.getValueString("id")).withHref(recordHref(runTable, run.getValue("id"))))
+         .withItem("Type", run.getValueString("runType"))
+         .withItem("Status", run.getValueString("status"))
+         .withItem("Started", run.getValueString("startTime"))
+         .withItem("Ended", run.getValueString("endTime"))
+         .withItem("Processed", run.getValueString("recordsProcessed"))
+         .withItem("Indexed", run.getValueString("recordsIndexed"))
+         .withItem("Errors", run.getValueString("errorCount"));
+      if(run.getValueString("errorMessage") != null)
+      {
+         block.withItem("Error", truncate(run.getValueString("errorMessage")));
+      }
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** The inspection: verdict text, document facts, field comparison table.
+    *******************************************************************************/
+   private void addInspectionBlocks(DeveloperPanel panel, RecordInspector.Inspection inspection)
+   {
+      if(inspection.error() != null)
+      {
+         panel.withBlock(DeveloperPanelBlock.text(inspection.error(), RecordInspector.VERDICT_NOT_INDEXED.equals(inspection.verdict()) ? DeveloperPanelTone.INFO : DeveloperPanelTone.WARNING)
+            .withLabel("Search document"));
+         return;
+      }
+
+      switch(inspection.verdict())
+      {
+         case RecordInspector.VERDICT_CURRENT -> panel.withBlock(DeveloperPanelBlock.text("Indexed and current: the search document matches the database.", DeveloperPanelTone.SUCCESS)
+            .withLabel("Search document"));
+         case RecordInspector.VERDICT_STALE -> panel.withBlock(DeveloperPanelBlock.text("Indexed but stale: some values differ from the database. Reindex this record to refresh it.", DeveloperPanelTone.WARNING)
+            .withLabel("Search document"));
+         default -> panel.withBlock(DeveloperPanelBlock.text("Not in the index: the global search cannot find this record. Reindex this record to add it (or wait for the next basepull or reconcile).",
+            DeveloperPanelTone.ERROR).withLabel("Search document"));
+      }
+
+      DocumentView indexed = inspection.indexed();
+      panel.withBlock(DeveloperPanelBlock.keyValues("Document")
+         .withItem(new DeveloperPanelKeyValue().withLabel("Document id").withValue(OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId())).withMonospace(true))
+         .withItem(new DeveloperPanelKeyValue().withLabel("Physical index").withValue(indexed == null ? null : indexed.physicalIndex()).withMonospace(true))
+         .withItem("Indexed version", indexed == null || indexed.version() == null ? null : String.valueOf(indexed.version()))
+         .withItem("Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)" : String.valueOf(inspection.expected().getVersion()))
+         .withItem("Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null)));
+
+      DeveloperPanelBlock table = DeveloperPanelBlock.table("Database vs index", List.of("Field", "Database", "Index", "Match"));
+      for(RecordInspector.FieldDiff diff : CollectionUtils.nonNullList(inspection.diffs()))
+      {
+         table.withRow(row(diff.name(), truncate(diff.databaseValue()), truncate(diff.indexValue()), diff.isSame() ? "yes" : "NO"));
+      }
+      panel.withBlock(table);
+   }
+
+
+
+   /*******************************************************************************
+    ** The record's failed index events, or null when there are none.
+    *******************************************************************************/
+   private DeveloperPanelBlock failedEventsBlock(String failedTable, String tableName, String recordId) throws QException
+   {
+      QueryInput queryInput = new QueryInput(failedTable);
+      queryInput.setFilter(new QQueryFilter(
+         new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+         new QFilterCriteria("recordId", QCriteriaOperator.EQUALS, recordId))
+         .withOrderBy(new QFilterOrderBy("id", false))
+         .withLimit(MAX_FAILED_EVENTS));
+      List<QRecord> events = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+      if(events.isEmpty())
+      {
+         return (null);
+      }
+
+      DeveloperPanelBlock block = DeveloperPanelBlock.table("Failed index events" + (events.size() >= MAX_FAILED_EVENTS ? " (latest " + MAX_FAILED_EVENTS + ")" : ""),
+         List.of("Id", "Action", "Status", "Attempts", "Created", "Error"));
+      for(QRecord event : events)
+      {
+         block.withRow(row(event.getValueString("id"), event.getValueString("action"), event.getValueString("status"), event.getValueString("attempts"),
+            event.getValueString("createDate"), truncate(event.getValueString("errorMessage"))));
+      }
+      return (block);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add a record action when the process exists and the user may run it.
+    *******************************************************************************/
+   private void addActionIfPermitted(DeveloperPanelBlock actions, String processName, String tableName, String label, String description)
+   {
+      if(QContext.getQInstance().getProcess(processName) != null && PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName))
+      {
+         actions.withAction(new DeveloperPanelAction().withLabel(label).withProcessName(processName).withTableName(tableName).withDescription(description));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the user may inspect records: the Inspect Search Document
+    ** process's permission (debugPermissionRules), evaluated on the
+    ** registered process, or on an unregistered copy when the record actions
+    ** are turned off.
+    *******************************************************************************/
+   private boolean mayInspect(QuickSearchQBitConfig config, String tableName)
+   {
+      String processName = QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName);
+      if(QContext.getQInstance().getProcess(processName) != null)
+      {
+         return (PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName));
+      }
+      return (PermissionsHelper.getPermissionCheckResult(new QueryInput(tableName), QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(config, tableName))
+         == PermissionCheckResult.ALLOW);
+   }
+
+
+
+   /*******************************************************************************
+    ** Status label and tone from the enabled flag and the inspection verdict.
+    *******************************************************************************/
+   private void setStatus(DeveloperPanel panel, boolean enabled, String verdict, boolean openSearchFailed)
+   {
+      if(!enabled)
+      {
+         panel.withStatusLabel(STATUS_DISABLED).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(openSearchFailed)
+      {
+         panel.withStatusLabel(STATUS_UNAVAILABLE).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(RecordInspector.VERDICT_CURRENT.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_INDEXED).withStatusTone(DeveloperPanelTone.SUCCESS);
+      }
+      else if(RecordInspector.VERDICT_STALE.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_STALE).withStatusTone(DeveloperPanelTone.WARNING);
+      }
+      else if(RecordInspector.VERDICT_MISSING.equals(verdict))
+      {
+         panel.withStatusLabel(STATUS_NOT_INDEXED).withStatusTone(DeveloperPanelTone.ERROR);
+      }
+      else
+      {
+         panel.withStatusLabel(STATUS_ENABLED).withStatusTone(DeveloperPanelTone.INFO);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the table exists and the user may read it.
+    *******************************************************************************/
+   private static boolean mayRead(String tableName)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      return (qInstance != null && qInstance.getTable(tableName) != null && PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
+   }
+
+
+
+   /*******************************************************************************
+    ** First row of a query (as the user), or null.
+    *******************************************************************************/
+   private static QRecord queryFirst(String tableName, QQueryFilter filter) throws QException
+   {
+      QueryInput queryInput = new QueryInput(tableName);
+      queryInput.setFilter(filter.withLimit(1));
+      List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeHasContents(records) ? records.get(0) : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** App-relative link to a record.
+    *******************************************************************************/
+   private static String recordHref(String tableName, Serializable id)
+   {
+      return ("/app/" + tableName + "/" + id);
+   }
+
+
+
+   /*******************************************************************************
+    ** A table row (null cells as empty strings).
+    *******************************************************************************/
+   private static List<String> row(String... cells)
+   {
+      List<String> row = new ArrayList<>();
+      for(String cell : cells)
+      {
+         row.add(cell == null ? "" : cell);
+      }
+      return (row);
+   }
+
+
+
+   /*******************************************************************************
+    ** Long values cut for display.
+    *******************************************************************************/
+   static String truncate(String value)
+   {
+      if(value == null || value.length() <= MAX_VALUE_LENGTH)
+      {
+         return (value);
+      }
+      return (value.substring(0, MAX_VALUE_LENGTH) + "... (" + value.length() + " characters)");
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/QuickSearchDiagnosticsWidgetRenderer.java

@@ -0,0 +1,217 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+
+
+/*******************************************************************************
+ ** The Search Diagnostics widget: the record-search provider's state on the
+ ** server that renders it, the most recent search failures (from every
+ ** server, via Quick Search Diagnostic Event), the latest validation run,
+ ** and where the tools are. Every part says plainly when it has nothing to
+ ** show or cannot be read.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchDiagnosticsWidgetRenderer.class);
+
+   public static final int RECENT_FAILURES = 10;
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null)
+      {
+         return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", "<p>Quick Search is not configured in this application.</p>")));
+      }
+      return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", renderHtml(runtime))));
+   }
+
+
+
+   /*******************************************************************************
+    ** The widget's HTML.
+    *******************************************************************************/
+   static String renderHtml(QuickSearchRuntime runtime)
+   {
+      QuickSearchQBitConfig config = runtime.getConfig();
+      DiagnosticsHtml       html   = new DiagnosticsHtml().open();
+
+      SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(QContext.getQInstance(), runtime);
+      html.subheading("Global search on this server");
+      if(!Boolean.TRUE.equals(state.quickSearchProvider()) || !Boolean.TRUE.equals(state.serveCoreRecordSearch()))
+      {
+         html.status(Level.INFO, "Quick Search does not serve the global search here (" + (state.registeredProvider() == null ? "no provider registered" : "provider " + state.registeredProvider())
+            + "); core searches every table itself.");
+      }
+      else if(state.backoffRemainingMillis() > 0)
+      {
+         html.status(Level.FAIL, "Backing off for " + (state.backoffRemainingMillis() / 1000) + " s after an OpenSearch failure: the global search uses core's database search meanwhile. Last error: "
+            + state.lastFailureMessage());
+      }
+      else
+      {
+         html.status(Level.OK, "Served by OpenSearch through Quick Search (timeout " + state.timeoutMillis() + " ms).");
+      }
+      html.facts(List.of(
+         new String[] { "Failed OpenSearch searches since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) },
+         new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) },
+         new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() }));
+
+      renderRecentFailures(html, config);
+      renderLastValidation(html, config);
+
+      html.subheading("Tools");
+      html.paragraph("On Quick Search Index (Actions): Search Playground, Validate Search Index, Test Search Analyzer. "
+         + "On any indexed record and on Quick Search Failed Event rows: Inspect Search Document and Reindex in Search. "
+         + "Every run is recorded in Quick Search Diagnostic Event.");
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderRecentFailures(DiagnosticsHtml html, QuickSearchQBitConfig config)
+   {
+      String tableName = config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
+      html.subheading("Recent search failures (all servers)");
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         html.paragraph("Not recorded: the diagnostic event table is not configured.");
+         return;
+      }
+
+      try
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("eventType", QCriteriaOperator.EQUALS, QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED))
+            .withOrderBy(new QFilterOrderBy("id", false))
+            .withLimit(RECENT_FAILURES));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            html.status(Level.OK, "None recorded (failures are kept for " + config.getRunHistoryRetentionDays() + " days).");
+            return;
+         }
+
+         List<String> items = new ArrayList<>();
+         for(QRecord row : rows)
+         {
+            items.add("<a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
+               + DiagnosticsHtml.escape(row.getValueString("createDate")) + "</a> on " + DiagnosticsHtml.escape(row.getValueString("hostName"))
+               + ": " + DiagnosticsHtml.escape(row.getValueString("source")) + ", " + DiagnosticsHtml.escape(row.getValueString("durationMillis")) + " ms, "
+               + DiagnosticsHtml.escape(shortClass(row.getValueString("errorClass"))) + ": " + DiagnosticsHtml.escape(abbreviate(row.getValueString("errorMessage"), 240)));
+         }
+         html.rawList(items);
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read recent search failures for the diagnostics widget", e);
+         html.status(Level.WARN, "Could not read the diagnostic events: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderLastValidation(DiagnosticsHtml html, QuickSearchQBitConfig config)
+   {
+      String tableName = config.applyPrefix(QuickSearchValidationRun.TABLE_NAME);
+      html.subheading("Latest index validation");
+      if(QContext.getQInstance().getTable(tableName) == null)
+      {
+         html.paragraph("Not kept: the validation run table is not configured.");
+         return;
+      }
+
+      try
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
+         List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
+         if(rows.isEmpty())
+         {
+            html.paragraph("None yet. Run Validate Search Index from Quick Search Index (Actions)"
+               + (config.getValidationCronExpression() == null ? "." : "; it is also scheduled (" + config.getValidationCronExpression() + ")."));
+            return;
+         }
+
+         QRecord row    = rows.get(0);
+         String  status = row.getValueString("status");
+         Level   level  = QuickSearchValidationRun.STATUS_PASS.equals(status) ? Level.OK : (QuickSearchValidationRun.STATUS_WARN.equals(status) ? Level.WARN : Level.FAIL);
+         html.raw("<p>" + DiagnosticsHtml.badge(level) + " <a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
+            + DiagnosticsHtml.escape(row.getValueString("startTime")) + "</a> (" + DiagnosticsHtml.escape(row.getValueString("triggerType")) + "): "
+            + DiagnosticsHtml.escape(row.getValueString("summary")) + "</p>");
+      }
+      catch(Exception e)
+      {
+         LOG.info("Could not read the latest validation run for the diagnostics widget", e);
+         html.status(Level.WARN, "Could not read the validation runs: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String shortClass(String className)
+   {
+      return (className == null ? "error" : className.substring(className.lastIndexOf('.') + 1));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String abbreviate(String text, int max)
+   {
+      return (text == null ? "" : (text.length() > max ? text.substring(0, max) + "..." : text));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/RecordInspector.java

@@ -0,0 +1,410 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Check;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** Inspect one record's search document: the document as indexed next to the
+ ** document indexing would write from the database now (field by field),
+ ** and, given a term, why the record does or does not match it.
+ ** The match analysis walks the same gates as the global search: term
+ ** length, table indexed and enabled, read permission, record readable,
+ ** document present and current, every query token present among the
+ ** document's indexed tokens (edge n-grams of 2 to 20 characters, AND
+ ** semantics), explain, and the record's rank against the provider's
+ ** per-table id window. Read-only.
+ *******************************************************************************/
+public class RecordInspector
+{
+   public static final String VERDICT_CURRENT     = "CURRENT";
+   public static final String VERDICT_STALE       = "STALE";
+   public static final String VERDICT_MISSING     = "MISSING";
+   public static final String VERDICT_NOT_INDEXED = "NOT_INDEXED";
+   public static final String VERDICT_NO_RECORD   = "NO_RECORD";
+
+   private static final int RANK_WINDOW = RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE;
+
+
+
+   /*******************************************************************************
+    ** One compared value: name, value from the database, value in the index.
+    *******************************************************************************/
+   public record FieldDiff(String name, String databaseValue, String indexValue)
+   {
+      /*******************************************************************************
+       ** Whether the two values are equal.
+       *******************************************************************************/
+      public boolean isSame()
+      {
+         return (Objects.equals(databaseValue, indexValue));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The inspection of one record.
+    *******************************************************************************/
+   public record Inspection(String tableName, String recordId, String verdict, List<FieldDiff> diffs, DocumentView indexed, OpenSearchDocument expected,
+                            List<Check> matchChecks, ExplainResult explain, String error)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect one record, with an optional term to analyze.
+    *******************************************************************************/
+   public Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, String term, AbstractActionInput permissionInput) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         return (new Inspection(tableName, recordId, VERDICT_NOT_INDEXED, List.of(), null, null, List.of(), null, "Quick Search does not index table [" + tableName + "]."));
+      }
+
+      Map<String, QRecord> records = DiagnosticRecordReads.loadForIndexing(tableName, List.of(recordId));
+      QRecord              record  = records.get(recordId);
+      if(record == null && !records.isEmpty())
+      {
+         record = records.values().iterator().next();
+      }
+      if(record == null)
+      {
+         return (new Inspection(tableName, recordId, VERDICT_NO_RECORD, List.of(), null, null, List.of(), null,
+            "Record " + recordId + " does not exist, or you cannot read it."));
+      }
+
+      QuickSearchDiagnosticsOperations ops      = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      OpenSearchDocument               expected = IndexingUtils.buildDocument(record, tableConfig);
+      DocumentView                     indexed  = ops.getDocument(tableName, recordId);
+      List<FieldDiff>                  diffs    = diff(tableConfig, expected, indexed);
+
+      String verdict = !Boolean.TRUE.equals(indexed.found()) ? VERDICT_MISSING : (diffs.stream().allMatch(FieldDiff::isSame) ? VERDICT_CURRENT : VERDICT_STALE);
+
+      List<Check>   checks  = new ArrayList<>();
+      ExplainResult explain = null;
+      String        cleaned = IndexingUtils.normalizeSearchText(term);
+      if(StringUtils.hasContent(cleaned))
+      {
+         explain = analyzeMatch(runtime, ops, tableConfig, recordId, cleaned, verdict, expected, indexed, checks, permissionInput);
+      }
+
+      return (new Inspection(tableName, recordId, verdict, diffs, indexed, expected, checks, explain, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare the expected document with the indexed one: label, searchable
+    ** text, then each configured field (in configuration order) and any extra
+    ** field only the index holds.
+    *******************************************************************************/
+   static List<FieldDiff> diff(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, DocumentView indexed)
+   {
+      JsonNode source = indexed == null || indexed.source() == null ? null : indexed.source();
+
+      List<FieldDiff> diffs = new ArrayList<>();
+      diffs.add(new FieldDiff("recordLabel", expected == null ? null : blankToNull(expected.getRecordLabel()), text(source, "recordLabel")));
+      diffs.add(new FieldDiff("searchableText", expected == null ? null : blankToNull(expected.getSearchableText()), text(source, "searchableText")));
+
+      Set<String> fieldNames = new LinkedHashSet<>(CollectionUtils.nonNullList(tableConfig.getSearchableFields()));
+      if(source != null && source.has("fieldValues"))
+      {
+         source.get("fieldValues").fieldNames().forEachRemaining(fieldNames::add);
+      }
+      for(String field : fieldNames)
+      {
+         Object expectedValue = expected == null || expected.getFieldValues() == null ? null : expected.getFieldValues().get(field);
+         diffs.add(new FieldDiff("fieldValues." + field, expectedValue == null ? null : String.valueOf(expectedValue),
+            source == null ? null : text(source.path("fieldValues"), field)));
+      }
+      return (diffs);
+   }
+
+
+
+   /*******************************************************************************
+    ** Walk the global search's gates for this record and term, adding a check
+    ** per gate; returns the explain result when the document exists.
+    *******************************************************************************/
+   private ExplainResult analyzeMatch(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, String recordId, String term, String verdict,
+      OpenSearchDocument expected, DocumentView indexed, List<Check> checks, AbstractActionInput permissionInput) throws QException
+   {
+      String tableName = tableConfig.getTableName();
+
+      checks.add(term.length() < QuickSearchAction.MIN_TERM_LENGTH
+         ? new Check("Term length", Level.FAIL, "Terms shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters never reach OpenSearch.")
+         : new Check("Term length", Level.OK, term.length() + " characters."));
+
+      boolean enabled = runtime.isTableEnabled(tableName);
+      checks.add(enabled
+         ? new Check("Table enabled", Level.OK, "The table is enabled in Quick Search Index.")
+         : new Check("Table enabled", Level.FAIL, "The table is disabled in Quick Search Index, so Quick Search does not search it (core searches it with its own database search)."));
+
+      boolean readable = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
+      checks.add(readable
+         ? new Check("Table permission", Level.OK, "You can read this table.")
+         : new Check("Table permission", Level.FAIL, "You cannot read this table, so the global search skips it for you."));
+
+      checks.add(new Check("Record access", Level.OK, "You can read this record (record security locks allow it)."));
+
+      switch(verdict)
+      {
+         case VERDICT_MISSING -> checks.add(new Check("Indexed document", Level.FAIL,
+            "The record has no document in the index, so OpenSearch cannot return it. Use Reindex in Search on the record (or wait for the next basepull / reconcile)."));
+         case VERDICT_STALE -> checks.add(new Check("Indexed document", Level.WARN,
+            "The indexed document differs from the database (see the comparison); the search matches the indexed values. Reindex the record to refresh it."));
+         default -> checks.add(new Check("Indexed document", Level.OK, "The indexed document matches the database."));
+      }
+
+      List<AnalyzedToken> queryTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
+      Set<String>         indexedGrams = Boolean.TRUE.equals(indexed.found()) ? tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, text(indexed.source(), "searchableText")))
+         : Set.of();
+      Set<String>         expectedGrams = expected == null ? Set.of() : tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, expected.getSearchableText()));
+
+      if(queryTokens.isEmpty())
+      {
+         checks.add(new Check("Term tokens", Level.FAIL, "The term has no searchable characters after analysis."));
+      }
+      for(AnalyzedToken token : queryTokens)
+      {
+         String value = token.token();
+         if(value.length() < 2)
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "One character: the index holds edge n-grams of 2 to 20 characters, so a single character never matches, and every token must match."));
+         }
+         else if(value.length() > 20)
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Longer than 20 characters: the index holds edge n-grams of at most 20 characters, so this token never matches. Search a shorter prefix."));
+         }
+         else if(indexedGrams.contains(value))
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.OK, "Found in the indexed text (as a word or a word prefix)."));
+         }
+         else if(expectedGrams.contains(value))
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.WARN, "Not in the indexed text, but in the database's current values: it will match once the record is reindexed."));
+         }
+         else
+         {
+            checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Not in this record's searchable text: no searchable field holds a word starting with it. Every token must match (AND)."));
+         }
+      }
+
+      ExplainResult explain = null;
+      if(Boolean.TRUE.equals(indexed.found()))
+      {
+         explain = ops.explain(term, tableName, recordId, runtime.getDiscoveredTables());
+         checks.add(Boolean.TRUE.equals(explain.matched())
+            ? new Check("OpenSearch explain", Level.OK, "The search query matches this document with score " + String.format("%.4f", explain.score()) + ".")
+            : new Check("OpenSearch explain", Level.FAIL, "The search query does not match this document."));
+
+         if(Boolean.TRUE.equals(explain.matched()))
+         {
+            checks.add(rankCheck(runtime, ops, tableName, recordId, term));
+         }
+      }
+      return (explain);
+   }
+
+
+
+   /*******************************************************************************
+    ** Where the record ranks among the table's matches, against the number of
+    ** ids the provider asks for with the default per-table limit.
+    *******************************************************************************/
+   private Check rankCheck(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, String tableName, String recordId, String term) throws QException
+   {
+      int         window = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
+      SearchProbe probe  = ops.probeSearch(term, tableName, RANK_WINDOW, runtime.getDiscoveredTables());
+      int         rank   = 0;
+      for(ProbeHit hit : probe.hits())
+      {
+         rank++;
+         if(recordId.equals(hit.recordId()))
+         {
+            if(rank <= RecordSearchAction.DEFAULT_LIMIT_PER_TABLE)
+            {
+               return (new Check("Rank", Level.OK, "#" + rank + " of " + probe.totalHits() + " matches in this table: within the global search's default " + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " results."));
+            }
+            if(rank <= window)
+            {
+               return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: Quick Search returns it, but the global search shows only the first "
+                  + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " readable results per table by default. A more specific term ranks it higher."));
+            }
+            return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: beyond the top " + window + " ids the provider requests per table, so the global search does not show it."));
+         }
+      }
+      return (new Check("Rank", Level.WARN, "Not in the top " + RANK_WINDOW + " of " + probe.totalHits() + " matches in this table, so the global search does not show it."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render inspections as HTML.
+    *******************************************************************************/
+   public static String render(List<Inspection> inspections, String term)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      for(Inspection inspection : inspections)
+      {
+         html.heading("Search document for " + inspection.tableName() + " #" + inspection.recordId());
+         if(inspection.error() != null)
+         {
+            html.status(Level.FAIL, inspection.error());
+            continue;
+         }
+
+         switch(inspection.verdict())
+         {
+            case VERDICT_CURRENT -> html.status(Level.OK, "Indexed and current: the document matches the database.");
+            case VERDICT_STALE -> html.status(Level.WARN, "Indexed but stale: some values differ from the database. Reindex in Search refreshes it.");
+            default -> html.status(Level.FAIL, "Not in the index. Reindex in Search adds it.");
+         }
+
+         DocumentView indexed = inspection.indexed();
+         List<String[]> facts = new ArrayList<>();
+         facts.add(new String[] { "Document id", OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId()) });
+         facts.add(new String[] { "Physical index", indexed == null ? null : indexed.physicalIndex() });
+         facts.add(new String[] { "Indexed version", indexed == null || indexed.version() == null ? null : indexed.version() + versionNote(indexed.version()) });
+         facts.add(new String[] { "Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)"
+            : inspection.expected().getVersion() + versionNote(inspection.expected().getVersion()) });
+         facts.add(new String[] { "Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null) });
+         html.facts(facts);
+
+         html.subheading("Database vs index");
+         List<String> items = new ArrayList<>();
+         for(FieldDiff diff : inspection.diffs())
+         {
+            items.add(DiagnosticsHtml.badge(diff.isSame() ? Level.OK : Level.WARN) + " <strong>" + DiagnosticsHtml.escape(diff.name()) + "</strong>"
+               + (diff.isSame() ? ": " + valueHtml(diff.databaseValue())
+               : "<br>database: " + valueHtml(diff.databaseValue()) + "<br>index: " + valueHtml(diff.indexValue())));
+         }
+         html.rawList(items);
+
+         if(!inspection.matchChecks().isEmpty())
+         {
+            html.subheading("Why does (or doesn't) this record match \"" + term + "\"?");
+            html.checks(inspection.matchChecks());
+            if(inspection.explain() != null && inspection.explain().explanationText() != null)
+            {
+               html.textDetails("Score explanation", inspection.explain().explanationText());
+            }
+         }
+
+         if(indexed != null && indexed.source() != null)
+         {
+            html.jsonDetails("Indexed document (JSON)", indexed.source().toString());
+         }
+      }
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String valueHtml(String value)
+   {
+      if(value == null)
+      {
+         return ("<em>none</em>");
+      }
+      String shown = value.length() > 500 ? value.substring(0, 500) + "... (" + value.length() + " characters)" : value;
+      return ("<code>" + DiagnosticsHtml.escape(shown) + "</code>");
+   }
+
+
+
+   /*******************************************************************************
+    ** " (time)" for a version that looks like epoch millis.
+    *******************************************************************************/
+   private static String versionNote(Long version)
+   {
+      return (version != null && version > 946_684_800_000L ? " (" + java.time.Instant.ofEpochMilli(version) + ")" : "");
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static Set<String> tokenSet(List<AnalyzedToken> tokens)
+   {
+      Set<String> set = new LinkedHashSet<>();
+      for(AnalyzedToken token : tokens)
+      {
+         set.add(token.token());
+      }
+      return (set);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String text(JsonNode node, String field)
+   {
+      if(node == null || !node.has(field) || node.get(field).isNull())
+      {
+         return (null);
+      }
+      return (blankToNull(node.get(field).isTextual() ? node.get(field).asText() : node.get(field).toString()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static String blankToNull(String value)
+   {
+      return (value == null || value.isEmpty() ? null : value);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/RecordReindexer.java

@@ -0,0 +1,201 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** Reindex chosen records of one table now: each record the user can read is
+ ** rebuilt from the database and written to the index (versioned, so a newer
+ ** document is never overwritten); a record that no longer exists has its
+ ** document removed; a record that exists but that the user cannot read is
+ ** skipped. The one diagnostic tool that writes to the index; it never
+ ** changes a record. A disabled table is refused.
+ *******************************************************************************/
+public class RecordReindexer
+{
+   public static final String OUTCOME_INDEXED = "INDEXED";
+   public static final String OUTCOME_DELETED = "DELETED";
+   public static final String OUTCOME_SKIPPED = "SKIPPED";
+   public static final String OUTCOME_FAILED  = "FAILED";
+
+
+
+   /*******************************************************************************
+    ** What happened to one record.
+    *******************************************************************************/
+   public record Outcome(String recordId, String outcome, String message)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Reindex the records of a table by id.
+    *******************************************************************************/
+   public List<Outcome> reindex(QuickSearchRuntime runtime, String tableName, List<String> recordIds) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Quick Search does not index table [" + tableName + "]"));
+      }
+      if(!runtime.isTableEnabled(tableName))
+      {
+         throw (new QException("Table [" + tableName + "] is disabled in Quick Search Index; enable it before reindexing its records"));
+      }
+
+      Set<String>          ids      = new LinkedHashSet<>(recordIds);
+      Map<String, QRecord> readable = DiagnosticRecordReads.loadForIndexing(tableName, ids);
+      Set<String>          validIds = new HashSet<>();
+      DiagnosticRecordReads.typedIds(QContext.getQInstance().getTable(tableName), ids).forEach(id -> validIds.add(String.valueOf(id)));
+
+      ////////////////////////////////////////////////////////////////////
+      // ids the user cannot read: exists at all (system view)? if not, //
+      // the record is gone and its document should go too              //
+      ////////////////////////////////////////////////////////////////////
+      List<String> unreadable = ids.stream().filter(id -> !readable.containsKey(id)).toList();
+      Set<String>  existing   = new HashSet<>();
+      if(!unreadable.isEmpty())
+      {
+         List<String> found = new ArrayList<>();
+         QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
+            found.addAll(DiagnosticRecordReads.existingIds(tableName, unreadable)));
+         existing.addAll(found);
+      }
+
+      QuickSearchOpenSearchClient client   = runtime.getClient();
+      List<Outcome>               outcomes = new ArrayList<>();
+
+      List<OpenSearchDocument> docs = new ArrayList<>();
+      for(String id : ids)
+      {
+         QRecord record = readable.get(id);
+         if(record != null)
+         {
+            OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
+            if(doc != null)
+            {
+               docs.add(doc);
+            }
+         }
+      }
+
+      BulkIndexResult indexResult = client.indexDocuments(docs, Math.max(1, runtime.getConfig().getBulkBatchSize() == null ? 500 : runtime.getConfig().getBulkBatchSize()));
+      Set<String>     failedIds   = new HashSet<>();
+      for(String failure : indexResult.getErrors())
+      {
+         ////////////////////////////////////////////////////
+         // failures read "table:recordId: reason"          //
+         ////////////////////////////////////////////////////
+         int    reasonStart = failure.indexOf(": ");
+         String documentId  = reasonStart < 0 ? failure : failure.substring(0, reasonStart);
+         failedIds.add(documentId.substring(documentId.indexOf(':') + 1));
+      }
+
+      List<String> toDelete = new ArrayList<>();
+      for(String id : ids)
+      {
+         if(readable.containsKey(id))
+         {
+            outcomes.add(failedIds.contains(id) ? new Outcome(id, OUTCOME_FAILED, "OpenSearch rejected the document; see the run's errors")
+               : new Outcome(id, OUTCOME_INDEXED, "Rebuilt from the database and written to the index."));
+         }
+         else if(!validIds.contains(id))
+         {
+            outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "Not a valid id for this table."));
+         }
+         else if(existing.contains(id))
+         {
+            outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "The record exists but you cannot read it, so it was not reindexed."));
+         }
+         else
+         {
+            toDelete.add(OpenSearchDocument.buildDocumentId(tableName, id));
+            outcomes.add(new Outcome(id, OUTCOME_DELETED, "The record no longer exists; its document (if any) was removed."));
+         }
+      }
+
+      if(!toDelete.isEmpty())
+      {
+         BulkIndexResult deleteResult = client.deleteDocuments(toDelete, 500);
+         if(!deleteResult.getErrors().isEmpty())
+         {
+            throw (new QException("Removing documents of deleted records failed: " + String.join("; ", deleteResult.getErrors())));
+         }
+      }
+
+      if(!indexResult.getErrors().isEmpty())
+      {
+         outcomes.add(new Outcome("-", OUTCOME_FAILED, "Index errors: " + String.join("; ", indexResult.getErrors())));
+      }
+
+      client.refreshIndex();
+      return (outcomes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Render outcomes as HTML.
+    *******************************************************************************/
+   public static String render(String tableName, List<Outcome> outcomes)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Reindex in search: " + tableName);
+      long indexed = outcomes.stream().filter(o -> OUTCOME_INDEXED.equals(o.outcome())).count();
+      long deleted = outcomes.stream().filter(o -> OUTCOME_DELETED.equals(o.outcome())).count();
+      long skipped = outcomes.stream().filter(o -> OUTCOME_SKIPPED.equals(o.outcome())).count();
+      long failed  = outcomes.stream().filter(o -> OUTCOME_FAILED.equals(o.outcome())).count();
+      html.status(failed > 0 ? Level.FAIL : (skipped > 0 ? Level.WARN : Level.OK),
+         indexed + " reindexed, " + deleted + " removed (record gone), " + skipped + " skipped, " + failed + " failed. The documents are searchable now.");
+
+      List<String> items = new ArrayList<>();
+      for(Outcome outcome : outcomes)
+      {
+         Level level = switch(outcome.outcome())
+         {
+            case OUTCOME_INDEXED, OUTCOME_DELETED -> Level.OK;
+            case OUTCOME_SKIPPED -> Level.WARN;
+            default -> Level.FAIL;
+         };
+         items.add(DiagnosticsHtml.badge(level) + " #" + DiagnosticsHtml.escape(outcome.recordId()) + ": " + DiagnosticsHtml.escape(outcome.message()));
+      }
+      html.rawList(items);
+      return (html.close().toString());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/SearchPathDiagnosis.java

@@ -0,0 +1,178 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+
+
+/*******************************************************************************
+ ** Which path the global search (core RecordSearchAction, POST
+ ** /qqq/v1/search) takes for each table, and why: OpenSearch through the
+ ** Quick Search record-search provider, core's own database search, or not
+ ** searched at all. Mirrors RecordSearchAction's table selection and
+ ** QuickSearchRecordSearchProvider.claimsTable without calling them, so a
+ ** diagnosis changes no state (no backoff, no counters).
+ *******************************************************************************/
+public final class SearchPathDiagnosis
+{
+   public static final String PATH_OPENSEARCH = "OpenSearch (Quick Search provider)";
+   public static final String PATH_CORE       = "Core database search";
+   public static final String PATH_NONE       = "Not searched";
+
+
+
+   /*******************************************************************************
+    ** The record-search provider's state in this JVM.
+    *******************************************************************************/
+   public record ProviderState(String registeredProvider, Boolean quickSearchProvider, Boolean serveCoreRecordSearch, Long backoffRemainingMillis,
+                               Long failureCount, Instant lastFailureTime, String lastFailureMessage, Long backoffDeclinedClaims, Integer timeoutMillis, Instant runtimeStartedAt)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's path and the reason for it.
+    *******************************************************************************/
+   public record TablePath(String tableName, String tableLabel, Boolean indexed, Boolean enabled, Boolean readable, Boolean coreSearchable, String path, String reason)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class - no instances.
+    *******************************************************************************/
+   private SearchPathDiagnosis()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The provider's state from the instance and the runtime.
+    *******************************************************************************/
+   public static ProviderState providerState(QInstance qInstance, QuickSearchRuntime runtime)
+   {
+      QCodeReference        provider = qInstance.getRecordSearchProvider();
+      QuickSearchQBitConfig config   = runtime.getConfig();
+      return (new ProviderState(
+         provider == null ? null : provider.getName(),
+         provider != null && QuickSearchRecordSearchProvider.class.getName().equals(provider.getName()),
+         Boolean.TRUE.equals(config.getServeCoreRecordSearch()),
+         runtime.getRecordSearchBackoffRemainingMillis(),
+         runtime.getRecordSearchFailureCount(),
+         runtime.getLastRecordSearchFailureTime(),
+         runtime.getLastRecordSearchFailureMessage(),
+         runtime.getBackoffDeclinedClaimCount(),
+         config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis(),
+         runtime.getStartedAt()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The path of each named table (unknown names are reported as not
+    ** searched), as seen by the session of permissionInput.
+    *******************************************************************************/
+   public static List<TablePath> diagnose(QInstance qInstance, QuickSearchRuntime runtime, List<String> tableNames, AbstractActionInput permissionInput)
+   {
+      ProviderState   state = providerState(qInstance, runtime);
+      List<TablePath> paths = new ArrayList<>();
+      for(String tableName : CollectionUtils.nonNullList(tableNames))
+      {
+         paths.add(diagnoseTable(qInstance, runtime, state, tableName, permissionInput));
+      }
+      return (paths);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static TablePath diagnoseTable(QInstance qInstance, QuickSearchRuntime runtime, ProviderState state, String tableName, AbstractActionInput permissionInput)
+   {
+      QTableMetaData table = qInstance.getTable(tableName);
+      if(table == null)
+      {
+         return (new TablePath(tableName, tableName, false, false, false, false, PATH_NONE, "There is no table with this name."));
+      }
+
+      boolean indexed        = runtime.getTableConfig(tableName) != null;
+      boolean enabled        = indexed && runtime.isTableEnabled(tableName);
+      boolean readable       = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
+      boolean coreSearchable = !CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) && table.isCapabilityEnabled(qInstance.getBackendForTable(tableName), Capability.TABLE_QUERY);
+      String  label          = table.getLabel() == null ? tableName : table.getLabel();
+
+      if(!coreSearchable)
+      {
+         return (new TablePath(tableName, label, indexed, enabled, readable, false, PATH_NONE,
+            "Global search skips this table: it has no core searchFields (or does not support query)."));
+      }
+      if(!readable)
+      {
+         return (new TablePath(tableName, label, indexed, enabled, false, true, PATH_NONE,
+            "Global search skips this table for you: you do not have read permission on it."));
+      }
+      if(!indexed)
+      {
+         return (new TablePath(tableName, label, false, false, true, true, PATH_CORE, "Quick Search does not index this table."));
+      }
+      if(!Boolean.TRUE.equals(state.serveCoreRecordSearch()))
+      {
+         return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, "serveCoreRecordSearch is off in the Quick Search configuration."));
+      }
+      if(!Boolean.TRUE.equals(state.quickSearchProvider()))
+      {
+         String reason = state.registeredProvider() == null
+            ? "No record search provider is registered on the instance."
+            : "Another record search provider is registered (" + state.registeredProvider() + "), so Quick Search does not serve global search.";
+         return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, reason));
+      }
+      if(!enabled)
+      {
+         return (new TablePath(tableName, label, true, false, true, true, PATH_CORE, "The table is disabled in Quick Search Index (enabled = false)."));
+      }
+      if(state.backoffRemainingMillis() != null && state.backoffRemainingMillis() > 0)
+      {
+         return (new TablePath(tableName, label, true, true, true, true, PATH_CORE,
+            "Quick Search is backing off for " + Math.max(1, state.backoffRemainingMillis() / 1000) + " more second(s) after an OpenSearch failure"
+               + (state.lastFailureTime() == null ? "" : " at " + state.lastFailureTime())
+               + (state.lastFailureMessage() == null ? "" : ": " + state.lastFailureMessage())));
+      }
+
+      return (new TablePath(tableName, label, true, true, true, true, PATH_OPENSEARCH,
+         "Claimed by the Quick Search provider. If OpenSearch fails or takes longer than " + state.timeoutMillis()
+            + " ms, core searches this table itself for that request and Quick Search backs off for "
+            + (QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS / 1000) + " seconds."));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/SearchPlayground.java

@@ -0,0 +1,654 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.diagnostics;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
+import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
+
+
+/*******************************************************************************
+ ** The search playground: runs a term the way the global search does, as the
+ ** current user, and shows everything behind it.
+ ** 1. Which path serves each table and why (SearchPathDiagnosis).
+ ** 2. The provider's own OpenSearch request (_msearch of record ids), timed,
+ **    without starting a backoff when it fails.
+ ** 3. The real global search (core RecordSearchAction) as the current user:
+ **    what users see, per table, timed. Like any user search, a provider
+ **    failure here starts the provider's normal backoff.
+ ** 4. Per indexed table, the production query with scores, highlights and
+ **    matched clauses, which hits users can read (record locks), which ones
+ **    the global search shows, the query DSL and the raw response (hits the
+ **    user cannot read are redacted to their id).
+ ** 5. Optionally, explain for one hit, and the term's tokens.
+ ** Read-only: no index or record is changed.
+ *******************************************************************************/
+public class SearchPlayground
+{
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+
+
+   /*******************************************************************************
+    ** What to run: term, tables (empty for every searchable table), results per
+    ** table as in the global search, and an optional hit to explain
+    ** ("table:recordId").
+    *******************************************************************************/
+   public record Input(String term, List<String> tableNames, Integer limitPerTable, String explainDocumentId)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One table's probe: the search, which hit ids the user can read, which
+    ** the global search showed, and an error when the probe failed.
+    *******************************************************************************/
+   public record TableProbe(String tableName, SearchProbe probe, Set<String> readableIds, Set<String> shownIds, Integer hiddenCount, String error)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Everything the playground found.
+    *******************************************************************************/
+   public record Report(String term, List<String> notices, List<AnalyzedToken> termTokens, String termTokensError,
+                        SearchPathDiagnosis.ProviderState providerState, List<SearchPathDiagnosis.TablePath> paths,
+                        Map<String, List<String>> providerIds, Map<String, Integer> providerHiddenCounts, Long providerMillis, String providerError,
+                        Map<String, List<RecordSearchResult>> coreResults, Long coreMillis, String coreError,
+                        List<TableProbe> probes, String explainDocumentId, ExplainResult explain, String explainError,
+                        Integer limitPerTable, Integer idsPerTable)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the playground for the input as the current session.
+    *******************************************************************************/
+   public Report run(QuickSearchRuntime runtime, Input input, AbstractActionInput permissionInput)
+   {
+      QInstance             qInstance = QContext.getQInstance();
+      QuickSearchQBitConfig config    = runtime.getConfig();
+      List<String>          notices   = new ArrayList<>();
+
+      String term          = IndexingUtils.normalizeSearchText(input.term());
+      int    limitPerTable = input.limitPerTable() == null || input.limitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : Math.min(input.limitPerTable(), RecordSearchAction.MAX_LIMIT_PER_TABLE);
+      int    idsPerTable   = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
+
+      List<String> tables = resolveTables(qInstance, runtime, input.tableNames(), notices);
+      List<SearchPathDiagnosis.TablePath> paths = SearchPathDiagnosis.diagnose(qInstance, runtime, tables, permissionInput);
+      SearchPathDiagnosis.ProviderState   state = SearchPathDiagnosis.providerState(qInstance, runtime);
+
+      if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         notices.add("The term is shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters: the Quick Search provider returns nothing for it, so only core's own search can match.");
+      }
+      if(term.length() > RecordSearchAction.MAX_SEARCH_TERM_LENGTH)
+      {
+         notices.add("The term is longer than " + RecordSearchAction.MAX_SEARCH_TERM_LENGTH + " characters: the global search rejects it.");
+      }
+
+      QuickSearchDiagnosticsOperations ops = null;
+      String                           opsError = null;
+      try
+      {
+         ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
+      }
+      catch(QException e)
+      {
+         opsError = e.getMessage();
+         notices.add("The OpenSearch client could not be built: " + e.getMessage());
+      }
+
+      ////////////////////////
+      // term tokens        //
+      ////////////////////////
+      List<AnalyzedToken> termTokens      = null;
+      String              termTokensError = opsError;
+      if(ops != null && StringUtils.hasContent(term))
+      {
+         try
+         {
+            termTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
+         }
+         catch(QException e)
+         {
+            termTokensError = e.getMessage();
+         }
+      }
+
+      ///////////////////////////////////////////////////////////
+      // the provider's own request, for the tables it claims  //
+      ///////////////////////////////////////////////////////////
+      List<String> claimed = paths.stream().filter(p -> SearchPathDiagnosis.PATH_OPENSEARCH.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
+      Map<String, List<String>> providerIds    = null;
+      Map<String, Integer>      providerHidden = new LinkedHashMap<>();
+      Long                      providerMillis = null;
+      String                    providerError  = null;
+      if(!claimed.isEmpty() && ops != null && term.length() >= QuickSearchAction.MIN_TERM_LENGTH)
+      {
+         int  timeout = state.timeoutMillis();
+         long start   = System.currentTimeMillis();
+         try
+         {
+            providerIds = runtime.getClient().searchRecordIdsPerTable(term, claimed, idsPerTable, runtime.getDiscoveredTables(), timeout);
+         }
+         catch(QException | RuntimeException e)
+         {
+            providerError = e.getClass().getSimpleName() + ": " + e.getMessage();
+         }
+         providerMillis = System.currentTimeMillis() - start;
+         providerIds    = providerIds == null ? null : readableOnly(providerIds, providerHidden);
+      }
+
+      ///////////////////////////////////////////////////////
+      // the real global search, as the current user       //
+      ///////////////////////////////////////////////////////
+      Map<String, List<RecordSearchResult>> coreResults = new LinkedHashMap<>();
+      Long                                  coreMillis  = null;
+      String                                coreError   = null;
+      List<String> searched = paths.stream().filter(p -> !SearchPathDiagnosis.PATH_NONE.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
+      if(!searched.isEmpty() && StringUtils.hasContent(term))
+      {
+         long start = System.currentTimeMillis();
+         try
+         {
+            RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm(term).withTableNames(searched).withLimitPerTable(limitPerTable));
+            for(RecordSearchResult result : CollectionUtils.nonNullList(output.getResults()))
+            {
+               coreResults.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
+            }
+         }
+         catch(Exception e)
+         {
+            coreError = e.getClass().getSimpleName() + ": " + e.getMessage();
+         }
+         coreMillis = System.currentTimeMillis() - start;
+      }
+
+      //////////////////////////////////////////
+      // per indexed table, the probe search  //
+      //////////////////////////////////////////
+      List<TableProbe> probes = new ArrayList<>();
+      for(SearchPathDiagnosis.TablePath path : paths)
+      {
+         if(!Boolean.TRUE.equals(path.indexed()) || !Boolean.TRUE.equals(path.readable()) || term.length() < QuickSearchAction.MIN_TERM_LENGTH)
+         {
+            continue;
+         }
+         if(ops == null)
+         {
+            probes.add(new TableProbe(path.tableName(), null, Set.of(), Set.of(), 0, opsError));
+            continue;
+         }
+         probes.add(probeTable(ops, runtime, term, path.tableName(), idsPerTable, coreResults.get(path.tableName())));
+      }
+
+      //////////////////////
+      // explain one hit  //
+      //////////////////////
+      ExplainResult explain      = null;
+      String        explainError = null;
+      String        explainId    = StringUtils.hasContent(input.explainDocumentId()) ? input.explainDocumentId().trim() : null;
+      if(explainId != null && ops != null)
+      {
+         try
+         {
+            explain = explainHit(ops, runtime, term, explainId);
+         }
+         catch(QException e)
+         {
+            explainError = e.getMessage();
+         }
+      }
+
+      return (new Report(term, notices, termTokens, termTokensError, state, paths, providerIds, providerHidden, providerMillis, providerError,
+         coreResults, coreMillis, coreError, probes, explainId, explain, explainError, limitPerTable, idsPerTable));
+   }
+
+
+
+   /*******************************************************************************
+    ** The requested tables that exist (else every table with core searchFields
+    ** or a Quick Search index), in instance order.
+    *******************************************************************************/
+   static List<String> resolveTables(QInstance qInstance, QuickSearchRuntime runtime, List<String> requested, List<String> notices)
+   {
+      List<String> tables = new ArrayList<>();
+      if(!CollectionUtils.nullSafeIsEmpty(requested))
+      {
+         for(String tableName : new LinkedHashSet<>(requested))
+         {
+            if(qInstance.getTable(tableName) == null)
+            {
+               notices.add("Ignored unknown table [" + tableName + "].");
+            }
+            else
+            {
+               tables.add(tableName);
+            }
+         }
+         return (tables);
+      }
+
+      for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values())
+      {
+         if(!CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) || runtime.getTableConfig(table.getName()) != null)
+         {
+            tables.add(table.getName());
+         }
+      }
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Probe one table and mark which hits the user can read and which ones the
+    ** global search showed.
+    *******************************************************************************/
+   private TableProbe probeTable(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String tableName, int size, List<RecordSearchResult> shown)
+   {
+      try
+      {
+         SearchProbe  probe = ops.probeSearch(term, tableName, size, runtime.getDiscoveredTables());
+         List<String> ids   = probe.hits().stream().map(ProbeHit::recordId).toList();
+
+         Set<String> readable = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, ids));
+         Set<String> shownIds = new HashSet<>();
+         for(RecordSearchResult result : CollectionUtils.nonNullList(shown))
+         {
+            shownIds.add(String.valueOf(result.getRecordId()));
+         }
+
+         List<ProbeHit> visible = probe.hits().stream().filter(h -> readable.contains(h.recordId())).toList();
+         int            hidden  = probe.hits().size() - visible.size();
+         SearchProbe redacted = new SearchProbe(probe.tableName(), probe.requestJson(), redactResponse(probe.responseJson(), readable),
+            probe.tookMillis(), probe.wallMillis(), hidden == 0 ? probe.totalHits() : null, probe.totalIsLowerBound(), visible);
+         return (new TableProbe(tableName, redacted, readable, shownIds, hidden, null));
+      }
+      catch(QException e)
+      {
+         return (new TableProbe(tableName, null, Set.of(), Set.of(), 0, e.getMessage()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The provider's ids per table, keeping only records the user can read;
+    ** how many were dropped goes in hiddenCounts. Ids of records the user
+    ** cannot read are never shown (they would reveal what a locked record
+    ** contains).
+    *******************************************************************************/
+   static Map<String, List<String>> readableOnly(Map<String, List<String>> idsByTable, Map<String, Integer> hiddenCounts)
+   {
+      Map<String, List<String>> readable = new LinkedHashMap<>();
+      for(Map.Entry<String, List<String>> entry : idsByTable.entrySet())
+      {
+         List<String> ids = entry.getValue() == null ? List.of() : entry.getValue();
+         Set<String>  ok;
+         try
+         {
+            ok = new HashSet<>(DiagnosticRecordReads.existingIds(entry.getKey(), ids));
+         }
+         catch(QException e)
+         {
+            ok = Set.of();
+         }
+         Set<String> finalOk = ok;
+         readable.put(entry.getKey(), ids.stream().filter(finalOk::contains).toList());
+         hiddenCounts.put(entry.getKey(), (int) ids.stream().filter(id -> !finalOk.contains(id)).count());
+      }
+      return (readable);
+   }
+
+
+
+   /*******************************************************************************
+    ** The raw response without the hits the user cannot read: they are
+    ** removed whole (id, score, source, highlights, matched queries), and
+    ** when any was removed the total and max score are redacted too, so the
+    ** response reveals nothing about records the user cannot read.
+    *******************************************************************************/
+   static String redactResponse(String responseJson, Set<String> readableIds)
+   {
+      try
+      {
+         JsonNode root    = MAPPER.readTree(responseJson);
+         JsonNode hitList = root.path("hits").path("hits");
+         int      removed = 0;
+         if(hitList.isArray())
+         {
+            ArrayNode array = (ArrayNode) hitList;
+            for(int i = array.size() - 1; i >= 0; i--)
+            {
+               String recordId = array.get(i).path("_source").path("recordId").asText(null);
+               if(recordId == null || !readableIds.contains(recordId))
+               {
+                  array.remove(i);
+                  removed++;
+               }
+            }
+         }
+         if(removed > 0 && root.path("hits") instanceof ObjectNode hits)
+         {
+            hits.put("total", "redacted: " + removed + " hit(s) you cannot read were removed");
+            hits.remove("max_score");
+         }
+         return (root.toString());
+      }
+      catch(Exception e)
+      {
+         return ("{\"redacted\":\"response could not be parsed for redaction\"}");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Explain one hit ("table:recordId") when the table is indexed and the user
+    ** can read the record.
+    *******************************************************************************/
+   private ExplainResult explainHit(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String documentId) throws QException
+   {
+      int colon = documentId.indexOf(':');
+      if(colon <= 0 || colon == documentId.length() - 1)
+      {
+         throw (new QException("Give the hit as table:recordId (its document id), for example blogPost:42"));
+      }
+      String tableName = documentId.substring(0, colon);
+      String recordId  = documentId.substring(colon + 1);
+
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         throw (new QException("Table [" + tableName + "] is not indexed by Quick Search"));
+      }
+      if(DiagnosticRecordReads.existingIds(tableName, List.of(recordId)).isEmpty())
+      {
+         throw (new QException("Record " + recordId + " of [" + tableName + "] does not exist or you cannot read it"));
+      }
+      return (ops.explain(term, tableName, recordId, runtime.getDiscoveredTables()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Render a report as HTML.
+    *******************************************************************************/
+   public static String render(Report report)
+   {
+      DiagnosticsHtml html = new DiagnosticsHtml().open();
+      html.heading("Search playground: \"" + report.term() + "\"");
+      html.paragraph("Run as you, at " + Instant.now() + ". The global search shows up to " + report.limitPerTable() + " results per table; the Quick Search provider asks OpenSearch for the top "
+         + report.idsPerTable() + " ids per table and core keeps the first " + report.limitPerTable() + " you can read.");
+
+      for(String notice : report.notices())
+      {
+         html.status(Level.WARN, notice);
+      }
+
+      renderProviderState(html, report.providerState());
+
+      html.subheading("Path per table");
+      List<String> pathItems = new ArrayList<>();
+      for(SearchPathDiagnosis.TablePath path : report.paths())
+      {
+         Level level = SearchPathDiagnosis.PATH_OPENSEARCH.equals(path.path()) ? Level.OK : (SearchPathDiagnosis.PATH_CORE.equals(path.path()) ? Level.INFO : Level.WARN);
+         pathItems.add(DiagnosticsHtml.badge(level) + " <strong>" + DiagnosticsHtml.escape(path.tableLabel()) + "</strong> (" + DiagnosticsHtml.escape(path.tableName()) + "): "
+            + DiagnosticsHtml.escape(path.path()) + ". " + DiagnosticsHtml.escape(path.reason()));
+      }
+      if(pathItems.isEmpty())
+      {
+         html.paragraph("No table is searchable by the global search.");
+      }
+      html.rawList(pathItems);
+
+      html.subheading("Term tokens (search analyzer)");
+      if(report.termTokensError() != null)
+      {
+         html.status(Level.FAIL, "Could not analyze the term: " + report.termTokensError());
+      }
+      else if(report.termTokens() != null)
+      {
+         html.paragraph(tokenSummary(report.termTokens()));
+         for(AnalyzedToken token : report.termTokens())
+         {
+            if(token.token().length() < 2)
+            {
+               html.status(Level.WARN, "\"" + token.token() + "\" is one character: the index holds edge n-grams of 2 to 20 characters, so it can never match, and every term must match.");
+            }
+            else if(token.token().length() > 20)
+            {
+               html.status(Level.WARN, "\"" + token.token() + "\" is longer than 20 characters: the index holds edge n-grams of at most 20 characters, so it can never match.");
+            }
+         }
+      }
+
+      html.subheading("Quick Search provider request");
+      if(report.providerMillis() == null)
+      {
+         html.paragraph("Not run: the provider claims none of these tables (see the paths above), or the term is too short.");
+      }
+      else if(report.providerError() != null)
+      {
+         html.status(Level.FAIL, "Failed after " + report.providerMillis() + " ms: " + report.providerError() + ". A user search hitting this falls back to core's database search for these tables.");
+      }
+      else
+      {
+         html.status(Level.OK, "Answered in " + report.providerMillis() + " ms (wall clock, limit " + report.providerState().timeoutMillis() + " ms).");
+         List<String> items = new ArrayList<>();
+         for(Map.Entry<String, List<String>> entry : report.providerIds().entrySet())
+         {
+            Integer hidden = report.providerHiddenCounts().getOrDefault(entry.getKey(), 0);
+            items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + entry.getValue().size() + " id(s) you can read"
+               + (hidden > 0 ? " (" + hidden + " more you cannot read, not shown)" : "")
+               + (entry.getValue().isEmpty() ? "" : ": " + DiagnosticsHtml.escape(String.join(", ", entry.getValue().subList(0, Math.min(20, entry.getValue().size()))))
+               + (entry.getValue().size() > 20 ? ", ..." : "")));
+         }
+         html.rawList(items);
+      }
+
+      html.subheading("Global search results (what you see)");
+      if(report.coreMillis() == null)
+      {
+         html.paragraph("Not run: no table is searchable, or the term is empty.");
+      }
+      else if(report.coreError() != null)
+      {
+         html.status(Level.FAIL, "The global search failed after " + report.coreMillis() + " ms: " + report.coreError());
+      }
+      else
+      {
+         int total = report.coreResults().values().stream().mapToInt(List::size).sum();
+         html.status(total == 0 ? Level.INFO : Level.OK, total + " result(s) in " + report.coreMillis() + " ms.");
+         List<String> items = new ArrayList<>();
+         for(Map.Entry<String, List<RecordSearchResult>> entry : report.coreResults().entrySet())
+         {
+            List<String> labels = new ArrayList<>();
+            for(RecordSearchResult result : entry.getValue())
+            {
+               labels.add(DiagnosticsHtml.escape(result.getRecordLabel() == null ? String.valueOf(result.getRecordId()) : result.getRecordLabel()) + " (#" + DiagnosticsHtml.escape(result.getRecordId()) + ")");
+            }
+            items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + String.join("; ", labels));
+         }
+         html.rawList(items);
+      }
+
+      html.subheading("OpenSearch detail per indexed table");
+      if(report.probes().isEmpty())
+      {
+         html.paragraph("No indexed table you can read was searched, or the term is too short.");
+      }
+      for(TableProbe tableProbe : report.probes())
+      {
+         renderProbe(html, tableProbe);
+      }
+
+      if(report.explainDocumentId() != null)
+      {
+         html.subheading("Explain " + report.explainDocumentId());
+         if(report.explainError() != null)
+         {
+            html.status(Level.FAIL, report.explainError());
+         }
+         else if(report.explain() != null && !Boolean.TRUE.equals(report.explain().documentFound()))
+         {
+            html.status(Level.FAIL, "There is no document " + report.explainDocumentId() + " in the index.");
+         }
+         else if(report.explain() != null)
+         {
+            html.status(Boolean.TRUE.equals(report.explain().matched()) ? Level.OK : Level.WARN,
+               Boolean.TRUE.equals(report.explain().matched()) ? "Matches with score " + report.explain().score() + "." : "Does not match this term.");
+            html.textDetails("Score explanation", report.explain().explanationText());
+            html.jsonDetails("Explain response (JSON)", report.explain().json());
+         }
+      }
+
+      return (html.close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderProviderState(DiagnosticsHtml html, SearchPathDiagnosis.ProviderState state)
+   {
+      html.subheading("Record search provider (this server)");
+      List<String[]> facts = new ArrayList<>();
+      facts.add(new String[] { "Registered provider", state.registeredProvider() == null ? null : state.registeredProvider() + (Boolean.TRUE.equals(state.quickSearchProvider()) ? " (Quick Search)" : " (not Quick Search)") });
+      facts.add(new String[] { "serveCoreRecordSearch", String.valueOf(state.serveCoreRecordSearch()) });
+      facts.add(new String[] { "Timeout", state.timeoutMillis() + " ms" });
+      facts.add(new String[] { "Backing off", state.backoffRemainingMillis() > 0 ? "yes, " + (state.backoffRemainingMillis() / 1000) + " s left" : "no" });
+      facts.add(new String[] { "Failures since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) });
+      facts.add(new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) });
+      facts.add(new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() });
+      html.facts(facts);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void renderProbe(DiagnosticsHtml html, TableProbe tableProbe)
+   {
+      html.raw("<h4>" + DiagnosticsHtml.escape(tableProbe.tableName()) + "</h4>");
+      if(tableProbe.error() != null)
+      {
+         html.status(Level.FAIL, "OpenSearch search failed: " + tableProbe.error());
+         return;
+      }
+
+      SearchProbe probe = tableProbe.probe();
+      html.paragraph((probe.totalHits() == null ? "Matching documents: not shown (some are records you cannot read)"
+         : probe.totalHits() + (Boolean.TRUE.equals(probe.totalIsLowerBound()) ? "+" : "") + " matching document(s)")
+         + "; OpenSearch took " + probe.tookMillis() + " ms (" + probe.wallMillis() + " ms wall clock). Showing the " + probe.hits().size() + " top hit(s) you can read.");
+      if(tableProbe.hiddenCount() != null && tableProbe.hiddenCount() > 0)
+      {
+         html.status(Level.INFO, tableProbe.hiddenCount() + " of the top hits are records you cannot read (deleted, or hidden by a record security lock); they are not shown.");
+      }
+
+      List<String> items    = new ArrayList<>();
+      int          position = 0;
+      for(ProbeHit hit : probe.hits())
+      {
+         position++;
+         StringBuilder item = new StringBuilder();
+         item.append("#").append(position).append(" <strong>")
+            .append(DiagnosticsHtml.escape(hit.recordLabel() == null ? "(no label)" : hit.recordLabel()))
+            .append("</strong> &middot; document <code>").append(DiagnosticsHtml.escape(OpenSearchDocument.buildDocumentId(tableProbe.tableName(), hit.recordId()))).append("</code>")
+            .append(" &middot; score ").append(hit.score() == null ? "n/a" : String.format("%.3f", hit.score()))
+            .append(" &middot; ").append(tableProbe.shownIds().contains(hit.recordId()) ? "shown in global search" : "not shown (beyond the per-table limit)");
+         if(!hit.matchedClauses().isEmpty())
+         {
+            List<String> fields = hit.matchedClauses().stream().map(c -> c.startsWith(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX) ? c.substring(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX.length()) : c).toList();
+            item.append("<br>Matched: ").append(DiagnosticsHtml.escape(String.join(", ", fields)));
+         }
+         for(Map.Entry<String, List<String>> highlight : hit.highlights().entrySet())
+         {
+            item.append("<br><em>").append(DiagnosticsHtml.escape(highlight.getKey())).append(":</em> ")
+               .append(DiagnosticsHtml.highlight(String.join(" ... ", highlight.getValue())));
+         }
+         items.add(item.toString());
+      }
+      if(items.isEmpty())
+      {
+         html.paragraph("No document of this table that you can read matches every term.");
+      }
+      html.rawList(items);
+      html.jsonDetails("Query DSL", probe.requestJson());
+      html.jsonDetails("Raw response (hits you cannot read removed)", probe.responseJson());
+   }
+
+
+
+   /*******************************************************************************
+    ** "a, b, c" of token strings.
+    *******************************************************************************/
+   static String tokenSummary(List<AnalyzedToken> tokens)
+   {
+      if(tokens.isEmpty())
+      {
+         return ("No tokens: the term has no searchable characters.");
+      }
+      List<String> parts = new ArrayList<>();
+      for(AnalyzedToken token : tokens)
+      {
+         parts.add("\"" + token.token() + "\"");
+      }
+      return (tokens.size() + " token(s), all of which must match: " + String.join(", ", parts));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java

@@ -0,0 +1,496 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.listeners;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
+import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Real-time indexing hook. Registered once per QInstance; applies to the
+ ** tables this QBit indexes.
+ **
+ ** Events are built while the write's transaction is open, but published only
+ ** after that transaction commits, via
+ ** QBackendTransaction.addAfterCommitCallback. A write without a transaction
+ ** is already committed, so it publishes immediately. A rolled-back write never
+ ** reaches the index.
+ **
+ ** Inserted and updated records are re-read inside that transaction, with
+ ** display values and possible-value labels, so a real-time document matches
+ ** the one basepull, reconcile and full reindex build. The event's own records
+ ** are not enough: for an update they are the sparse input, and for an insert
+ ** they carry only raw values.
+ **
+ ** This listener never fails the host's write. When the re-read of inserted or
+ ** updated records or publishing fails (including a LinkageError from a missing
+ ** optional library while the client is built), each affected (table, record,
+ ** action) is written to quickSearchFailedEvent for the scheduled basepull to
+ ** replay. A failed re-read is recorded after the commit too, so a rolled-back
+ ** write leaves no row.
+ **
+ ** While a full reindex rebuilds into a new physical index, the alias still
+ ** points at the old one, so a published delete never reaches the new index.
+ ** Such deletes are also written to quickSearchFailedEvent with status
+ ** AWAITING_REINDEX; the reindex applies them after the alias swap.
+ *******************************************************************************/
+public class QuickSearchRecordChangeListener implements RecordChangeListenerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordChangeListener.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean appliesTo(String tableName, RecordChangeType type)
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      return (runtime != null
+         && Boolean.TRUE.equals(runtime.getConfig().getEnableRealTimeIndexing())
+         && runtime.getTableConfig(tableName) != null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void onRecordsChanged(RecordChangeEvent event) throws QException
+   {
+      QuickSearchRuntime runtime = QuickSearchRuntime.get();
+      if(runtime == null || event == null || CollectionUtils.nullSafeIsEmpty(event.getRecords()))
+      {
+         return;
+      }
+
+      String                     tableName   = event.getTableName();
+      QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
+      if(tableConfig == null || !runtime.isTableEnabled(tableName))
+      {
+         return;
+      }
+
+      String primaryKeyField = tableConfig.getPrimaryKeyField() != null ? tableConfig.getPrimaryKeyField() : "id";
+
+      List<IndexEvent> indexEvents  = new ArrayList<>();
+      List<IndexEvent> deleteEvents = new ArrayList<>();
+
+      switch(event.getType())
+      {
+         case INSERT, UPDATE ->
+         {
+            try
+            {
+               indexEvents.addAll(toIndexEvents(tableName, primaryKeyField, fetchCurrentRecords(event, primaryKeyField)));
+            }
+            catch(Exception | LinkageError e)
+            {
+               ///////////////////////////////////////////////////////////////////////
+               // log now: with a transaction, nothing else is logged until commit, //
+               // and a rollback (a transaction the failed re-read aborted, too)    //
+               // would leave no trace. Then record the written keys after commit,  //
+               // so basepull re-reads and indexes them.                            //
+               ///////////////////////////////////////////////////////////////////////
+               LOG.warn("Could not re-read " + (event.getType() == RecordChangeType.INSERT ? "inserted" : "updated")
+                     + " records for indexing; recording them for replay once the write commits", e,
+                  logPair("tableName", tableName), logPair("recordCount", event.getRecords().size()));
+               List<IndexEvent> failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords());
+               runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e));
+               return;
+            }
+         }
+         case DELETE -> deleteEvents.addAll(toDeleteEvents(tableName, primaryKeyField, event.getRecords()));
+      }
+
+      if(indexEvents.isEmpty() && deleteEvents.isEmpty())
+      {
+         return;
+      }
+
+      runAfterCommit(event, () -> publish(runtime, indexEvents, deleteEvents));
+   }
+
+
+
+   /*******************************************************************************
+    ** Run work once the event's transaction commits, or now when the write had
+    ** no transaction (it is already committed).
+    *******************************************************************************/
+   private static void runAfterCommit(RecordChangeEvent event, Runnable work)
+   {
+      if(event.getTransaction() != null)
+      {
+         event.getTransaction().addAfterCommitCallback(work);
+      }
+      else
+      {
+         work.run();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Publish, recording any failure durably, and capture the deletes if a full
+    ** reindex is rebuilding the table. Never throws.
+    **
+    ** The status is read before the deletes are published, never after: a
+    ** reindex can swap the alias and clear REBUILDING while the publish is in
+    ** flight, and a read after that would miss a delete that reached only the
+    ** old index. A read before the publish that sees no REBUILDING means either
+    ** the reindex has not started reading the table (the record, deleted and
+    ** committed, cannot be read into the new index) or the alias already points
+    ** at the new index (the publish reaches it).
+    *******************************************************************************/
+   static void publish(QuickSearchRuntime runtime, List<IndexEvent> indexEvents, List<IndexEvent> deleteEvents)
+   {
+      boolean rebuilding = isRebuildingBeforePublish(runtime, deleteEvents);
+      publishOrRecordFailures(runtime, indexEvents, deleteEvents);
+      if(rebuilding)
+      {
+         captureDeletesDuringRebuild(runtime, deleteEvents);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A LinkageError is caught along with Exception: building the lazy client,
+    ** or a host publisher, can hit an optional library that is not on the
+    ** classpath (the AWS SDK for AWS_SIGV4), which a DEGRADED start leaves for
+    ** the first write to find.
+    *******************************************************************************/
+   private static void publishOrRecordFailures(QuickSearchRuntime runtime, List<IndexEvent> indexEvents, List<IndexEvent> deleteEvents)
+   {
+      IndexEventPublisher publisher;
+      try
+      {
+         publisher = runtime.getPublisher();
+      }
+      catch(Exception | LinkageError e)
+      {
+         recordFailures(runtime, indexEvents, e);
+         recordFailures(runtime, deleteEvents, e);
+         return;
+      }
+
+      if(!indexEvents.isEmpty())
+      {
+         try
+         {
+            publisher.publishIndexEvents(indexEvents);
+         }
+         catch(Exception | LinkageError e)
+         {
+            recordFailures(runtime, indexEvents, e);
+         }
+      }
+
+      if(!deleteEvents.isEmpty())
+      {
+         try
+         {
+            publisher.publishDeleteEvents(deleteEvents);
+         }
+         catch(Exception | LinkageError e)
+         {
+            recordFailures(runtime, deleteEvents, e);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Write one quickSearchFailedEvent row per event. Never throws.
+    *******************************************************************************/
+   static void recordFailures(QuickSearchRuntime runtime, List<IndexEvent> events, Throwable cause)
+   {
+      if(events.isEmpty())
+      {
+         return;
+      }
+
+      String message = cause.getMessage() == null ? cause.getClass().getSimpleName() : cause.getMessage();
+      LOG.warn("Real-time indexing failed; recording failed events for replay", cause,
+         logPair("tableName", events.get(0).getTableName()), logPair("eventCount", events.size()));
+
+      try
+      {
+         if(!hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)))
+         {
+            LOG.error("quickSearchFailedEvent table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
+            return;
+         }
+
+         insertFailedEventRows(runtime, events, message, QuickSearchFailedEvent.STATUS_PENDING);
+      }
+      catch(Exception e)
+      {
+         LOG.error("Could not record failed index events", e, logPair("eventCount", events.size()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a full reindex is rebuilding the deletes' table. The status is
+    ** read from the database (not cached) so a reindex on another node is seen.
+    ** Runs after the delete committed. Never throws.
+    *******************************************************************************/
+   static boolean isRebuildingBeforePublish(QuickSearchRuntime runtime, List<IndexEvent> deleteEvents)
+   {
+      if(deleteEvents.isEmpty())
+      {
+         return (false);
+      }
+
+      String tableName = deleteEvents.get(0).getTableName();
+      try
+      {
+         return (hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
+      }
+      catch(Exception e)
+      {
+         LOG.error("Could not read whether a full reindex is running; deletes made during one are not captured", e, logPair("tableName", tableName));
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Record deletes made while a full reindex is rebuilding the table, so the
+    ** reindex can apply them to the new physical index after the alias swap.
+    ** basepull does not replay AWAITING_REINDEX rows, so it cannot consume one
+    ** against the old index before the swap. Never throws.
+    *******************************************************************************/
+   static void captureDeletesDuringRebuild(QuickSearchRuntime runtime, List<IndexEvent> deleteEvents)
+   {
+      String tableName = deleteEvents.get(0).getTableName();
+      try
+      {
+         List<QRecord> inserted = insertFailedEventRows(runtime, deleteEvents, "deleted during a full reindex", QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
+         handOverIfRebuildFinished(runtime, tableName, inserted);
+      }
+      catch(Exception e)
+      {
+         LOG.error("Could not capture deletes made during a full reindex; the next full reindex or reconcile removes them", e,
+            logPair("tableName", tableName), logPair("eventCount", deleteEvents.size()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The reindex may have finished, and drained its captured deletes, between
+    ** the status read and the insert. Read the status again: if the table is no
+    ** longer REBUILDING, make the rows PENDING so basepull replays them against
+    ** the current index (deletes are idempotent). If it still is, the reindex
+    ** has not drained yet and will see these rows.
+    *******************************************************************************/
+   static void handOverIfRebuildFinished(QuickSearchRuntime runtime, String tableName, List<QRecord> inserted) throws QException
+   {
+      if(inserted.isEmpty() || isRebuilding(runtime, tableName))
+      {
+         return;
+      }
+
+      List<QRecord> updates = new ArrayList<>();
+      for(QRecord row : inserted)
+      {
+         updates.add(new QRecord().withValue("id", row.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING));
+      }
+
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
+      updateInput.setRecords(updates);
+      new UpdateAction().execute(updateInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the table's quickSearchIndex row is REBUILDING.
+    *******************************************************************************/
+   private static boolean isRebuilding(QuickSearchRuntime runtime, String tableName) throws QException
+   {
+      String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
+      if(!hasTable(indexTable))
+      {
+         return (false);
+      }
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(indexTable);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)).withLimit(1));
+      List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeHasContents(rows) && AbstractIndexingStep.STATUS_REBUILDING.equals(rows.get(0).getValueString("status")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one quickSearchFailedEvent row per event; returns the inserted rows.
+    *******************************************************************************/
+   static List<QRecord> insertFailedEventRows(QuickSearchRuntime runtime, List<IndexEvent> events, String message, String status) throws QException
+   {
+      List<QRecord> rows = new ArrayList<>();
+      for(IndexEvent event : events)
+      {
+         rows.add(new QRecord()
+            .withValue("tableName", event.getTableName())
+            .withValue("recordId", event.getRecordId())
+            .withValue("action", event.getAction() == null ? null : event.getAction().name())
+            .withValue("errorMessage", truncate(message, 4000))
+            .withValue("attempts", 0)
+            .withValue("status", status));
+      }
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
+      insertInput.setRecords(rows);
+      return (CollectionUtils.nonNullList(new InsertAction().execute(insertInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean hasTable(String tableName)
+   {
+      return (QContext.getQInstance() != null && QContext.getQInstance().getTable(tableName) != null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<IndexEvent> toIndexEvents(String tableName, String primaryKeyField, List<QRecord> records)
+   {
+      List<IndexEvent> events = new ArrayList<>();
+      for(QRecord record : CollectionUtils.nonNullList(records))
+      {
+         String recordId = record.getValueString(primaryKeyField);
+         if(recordId == null || CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            continue;
+         }
+         events.add(new IndexEvent().withAction(IndexEventAction.INDEX).withTableName(tableName).withRecordId(recordId).withRecord(record));
+      }
+      return (events);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<IndexEvent> toDeleteEvents(String tableName, String primaryKeyField, List<QRecord> records)
+   {
+      List<IndexEvent> events = new ArrayList<>();
+      for(QRecord record : CollectionUtils.nonNullList(records))
+      {
+         String recordId = record.getValueString(primaryKeyField);
+         if(recordId == null || CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            continue;
+         }
+         events.add(new IndexEvent().withAction(IndexEventAction.DELETE).withTableName(tableName).withRecordId(recordId));
+      }
+      return (events);
+   }
+
+
+
+   /*******************************************************************************
+    ** The event's records are "as the backend returned them": for an update on
+    ** RDBMS that is the sparse input, and for an insert the raw values with no
+    ** display values. Re-read the full rows inside the same transaction, with
+    ** display values so possible-value labels are indexed (which can run host
+    ** possible-value providers).
+    *******************************************************************************/
+   private static List<QRecord> fetchCurrentRecords(RecordChangeEvent event, String primaryKeyField) throws QException
+   {
+      List<Serializable> primaryKeys = new ArrayList<>();
+      for(QRecord record : event.getRecords())
+      {
+         Serializable primaryKey = record.getValue(primaryKeyField);
+         if(primaryKey != null && CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+         {
+            primaryKeys.add(primaryKey);
+         }
+      }
+
+      if(primaryKeys.isEmpty())
+      {
+         return (List.of());
+      }
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(event.getTableName());
+      queryInput.setTransaction(event.getTransaction());
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, primaryKeys)));
+
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String truncate(String s, int max)
+   {
+      return (s == null || s.length() <= max ? s : s.substring(0, max));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchAdminAppMetaDataProducer.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchAdmin app from the tables and processes the other
+ ** component producers added to the production output. Runs last.
+ *******************************************************************************/
+public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataProducer<QAppMetaData, QuickSearchQBitConfig>
+{
+   public static final String APP_NAME = "quickSearchAdmin";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QAppMetaData produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig config = getQBitConfig();
+
+      QAppMetaData app = new QAppMetaData()
+         .withName(config.applyPrefix(APP_NAME))
+         .withLabel("Quick Search Admin")
+         .withIcon(new QIcon("search"))
+         .withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
+
+      MetaDataProducerMultiOutput produced = QBitProductionContext.peekMetaDataProducerMultiOutput();
+      if(produced == null)
+      {
+         return (app);
+      }
+
+      QWidgetMetaData diagnosticsWidget = produced.get(QWidgetMetaData.class, config.applyPrefix(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME));
+      if(diagnosticsWidget != null)
+      {
+         app.withWidgets(new ArrayList<>(List.of(diagnosticsWidget.getName())));
+      }
+
+      for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME, QuickSearchFailedEvent.TABLE_NAME,
+         QuickSearchValidationRun.TABLE_NAME, QuickSearchDiagnosticEvent.TABLE_NAME })
+      {
+         QTableMetaData table = produced.get(QTableMetaData.class, config.applyPrefix(tableName));
+         if(table != null)
+         {
+            app.withChild(table);
+         }
+      }
+
+      for(String processName : new String[] { QuickSearchProcessMetaDataHelper.BASEPULL_PROCESS_NAME, QuickSearchProcessMetaDataHelper.FULL_REINDEX_PROCESS_NAME, QuickSearchProcessMetaDataHelper.RECONCILE_PROCESS_NAME })
+      {
+         QProcessMetaData process = produced.get(QProcessMetaData.class, config.applyPrefix(processName));
+         if(process != null)
+         {
+            app.withChild(process);
+         }
+      }
+
+      return (app);
+   }
+
+
+
+   /*******************************************************************************
+    ** last, so the tables and processes exist in the output
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (900);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchBasepullProcessMetaDataProducer.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the basepull process when enableBasepullProcess is true.
+ *******************************************************************************/
+public class QuickSearchBasepullProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchProcessMetaDataHelper.buildBasepullProcess(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableBasepullProcess()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDeveloperPanelsMetaDataProducer.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelsMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDeveloperPanelProvider;
+
+
+/*******************************************************************************
+ ** Registers the Quick Search record developer panel
+ ** ({@link QuickSearchDeveloperPanelProvider}) in the instance's developer
+ ** panels. The provider decides per table and user whether the panel shows
+ ** (indexed tables, users who may read Quick Search Index). Registration is
+ ** idempotent, so producing the QBit again adds nothing twice.
+ *******************************************************************************/
+public class QuickSearchDeveloperPanelsMetaDataProducer extends QBitComponentMetaDataProducer<DeveloperPanelsMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DeveloperPanelsMetaData produce(QInstance qInstance) throws QException
+   {
+      return (DeveloperPanelsMetaData.ofOrWithNew(qInstance).withProvider(new QCodeReference(QuickSearchDeveloperPanelProvider.class)));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticEventTableMetaDataProducer.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchDiagnosticEvent table when enableDiagnosticTools is true.
+ *******************************************************************************/
+public class QuickSearchDiagnosticEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildDiagnosticEventTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticsProcessMetaDataHelper.java

@@ -0,0 +1,279 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.AbstractDiagnosticToolStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.AnalyzeTextStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.InspectRecordStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.ReindexRecordsStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.SearchPlaygroundStep;
+import com.kingsrook.qbits.quicksearch.processes.diagnostics.ValidateIndexStep;
+
+
+/*******************************************************************************
+ ** Builds the diagnostic tool processes. Every one is a table or record
+ ** action (it has a tableName and is never added to an app's nav):
+ ** - on Quick Search Index: Search Playground, Validate Search Index (also
+ **   schedulable), Test Search Analyzer;
+ ** - on every indexed table and on Quick Search Failed Event: Inspect Search
+ **   Document and Reindex in Search.
+ ** Read-only tools use debugPermissionRules; Reindex in Search uses
+ ** adminPermissionRules.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsProcessMetaDataHelper
+{
+   public static final String PLAYGROUND_PROCESS_NAME     = "quickSearchPlayground";
+   public static final String VALIDATE_PROCESS_NAME       = "quickSearchValidateIndex";
+   public static final String ANALYZE_PROCESS_NAME        = "quickSearchAnalyzeText";
+   public static final String INSPECT_PROCESS_PREFIX      = "quickSearchInspect";
+   public static final String REINDEX_RECORDS_PROCESS_PREFIX = "quickSearchReindexRecords";
+
+
+
+   /*******************************************************************************
+    ** Name of the Inspect Search Document process of a table.
+    *******************************************************************************/
+   public static String inspectProcessName(QuickSearchQBitConfig config, String tableName)
+   {
+      return (config.applyPrefix(INSPECT_PROCESS_PREFIX) + "_" + tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Name of the Reindex in Search process of a table.
+    *******************************************************************************/
+   public static String reindexRecordsProcessName(QuickSearchQBitConfig config, String tableName)
+   {
+      return (config.applyPrefix(REINDEX_RECORDS_PROCESS_PREFIX) + "_" + tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Search Playground, a table action on Quick Search Index.
+    *******************************************************************************/
+   public static QProcessMetaData buildPlaygroundProcess(QuickSearchQBitConfig config)
+   {
+      return (new QProcessMetaData()
+         .withName(config.applyPrefix(PLAYGROUND_PROCESS_NAME))
+         .withLabel("Search Playground")
+         .withIcon(new QIcon("travel_explore"))
+         .withTableName(config.getQuickSearchIndexTableName())
+         .withMinInputRecords(0)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(SearchPlaygroundStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
+         .withStep(form("Search", List.of(
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_TERM, QFieldType.STRING).withLabel("Search term").withIsRequired(true).withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH),
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_TABLES, QFieldType.STRING).withLabel("Tables (optional; names separated by commas)"),
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_LIMIT_PER_TABLE, QFieldType.INTEGER).withLabel("Results per table").withDefaultValue(RecordSearchAction.DEFAULT_LIMIT_PER_TABLE),
+            new QFieldMetaData(SearchPlaygroundStep.FIELD_EXPLAIN_HIT, QFieldType.STRING).withLabel("Explain hit (optional document id, e.g. blogPost:42)"))))
+         .withStep(backend(SearchPlaygroundStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Test Search Analyzer, a table action on Quick Search Index.
+    *******************************************************************************/
+   public static QProcessMetaData buildAnalyzeProcess(QuickSearchQBitConfig config)
+   {
+      return (new QProcessMetaData()
+         .withName(config.applyPrefix(ANALYZE_PROCESS_NAME))
+         .withLabel("Test Search Analyzer")
+         .withIcon(new QIcon("text_fields"))
+         .withTableName(config.getQuickSearchIndexTableName())
+         .withMinInputRecords(0)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(AnalyzeTextStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
+         .withStep(form("Analyze", List.of(
+            new QFieldMetaData(AnalyzeTextStep.FIELD_TEXT, QFieldType.TEXT).withLabel("Text (as a field value)").withIsRequired(true),
+            new QFieldMetaData(AnalyzeTextStep.FIELD_QUERY, QFieldType.STRING).withLabel("Search term (optional)").withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH))))
+         .withStep(backend(AnalyzeTextStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate Search Index, a table action on Quick Search Index; scheduled
+    ** by cron when validationCronExpression and a schedulerName are set.
+    *******************************************************************************/
+   public static QProcessMetaData buildValidateProcess(QuickSearchQBitConfig config)
+   {
+      QProcessMetaData process = new QProcessMetaData()
+         .withName(config.applyPrefix(VALIDATE_PROCESS_NAME))
+         .withLabel("Validate Search Index")
+         .withIcon(new QIcon("fact_check"))
+         .withTableName(config.getQuickSearchIndexTableName())
+         .withMinInputRecords(0)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(ValidateIndexStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
+         .withStep(form("Validate", List.of(
+            new QFieldMetaData(ValidateIndexStep.FIELD_SAMPLE_SIZE, QFieldType.INTEGER).withLabel("Records and documents to sample per table")
+               .withDefaultValue(config.getValidationSampleSize()))))
+         .withStep(backend(ValidateIndexStep.class, AbstractDiagnosticToolStep.STEP_RUN, null))
+         .withStep(result(AbstractDiagnosticToolStep.STEP_RESULT, "Validation report"));
+
+      if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getValidationCronExpression()))
+      {
+         process.withSchedule(new QScheduleMetaData()
+            .withSchedulerName(config.getSchedulerName())
+            .withCronExpression(config.getValidationCronExpression())
+            .withCronTimeZoneId(config.getValidationCronTimeZoneId())
+            .withDescription("Quick Search index validation"));
+      }
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Inspect Search Document, a record action on a table (an indexed table,
+    ** or Quick Search Failed Event for the event's record).
+    *******************************************************************************/
+   public static QProcessMetaData buildInspectProcess(QuickSearchQBitConfig config, String tableName)
+   {
+      return (new QProcessMetaData()
+         .withName(inspectProcessName(config, tableName))
+         .withLabel("Inspect Search Document")
+         .withIcon(new QIcon("manage_search"))
+         .withTableName(tableName)
+         .withMinInputRecords(1)
+         .withMaxInputRecords(InspectRecordStep.MAX_RECORDS)
+         .withPermissionRules(clone(config.getDebugPermissionRules()))
+         .withStep(backend(InspectRecordStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, tableName))
+         .withStep(form("Inspect", List.of(
+            new QFieldMetaData(InspectRecordStep.FIELD_TERM, QFieldType.STRING).withLabel("Why doesn't it match... (optional search term)")
+               .withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH))))
+         .withStep(backend(InspectRecordStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Reindex in Search, a record action on a table (an indexed table, or
+    ** Quick Search Failed Event for the event's record). Writes to the index,
+    ** so it uses the admin permission rules.
+    *******************************************************************************/
+   public static QProcessMetaData buildReindexRecordsProcess(QuickSearchQBitConfig config, String tableName)
+   {
+      return (new QProcessMetaData()
+         .withName(reindexRecordsProcessName(config, tableName))
+         .withLabel("Reindex in Search")
+         .withIcon(new QIcon("sync"))
+         .withTableName(tableName)
+         .withMinInputRecords(1)
+         .withMaxInputRecords(ReindexRecordsStep.MAX_RECORDS)
+         .withPermissionRules(clone(config.getAdminPermissionRules()))
+         .withStep(backend(ReindexRecordsStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, tableName))
+         .withStep(result(AbstractDiagnosticToolStep.STEP_CONFIRM, "Confirm"))
+         .withStep(backend(ReindexRecordsStep.class, AbstractDiagnosticToolStep.STEP_RUN, null))
+         .withStep(result(AbstractDiagnosticToolStep.STEP_RESULT, "Reindexed")));
+   }
+
+
+
+   /*******************************************************************************
+    ** The fixed processes (on Quick Search Index and Quick Search Failed
+    ** Event).
+    *******************************************************************************/
+   public static List<QProcessMetaData> buildFixedProcesses(QuickSearchQBitConfig config)
+   {
+      List<QProcessMetaData> processes = new ArrayList<>();
+      processes.add(buildPlaygroundProcess(config));
+      processes.add(buildValidateProcess(config));
+      processes.add(buildAnalyzeProcess(config));
+      processes.add(buildInspectProcess(config, config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)));
+      processes.add(buildReindexRecordsProcess(config, config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)));
+      return (processes);
+   }
+
+
+
+   /*******************************************************************************
+    ** A backend step of a tool; with recordsTableName, the step receives the
+    ** selected records of that table.
+    *******************************************************************************/
+   private static QBackendStepMetaData backend(Class<?> stepClass, String name, String recordsTableName)
+   {
+      QBackendStepMetaData step = new QBackendStepMetaData().withName(name).withCode(new QCodeReference(stepClass));
+      if(recordsTableName != null)
+      {
+         step.withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName(recordsTableName)));
+      }
+      return (step);
+   }
+
+
+
+   /*******************************************************************************
+    ** The FORM step: fields above, the tool's HTML below.
+    *******************************************************************************/
+   private static QFrontendStepMetaData form(String label, List<QFieldMetaData> fields)
+   {
+      QFrontendStepMetaData step = new QFrontendStepMetaData()
+         .withName(AbstractDiagnosticToolStep.STEP_FORM)
+         .withLabel(label)
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML));
+      fields.forEach(step::withFormField);
+      return (step);
+   }
+
+
+
+   /*******************************************************************************
+    ** An HTML-only frontend step.
+    *******************************************************************************/
+   private static QFrontendStepMetaData result(String name, String label)
+   {
+      return (new QFrontendStepMetaData()
+         .withName(name)
+         .withLabel(label)
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QPermissionRules clone(QPermissionRules rules)
+   {
+      return (rules == null ? null : rules.clone());
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticsProcessesMetaDataProducer.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the fixed diagnostic tool processes (Search Playground, Validate
+ ** Search Index, Test Search Analyzer, and Inspect / Reindex on Quick Search
+ ** Failed Event) when enableDiagnosticTools is true. The per-table record
+ ** actions are added by QuickSearchQBitProducer once the indexed tables are
+ ** known.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsProcessesMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+      for(QProcessMetaData process : QuickSearchDiagnosticsProcessMetaDataHelper.buildFixedProcesses(getQBitConfig()))
+      {
+         output.add(process);
+      }
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchDiagnosticsWidgetMetaDataProducer.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDiagnosticsWidgetRenderer;
+
+
+/*******************************************************************************
+ ** Produces the Search Diagnostics widget (quickSearchDiagnostics) when
+ ** enableDiagnosticTools is true. The admin app shows it on its home; a host
+ ** with its own Quick Search home adds it there by name.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsWidgetMetaDataProducer extends QBitComponentMetaDataProducer<QWidgetMetaData, QuickSearchQBitConfig>
+{
+   public static final String WIDGET_NAME = "quickSearchDiagnostics";
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public QWidgetMetaData produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig config = getQBitConfig();
+      return (new QWidgetMetaData()
+         .withName(config.applyPrefix(WIDGET_NAME))
+         .withLabel("Search Diagnostics")
+         .withIcon("troubleshoot")
+         .withType(WidgetType.HTML.getType())
+         .withGridColumns(12)
+         .withIsCard(true)
+         .withCodeReference(new QCodeReference(QuickSearchDiagnosticsWidgetRenderer.class))
+         .withPermissionRules(config.getDebugPermissionRules() == null ? null : config.getDebugPermissionRules().clone()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (300);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchFailedEventTableMetaDataProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchFailedEvent table.
+ *******************************************************************************/
+public class QuickSearchFailedEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildFailedEventTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchFullReindexProcessMetaDataProducer.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the full reindex process when enableMaintenanceProcesses is true.
+ *******************************************************************************/
+public class QuickSearchFullReindexProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchProcessMetaDataHelper.buildFullReindexProcess(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableMaintenanceProcesses()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexPossibleValueSourceProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the possible value source over the quickSearchIndex table.
+ *******************************************************************************/
+public class QuickSearchIndexPossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QPossibleValueSource produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildIndexPossibleValueSource(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexRunTableMetaDataProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchIndexRun table.
+ *******************************************************************************/
+public class QuickSearchIndexRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildIndexRunTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchIndexTableMetaDataProducer.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchIndex table.
+ *******************************************************************************/
+public class QuickSearchIndexTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildIndexTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchProcessMetaDataHelper.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep;
+import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
+import com.kingsrook.qbits.quicksearch.processes.ReconcileIndexStep;
+
+
+/*******************************************************************************
+ ** Builds the three process definitions, with permission rules from config and
+ ** schedules when a scheduler is configured.
+ *******************************************************************************/
+public class QuickSearchProcessMetaDataHelper
+{
+   public static final String BASEPULL_PROCESS_NAME     = "quickSearchBasepullIndex";
+   public static final String FULL_REINDEX_PROCESS_NAME = "quickSearchFullReindex";
+   public static final String RECONCILE_PROCESS_NAME    = "quickSearchReconcileIndex";
+
+
+
+   /*******************************************************************************
+    ** Basepull catch-up; scheduled every basepullRepeatSeconds when a
+    ** schedulerName is set.
+    *******************************************************************************/
+   public static QProcessMetaData buildBasepullProcess(QuickSearchQBitConfig config)
+   {
+      QProcessMetaData process = buildProcess(config, BASEPULL_PROCESS_NAME, "Quick Search Basepull Index", "basepull", BasepullIndexStep.class, "update");
+      process.withStep(resultStep("Basepull complete", "tablesIndexed", "failedEventsReplayed"));
+
+      if(StringUtils.hasContent(config.getSchedulerName()))
+      {
+         process.withSchedule(new QScheduleMetaData()
+            .withSchedulerName(config.getSchedulerName())
+            .withRepeatSeconds(config.getBasepullRepeatSeconds() == null ? 300 : config.getBasepullRepeatSeconds())
+            .withDescription("Quick Search basepull catch-up"));
+      }
+
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Full reindex; on demand only.
+    *******************************************************************************/
+   public static QProcessMetaData buildFullReindexProcess(QuickSearchQBitConfig config)
+   {
+      QProcessMetaData process = buildProcess(config, FULL_REINDEX_PROCESS_NAME, "Quick Search Full Reindex", "fullReindex", FullReindexStep.class, "refresh");
+      process.withStep(0, inputStep(config, "Leave the table blank to rebuild every table into a fresh index and swap the alias; pick a table to rebuild just that table in place."));
+      process.withStep(resultStep("Full reindex complete", "recordsIndexed", "aliasSwapped"));
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Reconcile; scheduled by cron when reconcileCronExpression and a
+    ** schedulerName are set.
+    *******************************************************************************/
+   public static QProcessMetaData buildReconcileProcess(QuickSearchQBitConfig config)
+   {
+      QProcessMetaData process = buildProcess(config, RECONCILE_PROCESS_NAME, "Quick Search Reconcile Index", "reconcile", ReconcileIndexStep.class, "sync");
+      process.withStep(0, inputStep(config, "Leave the table blank to reconcile every table and remove documents of unconfigured tables; pick a table to reconcile just that one."));
+      process.withStep(resultStep("Reconcile complete", "recordsIndexed", "documentsRemoved", "orphanDocumentsRemoved", "tablesSkipped"));
+
+      if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getReconcileCronExpression()))
+      {
+         process.withSchedule(new QScheduleMetaData()
+            .withSchedulerName(config.getSchedulerName())
+            .withCronExpression(config.getReconcileCronExpression())
+            .withCronTimeZoneId(config.getReconcileCronTimeZoneId())
+            .withDescription("Quick Search reconcile"));
+      }
+
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** Optional table picker shown before the backend step.
+    *******************************************************************************/
+   private static QFrontendStepMetaData inputStep(QuickSearchQBitConfig config, String help)
+   {
+      return (new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Choose table")
+         .withFormField(new QFieldMetaData("tableName", QFieldType.STRING)
+            .withLabel("Table (optional)")
+            .withPossibleValueSourceName(config.applyPrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
+            .withIsRequired(false))
+         .withViewField(new QFieldMetaData("help", QFieldType.STRING).withLabel(" ").withDefaultValue(help)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Summary screen showing the step's output values.
+    *******************************************************************************/
+   private static QFrontendStepMetaData resultStep(String label, String... outputFields)
+   {
+      QFrontendStepMetaData step = new QFrontendStepMetaData().withName("result").withLabel(label);
+      for(String field : outputFields)
+      {
+         step.withViewField(new QFieldMetaData(field, QFieldType.STRING));
+      }
+      return (step);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QProcessMetaData buildProcess(QuickSearchQBitConfig config, String baseName, String label, String stepName, Class<?> stepClass, String icon)
+   {
+      return (new QProcessMetaData()
+         .withName(config.applyPrefix(baseName))
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone())
+         .withStep(new QBackendStepMetaData()
+            .withName(stepName)
+            .withCode(new QCodeReference(stepClass))));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchReconcileProcessMetaDataProducer.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the reconcile process when enableMaintenanceProcesses is true.
+ *******************************************************************************/
+public class QuickSearchReconcileProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchProcessMetaDataHelper.buildReconcileProcess(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableMaintenanceProcesses()));
+   }
+
+
+
+   /*******************************************************************************
+    ** after tables, before the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (200);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchSearchableTablePossibleValueSourceProducer.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.SearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
+
+
+/*******************************************************************************
+ ** Enum possible value source listing the searchable tables (from annotated
+ ** entity classes and config-driven tables), used by the process input screens
+ ** so an operator can pick one table for a full reindex or reconcile.
+ *******************************************************************************/
+public class QuickSearchSearchableTablePossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
+{
+   public static final String NAME = "quickSearchSearchableTable";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QPossibleValueSource produce(QInstance qInstance) throws QException
+   {
+      QuickSearchQBitConfig config = getQBitConfig();
+
+      Set<String> tableNames = new LinkedHashSet<>();
+      for(Class<?> entityClass : config.getSearchableEntityClasses() == null ? List.<Class<?>>of() : config.getSearchableEntityClasses())
+      {
+         QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class);
+         if(annotation != null)
+         {
+            tableNames.add(annotation.tableName());
+         }
+      }
+      for(SearchableTableConfig tableConfig : config.getSearchableTables() == null ? List.<SearchableTableConfig>of() : config.getSearchableTables())
+      {
+         if(tableConfig.getTableName() != null)
+         {
+            tableNames.add(tableConfig.getTableName());
+         }
+      }
+
+      List<QPossibleValue<?>> values = new ArrayList<>();
+      for(String tableName : tableNames)
+      {
+         QTableMetaData table = qInstance.getTable(tableName);
+         String         label = table != null && table.getLabel() != null ? table.getLabel() : tableName;
+         values.add(new QPossibleValue<>(tableName, label));
+      }
+
+      return (new QPossibleValueSource()
+         .withName(config.applyPrefix(NAME))
+         .withType(QPossibleValueSourceType.ENUM)
+         .withEnumValues(values));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchTableMetaDataHelper.java

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+
+
+/*******************************************************************************
+ ** Builds the QTableMetaData for the three operational tables. Shared by the
+ ** component producers and by tests that need the same shapes.
+ *******************************************************************************/
+public class QuickSearchTableMetaDataHelper
+{
+
+   /*******************************************************************************
+    ** quickSearchIndex: one row per indexed table.
+    *******************************************************************************/
+   public static QTableMetaData buildIndexTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyPrefix(QuickSearchIndex.TABLE_NAME))
+         .withLabel("Quick Search Index")
+         .withIcon(new QIcon("manage_search"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFields("tableName")
+         .withUniqueKey(new UniqueKey("tableName"))
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withIsEditable(false))
+         .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
+         .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("lastReconcileTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("lastRunStatus", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("lastErrorMessage", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("recordCount", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("documentCount", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("realTimeErrorCount", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** quickSearchIndexRun: history of basepull, reindex and reconcile runs.
+    *******************************************************************************/
+   public static QTableMetaData buildIndexRunTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyPrefix(QuickSearchIndexRun.TABLE_NAME))
+         .withLabel("Quick Search Index Run")
+         .withIcon(new QIcon("history"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withIsEditable(false)
+            .withPossibleValueSourceName(config.applyPrefix(QuickSearchIndex.TABLE_NAME)))
+         .withField(new QFieldMetaData("runType", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("errorCount", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** quickSearchFailedEvent: real-time index or delete events that could not
+    ** reach OpenSearch, replayed by the scheduled basepull.
+    *******************************************************************************/
+   public static QTableMetaData buildFailedEventTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME))
+         .withLabel("Quick Search Failed Event")
+         .withIcon(new QIcon("error_outline"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("recordId", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("action", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("attempts", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** quickSearchDiagnosticEvent: failed searches, diagnostic tool runs (the
+    ** audit trail) and refused runs. Written by the QBit only: insert, update
+    ** and delete are disabled for users and the API (core actions, which the
+    ** QBit uses, do not check capabilities).
+    *******************************************************************************/
+   public static QTableMetaData buildDiagnosticEventTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME))
+         .withLabel("Quick Search Diagnostic Event")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
+         .withIcon(new QIcon("bug_report"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("eventType", "source")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("eventType", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("outcome", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("userName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("tableNames", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("searchTerm", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("durationMillis", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("errorClass", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("detail", QFieldType.TEXT).withIsEditable(false))
+         .withField(new QFieldMetaData("hostName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Event", new QIcon("bug_report"), Tier.T1, List.of("id", "createDate", "eventType", "source", "outcome")))
+         .withSection(new QFieldSection("request", "Request", new QIcon("search"), Tier.T2, List.of("userName", "tableNames", "searchTerm", "durationMillis", "hostName")))
+         .withSection(new QFieldSection("error", "Error and detail", new QIcon("error_outline"), Tier.T2, List.of("errorClass", "errorMessage", "detail"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** quickSearchValidationRun: index validation reports (history). The
+    ** report is an HTML field, shown in full on the record view; only the
+    ** QBit writes rows (insert, update and delete are disabled for users and
+    ** the API), so the HTML is always the QBit's escaped output.
+    *******************************************************************************/
+   public static QTableMetaData buildValidationRunTable(QuickSearchQBitConfig config)
+   {
+      return (new QTableMetaData()
+         .withName(config.applyPrefix(QuickSearchValidationRun.TABLE_NAME))
+         .withLabel("Quick Search Validation Run")
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
+         .withIcon(new QIcon("fact_check"))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("status", "startTime")
+         .withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("triggerType", QFieldType.STRING).withLabel("Trigger").withIsEditable(false))
+         .withField(new QFieldMetaData("userName", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("tableNames", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("sampleSize", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("tablesChecked", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("failCount", QFieldType.INTEGER).withLabel("Failed Checks").withIsEditable(false))
+         .withField(new QFieldMetaData("warnCount", QFieldType.INTEGER).withLabel("Warnings").withIsEditable(false))
+         .withField(new QFieldMetaData("summary", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("reportHtml", QFieldType.HTML).withLabel("Report").withIsEditable(false))
+         .withField(new QFieldMetaData("reportJson", QFieldType.TEXT).withLabel("Report JSON").withIsEditable(false))
+         .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Run", new QIcon("fact_check"), Tier.T1, List.of("id", "status", "summary", "triggerType", "startTime", "endTime")))
+         .withSection(new QFieldSection("scope", "Scope", new QIcon("table_chart"), Tier.T2, List.of("userName", "tableNames", "sampleSize", "tablesChecked", "failCount", "warnCount", "createDate")))
+         .withSection(new QFieldSection("report", "Report", new QIcon("description"), Tier.T2, List.of("reportHtml")))
+         .withSection(new QFieldSection("data", "Report data (JSON)", new QIcon("data_object"), Tier.T3, List.of("reportJson"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Possible value source over quickSearchIndex, so run rows show the table
+    ** name instead of an id.
+    *******************************************************************************/
+   public static QPossibleValueSource buildIndexPossibleValueSource(QuickSearchQBitConfig config)
+   {
+      return (QPossibleValueSource.newForTable(config.applyPrefix(QuickSearchIndex.TABLE_NAME)));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/metadata/QuickSearchValidationRunTableMetaDataProducer.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+
+
+/*******************************************************************************
+ ** Produces the quickSearchValidationRun table when enableDiagnosticTools is true.
+ *******************************************************************************/
+public class QuickSearchValidationRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
+{
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      return (QuickSearchTableMetaDataHelper.buildValidationRunTable(getQBitConfig()));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
+   }
+
+
+
+   /*******************************************************************************
+    ** tables before processes and the app
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (100);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchDiagnosticEvent.java

@@ -0,0 +1,508 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ ** A Quick Search diagnostic event: a failed search (OpenSearch error or timeout,
+ ** after which core record search falls back to its own search), a diagnostic
+ ** tool run (the audit trail of the admin tools), or a refused run (rate limit).
+ ** Written by DiagnosticEventRecorder; read-only for operators.
+ *******************************************************************************/
+public class QuickSearchDiagnosticEvent extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quickSearchDiagnosticEvent";
+
+   public static final String TYPE_SEARCH_FAILED = "SEARCH_FAILED";
+   public static final String TYPE_TOOL_RUN = "TOOL_RUN";
+   public static final String TYPE_REINDEX = "REINDEX";
+   public static final String TYPE_RATE_LIMITED = "RATE_LIMITED";
+   public static final String SOURCE_CORE_RECORD_SEARCH = "CORE_RECORD_SEARCH";
+   public static final String SOURCE_QUICK_SEARCH_ACTION = "QUICK_SEARCH_ACTION";
+   public static final String OUTCOME_OK = "OK";
+   public static final String OUTCOME_ERROR = "ERROR";
+   public static final String OUTCOME_REFUSED = "REFUSED";
+
+   @QField()
+   private Integer id;
+
+   @QField()
+   private String eventType;
+
+   @QField()
+   private String source;
+
+   @QField()
+   private String outcome;
+
+   @QField()
+   private String userName;
+
+   @QField()
+   private String tableNames;
+
+   @QField()
+   private String searchTerm;
+
+   @QField()
+   private Integer durationMillis;
+
+   @QField()
+   private String errorClass;
+
+   @QField()
+   private String errorMessage;
+
+   @QField()
+   private String detail;
+
+   @QField()
+   private String hostName;
+
+   @QField()
+   private Instant createDate;
+
+
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor from a QRecord
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent(QRecord record) throws QException
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for eventType
+    *******************************************************************************/
+   public String getEventType()
+   {
+      return (this.eventType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for eventType
+    *******************************************************************************/
+   public void setEventType(String eventType)
+   {
+      this.eventType = eventType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for eventType
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withEventType(String eventType)
+   {
+      this.eventType = eventType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for source
+    *******************************************************************************/
+   public String getSource()
+   {
+      return (this.source);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for source
+    *******************************************************************************/
+   public void setSource(String source)
+   {
+      this.source = source;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for source
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withSource(String source)
+   {
+      this.source = source;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outcome
+    *******************************************************************************/
+   public String getOutcome()
+   {
+      return (this.outcome);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outcome
+    *******************************************************************************/
+   public void setOutcome(String outcome)
+   {
+      this.outcome = outcome;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outcome
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withOutcome(String outcome)
+   {
+      this.outcome = outcome;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userName
+    *******************************************************************************/
+   public String getUserName()
+   {
+      return (this.userName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userName
+    *******************************************************************************/
+   public void setUserName(String userName)
+   {
+      this.userName = userName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userName
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withUserName(String userName)
+   {
+      this.userName = userName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableNames
+    *******************************************************************************/
+   public String getTableNames()
+   {
+      return (this.tableNames);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableNames
+    *******************************************************************************/
+   public void setTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableNames
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for searchTerm
+    *******************************************************************************/
+   public String getSearchTerm()
+   {
+      return (this.searchTerm);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for searchTerm
+    *******************************************************************************/
+   public void setSearchTerm(String searchTerm)
+   {
+      this.searchTerm = searchTerm;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for searchTerm
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withSearchTerm(String searchTerm)
+   {
+      this.searchTerm = searchTerm;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for durationMillis
+    *******************************************************************************/
+   public Integer getDurationMillis()
+   {
+      return (this.durationMillis);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for durationMillis
+    *******************************************************************************/
+   public void setDurationMillis(Integer durationMillis)
+   {
+      this.durationMillis = durationMillis;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for durationMillis
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withDurationMillis(Integer durationMillis)
+   {
+      this.durationMillis = durationMillis;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorClass
+    *******************************************************************************/
+   public String getErrorClass()
+   {
+      return (this.errorClass);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorClass
+    *******************************************************************************/
+   public void setErrorClass(String errorClass)
+   {
+      this.errorClass = errorClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for errorClass
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withErrorClass(String errorClass)
+   {
+      this.errorClass = errorClass;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorMessage
+    *******************************************************************************/
+   public String getErrorMessage()
+   {
+      return (this.errorMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorMessage
+    *******************************************************************************/
+   public void setErrorMessage(String errorMessage)
+   {
+      this.errorMessage = errorMessage;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for errorMessage
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withErrorMessage(String errorMessage)
+   {
+      this.errorMessage = errorMessage;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for detail
+    *******************************************************************************/
+   public String getDetail()
+   {
+      return (this.detail);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for detail
+    *******************************************************************************/
+   public void setDetail(String detail)
+   {
+      this.detail = detail;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for detail
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withDetail(String detail)
+   {
+      this.detail = detail;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hostName
+    *******************************************************************************/
+   public String getHostName()
+   {
+      return (this.hostName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hostName
+    *******************************************************************************/
+   public void setHostName(String hostName)
+   {
+      this.hostName = hostName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hostName
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withHostName(String hostName)
+   {
+      this.hostName = hostName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchDiagnosticEvent withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchFailedEvent.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** A real-time index or delete event that could not be applied to OpenSearch.
+ ** Replayed by the scheduled basepull; status PENDING until it succeeds, or
+ ** EXHAUSTED after the retry limit.
+ *******************************************************************************/
+public class QuickSearchFailedEvent extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quickSearchFailedEvent";
+
+   public static final String STATUS_PENDING   = "PENDING";
+   public static final String STATUS_EXHAUSTED = "EXHAUSTED";
+
+   ///////////////////////////////////////////////////////////////////////////
+   // a delete made while a full reindex rebuilds into a new physical index; //
+   // the reindex applies it after the swap, and basepull does not replay it //
+   ///////////////////////////////////////////////////////////////////////////
+   public static final String STATUS_AWAITING_REINDEX = "AWAITING_REINDEX";
+
+   @QField()
+   private Integer id;
+
+   @QField()
+   private String tableName;
+
+   @QField()
+   private String recordId;
+
+   @QField()
+   private String action;
+
+   @QField()
+   private String errorMessage;
+
+   @QField()
+   private Integer attempts;
+
+   @QField()
+   private String status;
+
+   @QField()
+   private Instant createDate;
+
+   @QField()
+   private Instant modifyDate;
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return (this.tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for recordId
+    *******************************************************************************/
+   public String getRecordId()
+   {
+      return (this.recordId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for action
+    *******************************************************************************/
+   public String getAction()
+   {
+      return (this.action);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorMessage
+    *******************************************************************************/
+   public String getErrorMessage()
+   {
+      return (this.errorMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for attempts
+    *******************************************************************************/
+   public Integer getAttempts()
+   {
+      return (this.attempts);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for status
+    *******************************************************************************/
+   public String getStatus()
+   {
+      return (this.status);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchValidationRun.java

@@ -0,0 +1,572 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ ** One run of the index validation: overall status, counts of failed and
+ ** warning checks, and the full report (HTML for the record view, JSON for
+ ** tools). Kept as history; purged after runHistoryRetentionDays.
+ *******************************************************************************/
+public class QuickSearchValidationRun extends QRecordEntity
+{
+   public static final String TABLE_NAME = "quickSearchValidationRun";
+
+   public static final String STATUS_PASS = "PASS";
+   public static final String STATUS_WARN = "WARN";
+   public static final String STATUS_FAIL = "FAIL";
+   public static final String STATUS_ERROR = "ERROR";
+   public static final String TRIGGER_MANUAL = "MANUAL";
+   public static final String TRIGGER_SCHEDULED = "SCHEDULED";
+
+   @QField()
+   private Integer id;
+
+   @QField()
+   private String status;
+
+   @QField()
+   private String triggerType;
+
+   @QField()
+   private String userName;
+
+   @QField()
+   private String tableNames;
+
+   @QField()
+   private Integer sampleSize;
+
+   @QField()
+   private Integer tablesChecked;
+
+   @QField()
+   private Integer failCount;
+
+   @QField()
+   private Integer warnCount;
+
+   @QField()
+   private String summary;
+
+   @QField()
+   private String reportHtml;
+
+   @QField()
+   private String reportJson;
+
+   @QField()
+   private Instant startTime;
+
+   @QField()
+   private Instant endTime;
+
+   @QField()
+   private Instant createDate;
+
+
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public QuickSearchValidationRun()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor from a QRecord
+    *******************************************************************************/
+   public QuickSearchValidationRun(QRecord record) throws QException
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public QuickSearchValidationRun withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for status
+    *******************************************************************************/
+   public String getStatus()
+   {
+      return (this.status);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for status
+    *******************************************************************************/
+   public void setStatus(String status)
+   {
+      this.status = status;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for status
+    *******************************************************************************/
+   public QuickSearchValidationRun withStatus(String status)
+   {
+      this.status = status;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for triggerType
+    *******************************************************************************/
+   public String getTriggerType()
+   {
+      return (this.triggerType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for triggerType
+    *******************************************************************************/
+   public void setTriggerType(String triggerType)
+   {
+      this.triggerType = triggerType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for triggerType
+    *******************************************************************************/
+   public QuickSearchValidationRun withTriggerType(String triggerType)
+   {
+      this.triggerType = triggerType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userName
+    *******************************************************************************/
+   public String getUserName()
+   {
+      return (this.userName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userName
+    *******************************************************************************/
+   public void setUserName(String userName)
+   {
+      this.userName = userName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userName
+    *******************************************************************************/
+   public QuickSearchValidationRun withUserName(String userName)
+   {
+      this.userName = userName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableNames
+    *******************************************************************************/
+   public String getTableNames()
+   {
+      return (this.tableNames);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableNames
+    *******************************************************************************/
+   public void setTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableNames
+    *******************************************************************************/
+   public QuickSearchValidationRun withTableNames(String tableNames)
+   {
+      this.tableNames = tableNames;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sampleSize
+    *******************************************************************************/
+   public Integer getSampleSize()
+   {
+      return (this.sampleSize);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sampleSize
+    *******************************************************************************/
+   public void setSampleSize(Integer sampleSize)
+   {
+      this.sampleSize = sampleSize;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sampleSize
+    *******************************************************************************/
+   public QuickSearchValidationRun withSampleSize(Integer sampleSize)
+   {
+      this.sampleSize = sampleSize;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tablesChecked
+    *******************************************************************************/
+   public Integer getTablesChecked()
+   {
+      return (this.tablesChecked);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tablesChecked
+    *******************************************************************************/
+   public void setTablesChecked(Integer tablesChecked)
+   {
+      this.tablesChecked = tablesChecked;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tablesChecked
+    *******************************************************************************/
+   public QuickSearchValidationRun withTablesChecked(Integer tablesChecked)
+   {
+      this.tablesChecked = tablesChecked;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for failCount
+    *******************************************************************************/
+   public Integer getFailCount()
+   {
+      return (this.failCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for failCount
+    *******************************************************************************/
+   public void setFailCount(Integer failCount)
+   {
+      this.failCount = failCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for failCount
+    *******************************************************************************/
+   public QuickSearchValidationRun withFailCount(Integer failCount)
+   {
+      this.failCount = failCount;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for warnCount
+    *******************************************************************************/
+   public Integer getWarnCount()
+   {
+      return (this.warnCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for warnCount
+    *******************************************************************************/
+   public void setWarnCount(Integer warnCount)
+   {
+      this.warnCount = warnCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for warnCount
+    *******************************************************************************/
+   public QuickSearchValidationRun withWarnCount(Integer warnCount)
+   {
+      this.warnCount = warnCount;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for summary
+    *******************************************************************************/
+   public String getSummary()
+   {
+      return (this.summary);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for summary
+    *******************************************************************************/
+   public void setSummary(String summary)
+   {
+      this.summary = summary;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for summary
+    *******************************************************************************/
+   public QuickSearchValidationRun withSummary(String summary)
+   {
+      this.summary = summary;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reportHtml
+    *******************************************************************************/
+   public String getReportHtml()
+   {
+      return (this.reportHtml);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for reportHtml
+    *******************************************************************************/
+   public void setReportHtml(String reportHtml)
+   {
+      this.reportHtml = reportHtml;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reportHtml
+    *******************************************************************************/
+   public QuickSearchValidationRun withReportHtml(String reportHtml)
+   {
+      this.reportHtml = reportHtml;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reportJson
+    *******************************************************************************/
+   public String getReportJson()
+   {
+      return (this.reportJson);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for reportJson
+    *******************************************************************************/
+   public void setReportJson(String reportJson)
+   {
+      this.reportJson = reportJson;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reportJson
+    *******************************************************************************/
+   public QuickSearchValidationRun withReportJson(String reportJson)
+   {
+      this.reportJson = reportJson;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for startTime
+    *******************************************************************************/
+   public Instant getStartTime()
+   {
+      return (this.startTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for startTime
+    *******************************************************************************/
+   public void setStartTime(Instant startTime)
+   {
+      this.startTime = startTime;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for startTime
+    *******************************************************************************/
+   public QuickSearchValidationRun withStartTime(Instant startTime)
+   {
+      this.startTime = startTime;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for endTime
+    *******************************************************************************/
+   public Instant getEndTime()
+   {
+      return (this.endTime);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for endTime
+    *******************************************************************************/
+   public void setEndTime(Instant endTime)
+   {
+      this.endTime = endTime;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for endTime
+    *******************************************************************************/
+   public QuickSearchValidationRun withEndTime(Instant endTime)
+   {
+      this.endTime = endTime;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public QuickSearchValidationRun withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java

@@ -29,6 +29,7 @@ public class BulkIndexResult
 
    private Integer      successCount = 0;
    private Integer      failureCount = 0;
+   private Integer      skippedCount = 0;
    private List<String> errors       = new ArrayList<>();
 
 
@@ -125,4 +126,24 @@ public class BulkIndexResult
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Count a document skipped because a newer version is already indexed.
+    *******************************************************************************/
+   public void addSkipped()
+   {
+      skippedCount++;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for skippedCount
+    *******************************************************************************/
+   public Integer getSkippedCount()
+   {
+      return (skippedCount);
+   }
+
 }

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocument.java

@@ -38,6 +38,8 @@ public class OpenSearchDocument
    private Instant indexedAt;
    private Map<String, Object> fieldValues;
 
+   private transient Long version;
+
 
 
    /*******************************************************************************
@@ -170,8 +172,42 @@ public class OpenSearchDocument
     ** Returns the composite document ID used as the OpenSearch _id field.
     ** Excluded from JSON serialization to avoid including it in the document body.
     *******************************************************************************/
+   /*******************************************************************************
+    ** External version for optimistic indexing (epoch millis of the source
+    ** row's timestamp), or null to index unconditionally. Not stored in the
+    ** document body.
+    *******************************************************************************/
+   @JsonIgnore
+   public Long getVersion()
+   {
+      return (version);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for version
+    *******************************************************************************/
+   public OpenSearchDocument withVersion(Long version)
+   {
+      this.version = version;
+      return (this);
+   }
+
+
+
    @JsonIgnore
    public String getDocumentId()
+   {
+      return (buildDocumentId(sourceTable, recordId));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the composite OpenSearch _id for a source table and record ID.
+    *******************************************************************************/
+   public static String buildDocumentId(String sourceTable, String recordId)
    {
       return (sourceTable + ":" + recordId);
    }

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchTransportCustomizer.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.opensearch;
+
+
+import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
+
+
+/*******************************************************************************
+ ** Escape hatch for hosts that need something the built-in auth modes do not
+ ** cover: bearer or JWT tokens, API-key headers, request interceptors, proxy
+ ** settings, or client certificates loaded from a custom source.
+ **
+ ** Register an implementation on QuickSearchQBitConfig.transportCustomizer as a
+ ** QCodeReference. Both hooks run after the QBit has applied its own settings
+ ** (auth, TLS, timeouts) and before the transport is built. Not used for the
+ ** AWS_SIGV4 transport, which is built by the AWS SDK.
+ *******************************************************************************/
+public interface OpenSearchTransportCustomizer
+{
+
+   /*******************************************************************************
+    ** Adjust the transport builder (default headers, request config callback).
+    *******************************************************************************/
+   default ApacheHttpClient5TransportBuilder customizeTransport(ApacheHttpClient5TransportBuilder builder)
+   {
+      return (builder);
+   }
+
+
+
+   /*******************************************************************************
+    ** Adjust the Apache HttpAsyncClient builder (interceptors, connection
+    ** manager, credentials).
+    *******************************************************************************/
+   default HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder builder)
+   {
+      return (builder);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchTransportFactory.java

@@ -0,0 +1,394 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.opensearch;
+
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.util.Collection;
+import java.util.regex.Pattern;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
+import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
+import com.kingsrook.qbits.quicksearch.QuickSearchTlsConfig;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
+import org.opensearch.client.json.JsonpMapper;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.aws.AwsSdk2Transport;
+import org.opensearch.client.transport.aws.AwsSdk2TransportOptions;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Builds the OpenSearch transport for a config: NONE or BASIC auth over the
+ ** Apache HttpClient 5 transport with TLS, timeouts, pool size, user agent and
+ ** the optional OpenSearchTransportCustomizer; or AWS_SIGV4 over AwsSdk2Transport.
+ **
+ ** Secret-bearing values (username, password, store passwords, paths) are
+ ** interpreted here with QMetaDataVariableInterpreter, so ${env.X} references
+ ** resolve only inside this factory.
+ *******************************************************************************/
+public class OpenSearchTransportFactory
+{
+   private static final QLogger LOG = QLogger.getLogger(OpenSearchTransportFactory.class);
+
+   private static final String USER_AGENT = "qbit-quick-search/" + QuickSearchQBitProducer.getVersion();
+
+   ///////////////////////////////////////////////////////////////////////////
+   // Only a lone ${env.X} or ${prop.X} is echoed in errors: a ?? chain may //
+   // hold a literal fallback secret between its references.                //
+   ///////////////////////////////////////////////////////////////////////////
+   private static final Pattern SINGLE_REFERENCE = Pattern.compile("^\\$\\{(env|prop)\\.[A-Za-z0-9_.-]+\\}$");
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static OpenSearchTransport build(QuickSearchQBitConfig config) throws QException
+   {
+      try
+      {
+         JsonpMapper mapper = buildMapper();
+
+         return switch(config.getEffectiveAuthMode())
+         {
+            case AWS_SIGV4 -> buildAwsTransport(config, mapper);
+            case NONE, BASIC -> buildApacheTransport(config, mapper);
+         };
+      }
+      catch(QException e)
+      {
+         throw (e);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Failed to create OpenSearch transport: " + e.getMessage(), e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static JsonpMapper buildMapper()
+   {
+      ObjectMapper objectMapper = new ObjectMapper();
+      objectMapper.registerModule(new JavaTimeModule());
+      objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+      return (new JacksonJsonpMapper(objectMapper));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static OpenSearchTransport buildApacheTransport(QuickSearchQBitConfig config, JsonpMapper mapper) throws Exception
+   {
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+
+      HttpHost httpHost = new HttpHost(config.getEffectiveScheme(), config.getEffectiveHost(), config.getEffectivePort());
+
+      BasicCredentialsProvider credentialsProvider = null;
+      if(config.getEffectiveAuthMode() == QuickSearchAuthMode.BASIC)
+      {
+         String username = requireResolved(interpreter, config.getOpensearchUsername(), "opensearchUsername");
+         String password = requireResolved(interpreter, config.getOpensearchPassword(), "opensearchPassword");
+         credentialsProvider = new BasicCredentialsProvider();
+         credentialsProvider.setCredentials(new AuthScope(httpHost), new UsernamePasswordCredentials(username, password.toCharArray()));
+      }
+
+      OpenSearchTransportCustomizer customizer = loadCustomizer(config);
+      SSLContext                    sslContext = buildSslContext(config.getTls(), interpreter);
+      HostnameVerifier              verifier   = buildHostnameVerifier(config.getTls());
+
+      BasicCredentialsProvider finalCredentialsProvider = credentialsProvider;
+      int                      maxConnections           = config.getMaxConnections() == null ? ApacheHttpClient5TransportBuilder.DEFAULT_MAX_CONN_TOTAL : config.getMaxConnections();
+      int                      connectTimeout           = config.getConnectTimeoutMillis() == null ? ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS : config.getConnectTimeoutMillis();
+      int                      responseTimeout          = config.getResponseTimeoutMillis() == null ? ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS : config.getResponseTimeoutMillis();
+
+      ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder
+         .builder(httpHost)
+         .setMapper(mapper)
+         .setDefaultHeaders(new BasicHeader[] { new BasicHeader("User-Agent", USER_AGENT) })
+         .setRequestConfigCallback(requestConfig -> requestConfig
+            .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
+            .setResponseTimeout(Timeout.ofMilliseconds(responseTimeout)))
+         .setHttpClientConfigCallback(httpClientBuilder ->
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // httpclient5 5.6+ decompresses gzip responses itself but leaves    //
+            // the Content-Encoding header, so the OpenSearch transport would    //
+            // gunzip the body a second time and fail.  Let the transport alone //
+            // handle compression.                                               //
+            ///////////////////////////////////////////////////////////////////////
+            httpClientBuilder.disableContentCompression();
+
+            if(finalCredentialsProvider != null)
+            {
+               httpClientBuilder.setDefaultCredentialsProvider(finalCredentialsProvider);
+            }
+
+            ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create();
+            if(sslContext != null)
+            {
+               tlsStrategyBuilder.setSslContext(sslContext);
+            }
+            if(verifier != null)
+            {
+               tlsStrategyBuilder.setHostnameVerifier(verifier);
+            }
+
+            httpClientBuilder.setConnectionManager(PoolingAsyncClientConnectionManagerBuilder.create()
+               .setTlsStrategy(tlsStrategyBuilder.build())
+               .setMaxConnTotal(maxConnections)
+               .setMaxConnPerRoute(maxConnections)
+               .build());
+
+            return (customizer == null ? httpClientBuilder : customizer.customizeHttpClient(httpClientBuilder));
+         });
+
+      if(customizer != null)
+      {
+         builder = customizer.customizeTransport(builder);
+      }
+
+      return (builder.build());
+   }
+
+
+
+   /*******************************************************************************
+    ** AWS Signature V4 over the AWS SDK's Apache HTTP client. Credentials come
+    ** from the default chain, or from STS when awsAssumeRoleArn is set.
+    *******************************************************************************/
+   private static OpenSearchTransport buildAwsTransport(QuickSearchQBitConfig config, JsonpMapper mapper) throws Exception
+   {
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+
+      String region  = config.getEffectiveAwsRegion();
+      String service = config.getEffectiveAwsServiceName();
+      String host    = config.getEffectiveHost();
+      if(config.getEffectivePort() != null && config.getEffectivePort() != 443)
+      {
+         host = host + ":" + config.getEffectivePort();
+      }
+
+      AwsCredentialsProvider credentials = DefaultCredentialsProvider.create();
+      String                 roleArn     = interpreter.interpret(config.getAwsAssumeRoleArn());
+      if(StringUtils.hasContent(roleArn))
+      {
+         StsClient stsClient = StsClient.builder().region(Region.of(region)).credentialsProvider(credentials).build();
+         credentials = StsAssumeRoleCredentialsProvider.builder()
+            .stsClient(stsClient)
+            .refreshRequest(AssumeRoleRequest.builder().roleArn(roleArn).roleSessionName("qbit-quick-search").build())
+            .build();
+      }
+
+      LOG.info("Building AWS SigV4 OpenSearch transport", logPair("host", host), logPair("region", region), logPair("service", service), logPair("assumeRole", StringUtils.hasContent(roleArn)));
+
+      return (new AwsSdk2Transport(
+         ApacheHttpClient.builder().build(),
+         host,
+         service,
+         Region.of(region),
+         AwsSdk2TransportOptions.builder().setCredentials(credentials).setMapper(mapper).build()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build an SSLContext for the tls block, or null for JVM defaults.
+    *******************************************************************************/
+   static SSLContext buildSslContext(QuickSearchTlsConfig tls, QMetaDataVariableInterpreter interpreter) throws Exception
+   {
+      if(tls == null)
+      {
+         return (null);
+      }
+
+      SSLContextBuilder builder = SSLContextBuilder.create();
+      boolean           custom  = false;
+
+      if(Boolean.TRUE.equals(tls.getInsecureSkipVerify()))
+      {
+         LOG.warn("tls.insecureSkipVerify is enabled: OpenSearch server certificates are NOT verified");
+         builder.loadTrustMaterial(null, (chain, authType) -> true);
+         custom = true;
+      }
+      else if(StringUtils.hasContent(tls.getCaCertificatePath()))
+      {
+         builder.loadTrustMaterial(loadPemTrustStore(requireResolved(interpreter, tls.getCaCertificatePath(), "tls.caCertificatePath")), null);
+         custom = true;
+      }
+      else if(StringUtils.hasContent(tls.getTrustStorePath()))
+      {
+         String   password   = interpreter.interpret(tls.getTrustStorePassword());
+         KeyStore trustStore = KeyStore.getInstance(StringUtils.hasContent(tls.getTrustStoreType()) ? tls.getTrustStoreType() : KeyStore.getDefaultType());
+         try(InputStream inputStream = new FileInputStream(requireResolved(interpreter, tls.getTrustStorePath(), "tls.trustStorePath")))
+         {
+            trustStore.load(inputStream, password == null ? null : password.toCharArray());
+         }
+         builder.loadTrustMaterial(trustStore, null);
+         custom = true;
+      }
+
+      if(StringUtils.hasContent(tls.getKeyStorePath()))
+      {
+         String   password = interpreter.interpret(tls.getKeyStorePassword());
+         char[]   chars    = password == null ? new char[0] : password.toCharArray();
+         KeyStore keyStore = KeyStore.getInstance(StringUtils.hasContent(tls.getKeyStoreType()) ? tls.getKeyStoreType() : KeyStore.getDefaultType());
+         try(InputStream inputStream = new FileInputStream(requireResolved(interpreter, tls.getKeyStorePath(), "tls.keyStorePath")))
+         {
+            keyStore.load(inputStream, chars);
+         }
+         builder.loadKeyMaterial(keyStore, chars);
+         custom = true;
+      }
+
+      return (custom ? builder.build() : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The no-op verifier when insecureSkipVerify (warned in buildSslContext) or
+    ** hostnameVerification=false is set, else null for the default verifier.
+    *******************************************************************************/
+   static HostnameVerifier buildHostnameVerifier(QuickSearchTlsConfig tls)
+   {
+      if(tls == null)
+      {
+         return (null);
+      }
+
+      if(Boolean.TRUE.equals(tls.getInsecureSkipVerify()))
+      {
+         return (NoopHostnameVerifier.INSTANCE);
+      }
+
+      if(Boolean.FALSE.equals(tls.getHostnameVerification()))
+      {
+         LOG.warn("tls.hostnameVerification is disabled: OpenSearch server certificates are NOT checked against the host name");
+         return (NoopHostnameVerifier.INSTANCE);
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Interpret a value that must resolve to content; null, empty and blank all
+    ** count as missing. The error names the field and, for a single ${env.X} or
+    ** ${prop.X} reference, the reference itself; never a resolved value and
+    ** never any part of a ?? fallback chain.
+    *******************************************************************************/
+   static String requireResolved(QMetaDataVariableInterpreter interpreter, String value, String fieldName) throws QException
+   {
+      String resolved = interpreter.interpret(value);
+      if(StringUtils.hasContent(resolved))
+      {
+         return (resolved);
+      }
+
+      String  trimmed     = value == null ? "" : value.trim();
+      boolean isReference = SINGLE_REFERENCE.matcher(trimmed).matches();
+      throw (new QException(fieldName + " is missing or empty" + (isReference ? " after resolving " + trimmed : "")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Load one or more PEM certificates into an in-memory trust store.
+    *******************************************************************************/
+   static KeyStore loadPemTrustStore(String path) throws Exception
+   {
+      KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+      trustStore.load(null, null);
+
+      try(InputStream inputStream = new FileInputStream(path))
+      {
+         Collection<? extends Certificate> certificates = CertificateFactory.getInstance("X.509").generateCertificates(inputStream);
+         int                               i            = 0;
+         for(Certificate certificate : certificates)
+         {
+            trustStore.setCertificateEntry("quick-search-ca-" + (i++), certificate);
+         }
+         if(i == 0)
+         {
+            throw (new QException("No certificates found in tls.caCertificatePath"));
+         }
+      }
+
+      return (trustStore);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static OpenSearchTransportCustomizer loadCustomizer(QuickSearchQBitConfig config) throws QException
+   {
+      if(config.getTransportCustomizer() == null)
+      {
+         return (null);
+      }
+
+      OpenSearchTransportCustomizer customizer = QCodeLoader.getAdHoc(OpenSearchTransportCustomizer.class, config.getTransportCustomizer());
+      if(customizer == null)
+      {
+         throw (new QException("Could not load transportCustomizer [" + config.getTransportCustomizer().getName() + "]"));
+      }
+      return (customizer);
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchDiagnosticsOperations.java

@@ -0,0 +1,566 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.opensearch;
+
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import jakarta.json.stream.JsonGenerator;
+import org.opensearch.client.json.JsonpMapper;
+import org.opensearch.client.json.JsonpSerializable;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch._types.OpenSearchException;
+import org.opensearch.client.opensearch._types.query_dsl.Query;
+import org.opensearch.client.opensearch.core.ExplainResponse;
+import org.opensearch.client.opensearch.core.SearchRequest;
+import org.opensearch.client.opensearch.core.SearchResponse;
+import org.opensearch.client.opensearch.core.search.Hit;
+import org.opensearch.client.opensearch.core.search.HighlightField;
+import org.opensearch.client.opensearch.core.search.MatchedQueries;
+import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
+import org.opensearch.client.opensearch.generic.Body;
+import org.opensearch.client.opensearch.generic.Requests;
+import org.opensearch.client.opensearch.generic.Response;
+
+
+/*******************************************************************************
+ ** Read-only OpenSearch operations behind the Quick Search admin diagnostics:
+ ** a probe search that reports scores, highlights and matched clauses with
+ ** its request and response JSON, explain for one document, document reads,
+ ** _analyze, random document sampling, and raw cluster / index / mapping /
+ ** settings reads. Nothing here writes to the cluster.
+ ** The probe uses exactly the query production search builds (same clauses,
+ ** operator, boosts and table filter); it only adds query names and
+ ** highlighting on the field values, which change neither matching nor
+ ** scoring.
+ *******************************************************************************/
+public class QuickSearchDiagnosticsOperations
+{
+   public static final int MAX_ANALYZE_TEXT_LENGTH = 10_000;
+   public static final int MAX_PROBE_SIZE          = 100;
+   public static final int MAX_SAMPLE_SIZE         = 1_000;
+
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+   private final QuickSearchOpenSearchClient quickSearchClient;
+   private final OpenSearchClient            client;
+   private final String                      indexName;
+
+
+
+   /*******************************************************************************
+    ** One hit of a probe search.
+    *******************************************************************************/
+   public record ProbeHit(String recordId, String recordLabel, Double score, List<String> matchedClauses, Map<String, List<String>> highlights)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A probe search of one table: request and response JSON, server and
+    ** wall-clock time, total hits and the hits.
+    *******************************************************************************/
+   public record SearchProbe(String tableName, String requestJson, String responseJson, Long tookMillis, Long wallMillis, Long totalHits, Boolean totalIsLowerBound, List<ProbeHit> hits)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The result of explain for one document and the search query.
+    *******************************************************************************/
+   public record ExplainResult(Boolean documentFound, Boolean matched, Double score, String explanationText, String json)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One indexed document as stored: whether it exists, its version, the
+    ** physical index holding it and its source.
+    *******************************************************************************/
+   public record DocumentView(Boolean found, Long version, String physicalIndex, JsonNode source)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** One token produced by _analyze.
+    *******************************************************************************/
+   public record AnalyzedToken(String token, Integer position, Integer startOffset, Integer endOffset, String type)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A raw JSON response: HTTP status and parsed body (null when empty).
+    *******************************************************************************/
+   public record RawResponse(Integer status, JsonNode body)
+   {
+      /*******************************************************************************
+       ** Whether the status is 2xx.
+       *******************************************************************************/
+      public boolean isOk()
+      {
+         return (status != null && status >= 200 && status < 300);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public QuickSearchDiagnosticsOperations(QuickSearchOpenSearchClient quickSearchClient)
+   {
+      this.quickSearchClient = quickSearchClient;
+      this.client            = quickSearchClient.getOpenSearchClient();
+      this.indexName         = quickSearchClient.getIndexName();
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for the alias name the operations read.
+    *******************************************************************************/
+   public String getIndexName()
+   {
+      return (indexName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the production search query against one table with query names and
+    ** field highlighting, returning up to size hits (capped at MAX_PROBE_SIZE).
+    *******************************************************************************/
+   public SearchProbe probeSearch(String term, String tableName, int size, List<QuickSearchableTableConfig> tableConfigs) throws QException
+   {
+      int           boundedSize = Math.max(1, Math.min(size, MAX_PROBE_SIZE));
+      Query         query       = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, true);
+      SearchRequest request     = SearchRequest.of(r -> r
+         .index(indexName)
+         .query(query)
+         .highlight(h -> h
+            .requireFieldMatch(true)
+            .fields("searchableText", HighlightField.of(hf -> hf))
+            .fields("fieldValues.*", HighlightField.of(hf -> hf)))
+         .trackTotalHits(t -> t.count(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW))
+         .size(boundedSize));
+
+      long start = System.nanoTime();
+      try
+      {
+         SearchResponse<OpenSearchDocument> response = client.search(request, OpenSearchDocument.class);
+         long                               wall     = (System.nanoTime() - start) / 1_000_000;
+
+         List<ProbeHit> hits = new ArrayList<>();
+         for(Hit<OpenSearchDocument> hit : response.hits().hits())
+         {
+            OpenSearchDocument source = hit.source();
+            hits.add(new ProbeHit(
+               source == null ? idFromDocumentId(hit.id()) : source.getRecordId(),
+               source == null ? null : source.getRecordLabel(),
+               hit.score(),
+               matchedClauses(hit.matchedQueries()),
+               hit.highlight() == null ? Map.of() : new LinkedHashMap<>(hit.highlight())));
+         }
+
+         Long    total      = response.hits().total() == null ? (long) hits.size() : response.hits().total().value();
+         Boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
+         return (new SearchProbe(tableName, toJson(request), toJson(response), response.took(), wall, total, lowerBound, hits));
+      }
+      catch(OpenSearchException | IOException e)
+      {
+         throw (wrap("Probe search failed for table [" + tableName + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Names of the clauses a hit matched (query names or, in newer servers,
+    ** the keys of the named-query scores).
+    *******************************************************************************/
+   static List<String> matchedClauses(MatchedQueries matchedQueries)
+   {
+      if(matchedQueries == null)
+      {
+         return (List.of());
+      }
+      if(matchedQueries.isScores())
+      {
+         return (new ArrayList<>(matchedQueries.scores().keySet()));
+      }
+      return (new ArrayList<>(matchedQueries.names()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Explain how the production query scores one document of a table. A
+    ** missing document gives documentFound false (not an error).
+    *******************************************************************************/
+   public ExplainResult explain(String term, String tableName, String recordId, List<QuickSearchableTableConfig> tableConfigs) throws QException
+   {
+      Query  query      = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, false);
+      String documentId = OpenSearchDocument.buildDocumentId(tableName, recordId);
+      try
+      {
+         ExplainResponse<OpenSearchDocument> response = client.explain(e -> e.index(indexName).id(documentId).query(query), OpenSearchDocument.class);
+         String   json        = toJson(response);
+         JsonNode tree        = MAPPER.readTree(json);
+         JsonNode explanation = tree.path("explanation");
+         Double   score       = explanation.isMissingNode() ? null : explanation.path("value").asDouble();
+         return (new ExplainResult(true, response.matched(), score, formatExplanation(explanation), json));
+      }
+      catch(OpenSearchException e)
+      {
+         if(e.status() == 404)
+         {
+            return (new ExplainResult(false, false, null, null, null));
+         }
+         throw (wrap("Explain failed for document [" + documentId + "]", e));
+      }
+      catch(IOException e)
+      {
+         throw (wrap("Explain failed for document [" + documentId + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An explanation tree as indented text ("value  description"), bounded to
+    ** 400 lines.
+    *******************************************************************************/
+   public static String formatExplanation(JsonNode explanation)
+   {
+      if(explanation == null || explanation.isMissingNode() || explanation.isNull())
+      {
+         return ("");
+      }
+      List<String> lines = new ArrayList<>();
+      appendExplanation(explanation, 0, lines);
+      return (String.join("\n", lines));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static void appendExplanation(JsonNode node, int depth, List<String> lines)
+   {
+      if(lines.size() >= 400)
+      {
+         return;
+      }
+      lines.add("  ".repeat(depth) + String.format("%.4f", node.path("value").asDouble()) + "  " + node.path("description").asText(""));
+      for(JsonNode detail : node.path("details"))
+      {
+         appendExplanation(detail, depth + 1, lines);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Read one document of a table by record id.
+    *******************************************************************************/
+   public DocumentView getDocument(String tableName, String recordId) throws QException
+   {
+      RawResponse response = raw("GET", "/" + encode(indexName) + "/_doc/" + encode(OpenSearchDocument.buildDocumentId(tableName, recordId)), null);
+      if(response.status() == 404)
+      {
+         return (new DocumentView(false, null, null, null));
+      }
+      requireOk(response, "Document read failed");
+      JsonNode body = response.body();
+      return (new DocumentView(body.path("found").asBoolean(false), body.has("_version") ? body.get("_version").asLong() : null,
+         body.path("_index").asText(null), body.get("_source")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Read many documents of a table by record id: record id to source, for
+    ** the documents that exist.
+    *******************************************************************************/
+   public Map<String, JsonNode> getDocuments(String tableName, Collection<String> recordIds) throws QException
+   {
+      Map<String, JsonNode> sources = new LinkedHashMap<>();
+      if(recordIds == null || recordIds.isEmpty())
+      {
+         return (sources);
+      }
+
+      ObjectNode body = JsonNodeFactory.instance.objectNode();
+      ArrayNode  ids  = body.putArray("ids");
+      for(String recordId : recordIds)
+      {
+         ids.add(OpenSearchDocument.buildDocumentId(tableName, recordId));
+      }
+
+      RawResponse response = raw("POST", "/" + encode(indexName) + "/_mget", body.toString());
+      requireOk(response, "Document read failed");
+      for(JsonNode doc : response.body().path("docs"))
+      {
+         if(doc.path("found").asBoolean(false) && doc.has("_source"))
+         {
+            sources.put(idFromDocumentId(doc.path("_id").asText()), doc.get("_source"));
+         }
+      }
+      return (sources);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run _analyze on the index with a named analyzer (an index analyzer or a
+    ** built-in one). Text is truncated to MAX_ANALYZE_TEXT_LENGTH.
+    *******************************************************************************/
+   public List<AnalyzedToken> analyze(String analyzer, String text) throws QException
+   {
+      String     bounded = text == null ? "" : (text.length() > MAX_ANALYZE_TEXT_LENGTH ? text.substring(0, MAX_ANALYZE_TEXT_LENGTH) : text);
+      ObjectNode body    = JsonNodeFactory.instance.objectNode().put("analyzer", analyzer).put("text", bounded);
+
+      RawResponse response = raw("POST", "/" + encode(indexName) + "/_analyze", body.toString());
+      requireOk(response, "Analyze failed for analyzer [" + analyzer + "]");
+
+      List<AnalyzedToken> tokens = new ArrayList<>();
+      for(JsonNode token : response.body().path("tokens"))
+      {
+         tokens.add(new AnalyzedToken(token.path("token").asText(), token.path("position").asInt(), token.path("start_offset").asInt(),
+            token.path("end_offset").asInt(), token.path("type").asText(null)));
+      }
+      return (tokens);
+   }
+
+
+
+   /*******************************************************************************
+    ** Up to n record ids of a table's documents, chosen at random.
+    *******************************************************************************/
+   public List<String> sampleRecordIds(String tableName, int n) throws QException
+   {
+      int        size  = Math.max(1, Math.min(n, MAX_SAMPLE_SIZE));
+      ObjectNode body  = JsonNodeFactory.instance.objectNode();
+      body.put("size", size);
+      body.putArray("_source").add("recordId");
+      ObjectNode functionScore = body.putObject("query").putObject("function_score");
+      functionScore.putObject("query").putObject("term").put("sourceTable", tableName);
+      functionScore.putObject("random_score");
+
+      RawResponse response = raw("POST", "/" + encode(indexName) + "/_search", body.toString());
+      requireOk(response, "Sampling documents failed for table [" + tableName + "]");
+
+      List<String> recordIds = new ArrayList<>();
+      for(JsonNode hit : response.body().path("hits").path("hits"))
+      {
+         String recordId = hit.path("_source").path("recordId").asText(null);
+         recordIds.add(recordId != null ? recordId : idFromDocumentId(hit.path("_id").asText()));
+      }
+      return (recordIds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Cluster health, with the alias's indexes (level=indices).
+    *******************************************************************************/
+   public RawResponse clusterHealth() throws QException
+   {
+      return (raw("GET", "/_cluster/health/" + encode(indexName) + "?level=indices", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The live mapping of the alias's index(es).
+    *******************************************************************************/
+   public RawResponse mapping() throws QException
+   {
+      return (raw("GET", "/" + encode(indexName) + "/_mapping", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The live settings of the alias's index(es).
+    *******************************************************************************/
+   public RawResponse settings() throws QException
+   {
+      return (raw("GET", "/" + encode(indexName) + "/_settings", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The physical indexes named like this alias's ({alias}-v*), with health,
+    ** status, document counts and size, from _cat/indices.
+    *******************************************************************************/
+   public RawResponse physicalIndexes() throws QException
+   {
+      return (raw("GET", "/_cat/indices/" + encode(indexName) + "-v*?format=json&h=index,health,status,docs.count,docs.deleted,store.size&expand_wildcards=all", null));
+   }
+
+
+
+   /*******************************************************************************
+    ** The record id part of a document id ("table:id").
+    *******************************************************************************/
+   static String idFromDocumentId(String documentId)
+   {
+      if(documentId == null)
+      {
+         return (null);
+      }
+      int colon = documentId.indexOf(':');
+      return (colon < 0 ? documentId : documentId.substring(colon + 1));
+   }
+
+
+
+   /*******************************************************************************
+    ** Send a raw request and parse the JSON body. HTTP errors are returned
+    ** (status and body), not thrown; transport errors throw.
+    *******************************************************************************/
+   public RawResponse raw(String method, String endpoint, String jsonBody) throws QException
+   {
+      Requests.JsonBodyBuilder builder = Requests.builder().method(method).endpoint(endpoint);
+      if(jsonBody != null)
+      {
+         builder.json(jsonBody);
+      }
+
+      try(Response response = client.generic().execute(builder.build()))
+      {
+         JsonNode body = null;
+         if(response.getBody().isPresent())
+         {
+            try(Body responseBody = response.getBody().get())
+            {
+               String text = responseBody.bodyAsString();
+               body = text == null || text.isBlank() ? null : MAPPER.readTree(text);
+            }
+         }
+         return (new RawResponse(response.getStatus(), body));
+      }
+      catch(OpenSearchException | IOException e)
+      {
+         throw (wrap(method + " " + endpoint.replaceAll("\\?.*", "") + " failed", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Throw a QException with OpenSearch's reason when a raw response is not
+    ** 2xx.
+    *******************************************************************************/
+   public static void requireOk(RawResponse response, String message) throws QException
+   {
+      if(!response.isOk())
+      {
+         throw (new QException(message + ": OpenSearch returned status " + response.status() + ": " + errorReason(response.body())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** OpenSearch's error reason from an error body, or a short form of it.
+    *******************************************************************************/
+   public static String errorReason(JsonNode body)
+   {
+      if(body == null)
+      {
+         return ("no response body");
+      }
+      JsonNode error = body.path("error");
+      if(error.isTextual())
+      {
+         return (error.asText());
+      }
+      String reason = error.path("reason").asText(null);
+      if(reason != null)
+      {
+         return (reason);
+      }
+      String text = body.toString();
+      return (text.length() > 300 ? text.substring(0, 300) + "..." : text);
+   }
+
+
+
+   /*******************************************************************************
+    ** Serialize a request or response with the transport's mapper (which knows
+    ** OpenSearchDocument; the default mapper of toJsonString does not).
+    *******************************************************************************/
+   private String toJson(JsonpSerializable value)
+   {
+      JsonpMapper  mapper = client._transport().jsonpMapper();
+      StringWriter writer = new StringWriter();
+      try(JsonGenerator generator = mapper.jsonProvider().createGenerator(writer))
+      {
+         value.serialize(generator, mapper);
+      }
+      return (writer.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** URL-encode one path segment.
+    *******************************************************************************/
+   static String encode(String segment)
+   {
+      return (URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20"));
+   }
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   private static QException wrap(String message, Exception e)
+   {
+      if(e instanceof OpenSearchException openSearchException)
+      {
+         String reason = openSearchException.error() == null ? openSearchException.getMessage() : openSearchException.error().reason();
+         return (new QException(message + ": OpenSearch returned status " + openSearchException.status() + ": " + reason, e));
+      }
+      return (new QException(message + ": " + e.getMessage(), e));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java

@@ -13,64 +13,88 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.kingsrook.qbits.quicksearch.processes;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
+import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** Abstract base class for Quick Search indexing process steps.
- **
- ** Implements {@link BackendStep} but does NOT provide a run() implementation;
- ** that is left to concrete subclasses (BasepullIndexStep, FullReindexStep).
- **
- ** Provides protected helper methods for:
- ** - Accessing QBit config, OpenSearch client, and discovered tables
- ** - Ensuring quickSearchIndex rows exist for each table
- ** - Creating and completing quickSearchIndexRun records
- ** - Querying batches of records from source tables
+ ** Shared machinery for the indexing process steps: resolving the QBit's live
+ ** state, operational-table bookkeeping (index rows, run records, drift
+ ** detection, run-history purge), and primary-key-ordered paging of source
+ ** tables with display values.
  *******************************************************************************/
 public abstract class AbstractIndexingStep implements BackendStep
 {
    private static final QLogger LOG = QLogger.getLogger(AbstractIndexingStep.class);
 
+   public static final String STATUS_ACTIVE        = "ACTIVE";
+   public static final String STATUS_NEEDS_REINDEX = "NEEDS_REINDEX";
+   public static final String STATUS_REBUILDING    = "REBUILDING";
+   public static final String RUN_COMPLETED        = "COMPLETED";
+   public static final String RUN_FAILED           = "FAILED";
+   public static final String RUN_RUNNING          = "RUNNING";
+
+
+
+   /*******************************************************************************
+    ** Counts from indexing one table.
+    *******************************************************************************/
+   public record IndexCounts(Integer processed, Integer indexed, Integer errors, Integer skipped, String firstError)
+   {
+   }
+
 
 
    /*******************************************************************************
-    ** Return the active QuickSearchQBitConfig from context.
     **
-    ** @throws QException if config is null (QBit not initialized)
     *******************************************************************************/
    protected QuickSearchQBitConfig getConfig() throws QException
    {
       QuickSearchQBitConfig config = QuickSearchQBitContext.getConfig();
       if(config == null)
       {
-         throw new QException("QuickSearchQBitConfig is not initialized in QuickSearchQBitContext");
+         throw new QException("Quick Search QBit is not initialized (no config found in the QInstance)");
       }
       return (config);
    }
@@ -78,16 +102,14 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Return the active OpenSearch client cast from context.
     **
-    ** @throws QException if client is null (not initialized)
     *******************************************************************************/
    protected QuickSearchOpenSearchClient getClient() throws QException
    {
       QuickSearchOpenSearchClient client = QuickSearchQBitContext.getClient();
       if(client == null)
       {
-         throw new QException("QuickSearchOpenSearchClient is not initialized in QuickSearchQBitContext");
+         throw new QException("Quick Search OpenSearch client is not available");
       }
       return (client);
    }
@@ -95,27 +117,18 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Return the list of discovered table configs from context.
     **
-    ** Returns an empty list if discoveredTables is null.
     *******************************************************************************/
    protected List<QuickSearchableTableConfig> getDiscoveredTables()
    {
       List<QuickSearchableTableConfig> tables = QuickSearchQBitContext.getDiscoveredTables();
-      if(tables == null)
-      {
-         return (Collections.emptyList());
-      }
-      return (tables);
+      return (tables == null ? Collections.emptyList() : tables);
    }
 
 
 
    /*******************************************************************************
-    ** Return the table config for the given table name from context.
     **
-    ** Delegates to QuickSearchQBitContext.getTableConfig().
-    ** Returns null if not found.
     *******************************************************************************/
    protected QuickSearchableTableConfig getTableConfig(String tableName)
    {
@@ -125,97 +138,162 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Ensure a quickSearchIndex row exists for the given table.
-    **
-    ** Queries the quickSearchIndex table for a row matching tableName.
-    ** If no row is found, inserts one with:
-    ** - tableName
-    ** - enabled = tableConfig.getEnabledByDefault()
-    ** - basepullIntervalMinutes from tableConfig
-    ** - basepullTimestampField from tableConfig
-    ** - searchableFieldsJson built from tableConfig's fields/weights/labels
-    ** - status = "ACTIVE"
-    **
-    ** @param tableName   the name of the source table
-    ** @param tableConfig the configuration for that table
-    ** @throws QException if the query or insert fails
+    ** Make sure a quickSearchIndex row exists for the table. The table's unique
+    ** key on tableName makes a concurrent first insert fail, which is handled by
+    ** re-querying.
     *******************************************************************************/
    protected void ensureIndexRowExists(String tableName, QuickSearchableTableConfig tableConfig) throws QException
    {
-      //////////////////////////////////////////////////////////////////////////////
-      // Query for an existing index row                                          //
-      //////////////////////////////////////////////////////////////////////////////
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
-      queryInput.setFilter(new QQueryFilter()
-         .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
-
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      if(!queryOutput.getRecords().isEmpty())
+      if(queryIndexRow(tableName) != null)
       {
          return;
       }
 
-      //////////////////////////////////////////////////////////////////////////////
-      // Build and insert a new index row. Wrapped in try/catch to handle the   //
-      // race condition where another thread inserts between our check and our  //
-      // insert -- a duplicate key exception in that case is harmless.          //
-      //////////////////////////////////////////////////////////////////////////////
       try
       {
-         String searchableFieldsJson = buildSearchableFieldsJson(tableConfig);
-
          QRecord record = new QRecord()
             .withValue("tableName", tableName)
             .withValue("enabled", tableConfig.getEnabledByDefault())
             .withValue("basepullIntervalMinutes", tableConfig.getBasepullIntervalMinutes())
             .withValue("basepullTimestampField", tableConfig.getBasepullTimestampField())
-            .withValue("searchableFieldsJson", searchableFieldsJson)
-            .withValue("status", "ACTIVE");
+            .withValue("searchableFieldsJson", buildSearchableFieldsJson(tableConfig))
+            .withValue("realTimeErrorCount", 0)
+            .withValue("status", STATUS_ACTIVE);
 
          InsertInput insertInput = new InsertInput();
          insertInput.setTableName(getConfig().getQuickSearchIndexTableName());
          insertInput.setRecords(List.of(record));
+         InsertOutput insertOutput = new InsertAction().execute(insertInput);
 
-         new InsertAction().execute(insertInput);
+         if(insertOutput.getRecords().isEmpty() || CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
+         {
+            throw (new QException("Insert returned errors"));
+         }
       }
       catch(Exception e)
       {
-         ////////////////////////////////////////////////////////////////////
-         // Re-query to check if another thread inserted the row           //
-         ////////////////////////////////////////////////////////////////////
-         QueryOutput recheck = new QueryAction().execute(queryInput);
-         if(recheck.getRecords().isEmpty())
+         if(queryIndexRow(tableName) == null)
          {
             throw new QException("Failed to create QuickSearchIndex row for table [" + tableName + "]", e);
          }
-         LOG.debug("QuickSearchIndex row already exists (concurrent insert)", "tableName", tableName);
+         LOG.debug("QuickSearchIndex row already exists (concurrent insert)", logPair("tableName", tableName));
       }
    }
 
 
 
    /*******************************************************************************
-    ** Build a JSON array string representing the searchable fields configuration
-    ** for a table.
-    **
-    ** Each element looks like:
-    ** {"fieldName":"name","weight":1,"includeLabel":false}
-    **
-    ** @param tableConfig the table configuration containing field lists/maps
-    ** @return JSON string, e.g. [{"fieldName":"name","weight":1,"includeLabel":false}]
+    ** The quickSearchIndex row for a table, or null.
     *******************************************************************************/
-   private String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
+   protected QRecord queryIndexRow(String tableName) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
+      List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
+      return (CollectionUtils.nullSafeIsEmpty(rows) ? null : rows.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** The id of the quickSearchIndex row for a table.
+    *******************************************************************************/
+   protected Integer queryIndexId(String tableName) throws QException
+   {
+      QRecord row = queryIndexRow(tableName);
+      if(row == null)
+      {
+         throw new QException("No QuickSearchIndex row found for table: " + tableName);
+      }
+      return (row.getValueInteger("id"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an index row is enabled (null counts as enabled).
+    *******************************************************************************/
+   protected static boolean isRowEnabled(QRecord row)
+   {
+      Boolean enabled = row == null ? null : row.getValueBoolean("enabled");
+      return (enabled == null || enabled);
+   }
+
+
+
+   /*******************************************************************************
+    ** Compare the configured fields with what the row was created with; mark
+    ** the row NEEDS_REINDEX when they differ. Returns true when drift was found.
+    ** A REBUILDING row is left alone: the full reindex in progress records the
+    ** current fields, and the status tells the listener to capture deletes.
+    *******************************************************************************/
+   protected boolean detectDrift(QRecord row, QuickSearchableTableConfig tableConfig) throws QException
+   {
+      if(row == null || tableConfig == null)
+      {
+         return (false);
+      }
+
+      String current = buildSearchableFieldsJson(tableConfig);
+      if(!isUnmarkedDrift(row, current))
+      {
+         return (hasDrift(row, current));
+      }
+
+      ///////////////////////////////////////////////////////////////////////////
+      // the row may be a snapshot taken before a full reindex started or     //
+      // finished; decide on a fresh read, so REBUILDING is kept and fields a //
+      // reindex has since recorded are not flagged                           //
+      ///////////////////////////////////////////////////////////////////////////
+      QRecord latest = queryIndexRow(tableConfig.getTableName());
+      if(latest == null || !isUnmarkedDrift(latest, current))
+      {
+         return (latest != null && hasDrift(latest, current));
+      }
+
+      // a status change in the moment between this read and the update below is not guarded
+      LOG.warn("Searchable field configuration changed since the index row was created; marking NEEDS_REINDEX", logPair("tableName", tableConfig.getTableName()));
+      updateIndexRow(latest.getValueInteger("id"), Map.of("status", STATUS_NEEDS_REINDEX));
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the row records fields that differ from the configured ones.
+    *******************************************************************************/
+   private static boolean hasDrift(QRecord row, String current)
+   {
+      String recorded = row.getValueString("searchableFieldsJson");
+      return (recorded != null && !recorded.equals(current));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the row has drifted and is not yet NEEDS_REINDEX or REBUILDING.
+    *******************************************************************************/
+   private static boolean isUnmarkedDrift(QRecord row, String current)
+   {
+      String status = row.getValueString("status");
+      return (hasDrift(row, current) && !STATUS_NEEDS_REINDEX.equals(status) && !STATUS_REBUILDING.equals(status));
+   }
+
+
+
+   /*******************************************************************************
+    ** JSON array describing a table's searchable fields, weights and labels.
+    *******************************************************************************/
+   protected static String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
    {
       if(tableConfig.getSearchableFields() == null || tableConfig.getSearchableFields().isEmpty())
       {
          return ("[]");
       }
 
-      StringBuilder sb = new StringBuilder("[");
-      boolean first = true;
-
+      StringBuilder sb    = new StringBuilder("[");
+      boolean       first = true;
       for(String fieldName : tableConfig.getSearchableFields())
       {
          if(!first)
@@ -224,153 +302,352 @@ public abstract class AbstractIndexingStep implements BackendStep
          }
          first = false;
 
-         Integer weight = 1;
-         if(tableConfig.getFieldWeights() != null && tableConfig.getFieldWeights().containsKey(fieldName))
-         {
-            weight = tableConfig.getFieldWeights().get(fieldName);
-         }
+         Integer weight       = tableConfig.getFieldWeights() != null && tableConfig.getFieldWeights().containsKey(fieldName) ? tableConfig.getFieldWeights().get(fieldName) : 1;
+         Boolean includeLabel = tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName) ? tableConfig.getFieldIncludeLabels().get(fieldName) : false;
+         String  escapedName  = fieldName.replace("\\", "\\\\").replace("\"", "\\\"");
 
-         Boolean includeLabel = false;
-         if(tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName))
-         {
-            includeLabel = tableConfig.getFieldIncludeLabels().get(fieldName);
-         }
-
-         String escapedName = fieldName.replace("\\", "\\\\").replace("\"", "\\\"");
-
-         sb.append("{\"fieldName\":\"").append(escapedName).append("\"")
-           .append(",\"weight\":").append(weight)
-           .append(",\"includeLabel\":").append(includeLabel)
-           .append("}");
+         sb.append("{\"fieldName\":\"").append(escapedName).append("\",\"weight\":").append(weight).append(",\"includeLabel\":").append(includeLabel).append("}");
       }
-
-      sb.append("]");
-      return (sb.toString());
+      return (sb.append("]").toString());
    }
 
 
 
    /*******************************************************************************
-    ** Create a quickSearchIndexRun record and return it with its generated ID.
-    **
-    ** Inserts a new record with status=RUNNING and startTime=now, then reads
-    ** back the inserted record to capture the auto-generated primary key.
-    **
-    ** @param indexId  the ID of the parent quickSearchIndex row
-    ** @param runType  the type label for this run (e.g. "BASEPULL", "FULL_REINDEX")
-    ** @return a QuickSearchIndexRun entity with id populated
-    ** @throws QException if insert fails or no record is returned
+    ** Insert a RUNNING run record and return it with its id.
     *******************************************************************************/
    protected QuickSearchIndexRun createRunRecord(Integer indexId, String runType) throws QException
    {
-      QRecord record = new QRecord()
-         .withValue("quickSearchIndexId", indexId)
-         .withValue("runType", runType)
-         .withValue("status", "RUNNING")
-         .withValue("startTime", Instant.now());
-
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
-      insertInput.setRecords(List.of(record));
+      insertInput.setRecords(List.of(new QRecord()
+         .withValue("quickSearchIndexId", indexId)
+         .withValue("runType", runType)
+         .withValue("status", RUN_RUNNING)
+         .withValue("startTime", Instant.now())));
 
       InsertOutput insertOutput = new InsertAction().execute(insertInput);
-
       if(insertOutput.getRecords().isEmpty())
       {
          throw new QException("Failed to create quickSearchIndexRun record: no record returned from insert");
       }
 
-      QRecord insertedRecord = insertOutput.getRecords().get(0);
-
-      QuickSearchIndexRun run = new QuickSearchIndexRun()
-         .withId(insertedRecord.getValueInteger("id"))
-         .withQuickSearchIndexId(insertedRecord.getValueInteger("quickSearchIndexId"))
-         .withRunType(insertedRecord.getValueString("runType"))
-         .withStatus(insertedRecord.getValueString("status"))
-         .withStartTime(insertedRecord.getValueInstant("startTime"));
-      return (run);
+      QRecord inserted = insertOutput.getRecords().get(0);
+      return (new QuickSearchIndexRun()
+         .withId(inserted.getValueInteger("id"))
+         .withQuickSearchIndexId(inserted.getValueInteger("quickSearchIndexId"))
+         .withRunType(inserted.getValueString("runType"))
+         .withStatus(inserted.getValueString("status"))
+         .withStartTime(inserted.getValueInstant("startTime")));
    }
 
 
 
    /*******************************************************************************
-    ** Mark a quickSearchIndexRun as complete by updating its status, endTime,
-    ** record counts, and error information.
-    **
-    ** Wrapped in try/catch so that failures to update the run record do not
-    ** mask original exceptions when called from finally blocks.
-    **
-    ** @param run              the run record to update
-    ** @param status           the final status (e.g. "SUCCESS", "ERROR")
-    ** @param recordsProcessed number of records examined
-    ** @param recordsIndexed   number of records successfully indexed
-    ** @param errorCount       number of errors encountered
-    ** @param errorMessage     summary error message, or null
+    ** Finish a run record and mirror its status onto the index row. Never
+    ** throws, so it is safe from finally and catch blocks.
     *******************************************************************************/
-   protected void completeRunRecord(QuickSearchIndexRun run, String status, Integer recordsProcessed,
-      Integer recordsIndexed, Integer errorCount, String errorMessage)
+   protected void completeRunRecord(QuickSearchIndexRun run, String status, Integer recordsProcessed, Integer recordsIndexed, Integer errorCount, String errorMessage)
    {
+      run.withStatus(status);
+
       try
       {
-         QRecord record = new QRecord()
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+         updateInput.setRecords(List.of(new QRecord()
             .withValue("id", run.getId())
             .withValue("status", status)
             .withValue("endTime", Instant.now())
             .withValue("recordsProcessed", recordsProcessed)
             .withValue("recordsIndexed", recordsIndexed)
             .withValue("errorCount", errorCount)
-            .withValue("errorMessage", errorMessage);
-
-         UpdateInput updateInput = new UpdateInput();
-         updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
-         updateInput.setRecords(List.of(record));
-
+            .withValue("errorMessage", truncate(errorMessage, 4000))));
          new UpdateAction().execute(updateInput);
+
+         if(run.getQuickSearchIndexId() != null)
+         {
+            Map<String, Serializable> values = new HashMap<>();
+            values.put("lastRunStatus", run.getRunType() + " " + status);
+            values.put("lastErrorMessage", truncate(errorMessage, 4000));
+            updateIndexRow(run.getQuickSearchIndexId(), values);
+         }
       }
       catch(Exception e)
       {
-         LOG.warn("Failed to complete run record", e, "runId", run.getId(), "status", status);
+         LOG.warn("Failed to complete run record", e, logPair("runId", run.getId()), logPair("status", status));
       }
    }
 
 
 
    /*******************************************************************************
-    ** Query a batch of records from the given source table.
-    **
-    ** @param tableName the name of the source table to query
-    ** @param filter    the filter to apply (may be null for no filtering)
-    ** @param limit     maximum number of records to return
-    ** @param offset    number of records to skip (for pagination)
-    ** @return list of QRecord from the query
-    ** @throws QException if the query fails
+    ** Refresh a FULL_REINDEX run record's modifyDate: the heartbeat by which
+    ** other nodes tell a full reindex that is still going from one that
+    ** stopped (see FullReindexStep.recoverDeadRun). Nothing reads it for other
+    ** run types. Best effort: a failure is logged and never stops the run (one
+    ** judged stopped as a result fails safely before its swap).
     *******************************************************************************/
-   protected List<QRecord> querySourceTableBatch(String tableName, QQueryFilter filter, Integer limit, Integer offset) throws QException
+   protected void heartbeat(QuickSearchIndexRun run)
    {
-      QQueryFilter effectiveFilter = new QQueryFilter();
-      if(filter != null && filter.getCriteria() != null)
+      if(run == null || run.getId() == null || !FullReindexStep.RUN_TYPE.equals(run.getRunType()))
       {
-         for(QFilterCriteria criteria : filter.getCriteria())
-         {
-            effectiveFilter.withCriteria(criteria);
-         }
+         return;
       }
-      effectiveFilter.setLimit(limit);
-      effectiveFilter.setSkip(offset);
+
+      try
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
+         updateInput.setRecords(List.of(new QRecord().withValue("id", run.getId()).withValue("modifyDate", Instant.now())));
+         new UpdateAction().execute(updateInput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not refresh the full reindex heartbeat", e, logPair("runId", run.getId()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The record IDs, among the rows' recordIds, that exist in the source table.
+    *******************************************************************************/
+   Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
+   {
+      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
+      if(tableConfig == null)
+      {
+         return (Set.of());
+      }
+
+      String             primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
+      List<Serializable> recordIds       = new ArrayList<>();
+      rows.forEach(row -> recordIds.add(row.getValueString("recordId")));
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
-      queryInput.setFilter(effectiveFilter);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, recordIds)));
 
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      List<QRecord> records = queryOutput.getRecords();
-      if(records == null)
+      Set<String> existingIds = new HashSet<>();
+      for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
       {
-         return (new ArrayList<>());
+         existingIds.add(record.getValueString(primaryKeyField));
+      }
+      return (existingIds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update fields on a quickSearchIndex row.
+    *******************************************************************************/
+   protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
+   {
+      QRecord record = new QRecord().withValue("id", indexId);
+      for(Map.Entry<String, ? extends Serializable> entry : values.entrySet())
+      {
+         record.setValue(entry.getKey(), entry.getValue());
       }
 
-      return (records);
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
+      updateInput.setRecords(List.of(record));
+      new UpdateAction().execute(updateInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** Next page of source records in primary-key order after afterPrimaryKey
+    ** (null starts at the beginning), with the extra criteria applied and
+    ** display values generated.
+    *******************************************************************************/
+   protected List<QRecord> querySourceBatchAfter(String tableName, String primaryKeyField, Serializable afterPrimaryKey, List<QFilterCriteria> extraCriteria, Integer limit) throws QException
+   {
+      QQueryFilter filter = new QQueryFilter()
+         .withOrderBy(new QFilterOrderBy(primaryKeyField, true))
+         .withLimit(limit);
+
+      if(afterPrimaryKey != null)
+      {
+         filter.withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.GREATER_THAN, afterPrimaryKey));
+      }
+      for(QFilterCriteria criteria : CollectionUtils.nonNullList(extraCriteria))
+      {
+         filter.withCriteria(criteria);
+      }
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(filter);
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldTranslatePossibleValues(true);
+
+      return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Index every source record matching the criteria into targetIndex (null
+    ** for the alias), paging by primary key so each record is read once even
+    ** while rows are inserted or deleted. The run's heartbeat is refreshed
+    ** after every page.
+    *******************************************************************************/
+   protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
+   {
+      QuickSearchQBitConfig config          = getConfig();
+      String                tableName       = tableConfig.getTableName();
+      String                primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
+
+      int    processed  = 0;
+      int    indexed    = 0;
+      int    errors     = 0;
+      int    skipped    = 0;
+      String firstError = null;
+
+      Serializable lastPrimaryKey = null;
+      while(true)
+      {
+         List<QRecord> batch = querySourceBatchAfter(tableName, primaryKeyField, lastPrimaryKey, extraCriteria, config.getSourceBatchSize());
+         if(batch.isEmpty())
+         {
+            break;
+         }
+
+         List<OpenSearchDocument> documents = new ArrayList<>();
+         for(QRecord record : batch)
+         {
+            try
+            {
+               OpenSearchDocument document = IndexingUtils.buildDocument(record, tableConfig);
+               if(document != null)
+               {
+                  documents.add(document);
+               }
+            }
+            catch(Exception e)
+            {
+               errors++;
+               if(firstError == null)
+               {
+                  firstError = "Could not build document for " + tableName + ":" + record.getValueString(primaryKeyField) + ": " + e.getMessage();
+               }
+            }
+         }
+
+         if(!documents.isEmpty())
+         {
+            BulkIndexResult result = targetIndex == null ? client.indexDocuments(documents, config.getBulkBatchSize()) : client.indexDocuments(targetIndex, documents, config.getBulkBatchSize());
+            indexed += result.getSuccessCount();
+            errors += result.getFailureCount();
+            skipped += result.getSkippedCount();
+            if(firstError == null && !result.getErrors().isEmpty())
+            {
+               firstError = result.getErrors().get(0);
+            }
+         }
+
+         processed += batch.size();
+         lastPrimaryKey = batch.get(batch.size() - 1).getValue(primaryKeyField);
+         if(lastPrimaryKey == null)
+         {
+            throw new QException("Source record has no value in primary key field [" + primaryKeyField + "]");
+         }
+
+         heartbeat(run);
+      }
+
+      return (new IndexCounts(processed, indexed, errors, skipped, firstError));
+   }
+
+
+
+   /*******************************************************************************
+    ** Delete run records older than runHistoryRetentionDays. Never throws.
+    *******************************************************************************/
+   protected void purgeOldRuns()
+   {
+      Integer retentionDays = getConfigOrNull() == null ? null : getConfigOrNull().getRunHistoryRetentionDays();
+      if(retentionDays == null || retentionDays <= 0)
+      {
+         return;
+      }
+
+      Instant cutoff = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
+      purgeOlderThan(getConfigOrNull().getQuickSearchIndexRunTableName(), "startTime", cutoff, retentionDays);
+      purgeOlderThan(getConfigOrNull().applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME), "createDate", cutoff, retentionDays);
+      purgeOlderThan(getConfigOrNull().applyPrefix(QuickSearchValidationRun.TABLE_NAME), "createDate", cutoff, retentionDays);
+   }
+
+
+
+   /*******************************************************************************
+    ** Delete up to 5000 rows of a history table older than the cutoff. A table
+    ** not in the instance is skipped; failures are logged.
+    *******************************************************************************/
+   private void purgeOlderThan(String tableName, String dateField, Instant cutoff, Integer retentionDays)
+   {
+      try
+      {
+         if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
+         {
+            return;
+         }
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(dateField, QCriteriaOperator.LESS_THAN, cutoff)).withLimit(5000));
+         List<QRecord> old = new QueryAction().execute(queryInput).getRecords();
+         if(CollectionUtils.nullSafeIsEmpty(old))
+         {
+            return;
+         }
+
+         List<Serializable> ids = new ArrayList<>();
+         for(QRecord record : old)
+         {
+            ids.add(record.getValue("id"));
+         }
+
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(tableName);
+         deleteInput.setPrimaryKeys(ids);
+         new DeleteAction().execute(deleteInput);
+         LOG.info("Purged old Quick Search history records", logPair("tableName", tableName), logPair("count", ids.size()), logPair("olderThanDays", retentionDays));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Failed to purge old Quick Search history records", e, logPair("tableName", tableName));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The config, or null when it cannot be resolved (purging is best effort).
+    *******************************************************************************/
+   private QuickSearchQBitConfig getConfigOrNull()
+   {
+      try
+      {
+         return (getConfig());
+      }
+      catch(Exception e)
+      {
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected static String truncate(String s, int max)
+   {
+      return (s == null || s.length() <= max ? s : s.substring(0, max));
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStep.java

@@ -13,96 +13,117 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.kingsrook.qbits.quicksearch.processes;
 
 
+import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
 import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** Incremental basepull indexing process step.
+ ** Scheduled catch-up. For each enabled table that is due, re-indexes records
+ ** whose basepull timestamp is at or after the previous run's start minus the
+ ** configured overlap, paging by primary key. First recovers what a full
+ ** reindex that stopped without finishing left behind (so its captured deletes
+ ** are replayed in the same run) and replays failed real-time events, and
+ ** afterwards purges old run history.
  **
- ** Queries all enabled QuickSearchIndex rows, checks whether each is due for a
- ** basepull run, and for those that are, queries the source table for records
- ** modified since the last basepull, builds OpenSearch documents, and bulk-indexes
- ** them.
+ ** The watermark stored on the quickSearchIndex row is the run's start time,
+ ** captured before the first query, so rows modified during the run are read
+ ** again next time (re-indexing is idempotent). A table without a timestamp
+ ** field is left to the reconcile process.
  **
- ** Run records are created at the start of each per-table run and updated with
- ** final status and counts on completion.  Per-table failures are caught and
- ** logged so that one bad table does not prevent others from being processed.
+ ** One table's failure does not stop the others, but the step fails at the end
+ ** if any table failed, so the scheduler and run history show it.
  *******************************************************************************/
 public class BasepullIndexStep extends AbstractIndexingStep
 {
    private static final QLogger LOG = QLogger.getLogger(BasepullIndexStep.class);
 
+   public static final String RUN_TYPE          = "BASEPULL";
+   public static final int    MAX_REPLAY_ATTEMPTS = 10;
+
 
 
    /*******************************************************************************
-    ** Main entry point called by the QQQ process engine.
     **
-    ** 1. For each discovered table, ensure a quickSearchIndex row exists.
-    ** 2. Query all enabled quickSearchIndex rows.
-    ** 3. For each enabled row, check isDueForBasepull and run basepullIndex.
-    **
-    ** @param input  process step input (not directly used)
-    ** @param output process step output (not directly used)
-    ** @throws QException if an unrecoverable error occurs (per-table errors are caught)
     *******************************************************************************/
    @Override
    public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
    {
-      ////////////////////////////////////////////////////
-      // 1. Ensure index rows exist for all tables      //
-      ////////////////////////////////////////////////////
-      List<QuickSearchableTableConfig> discoveredTables = getDiscoveredTables();
-      for(QuickSearchableTableConfig tableConfig : discoveredTables)
+      List<String> failures = new ArrayList<>();
+
+      try
+      {
+         new FullReindexStep().recoverDeadRun(true);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Recovering from a stopped full reindex did not complete", e);
+         failures.add("full reindex recovery: " + e.getMessage());
+      }
+
+      try
+      {
+         replayFailedEvents(output);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Replaying failed events did not complete", e);
+         failures.add("failed-event replay: " + e.getMessage());
+      }
+
+      for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
       {
          ensureIndexRowExists(tableConfig.getTableName(), tableConfig);
       }
 
-      ////////////////////////////////////////////////////
-      // 2. Query all enabled quickSearchIndex rows     //
-      ////////////////////////////////////////////////////
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
-      queryInput.setFilter(new QQueryFilter()
-         .withCriteria(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)));
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)));
+      List<QRecord> indexRecords = new QueryAction().execute(queryInput).getRecords();
 
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-      List<QRecord> indexRecords = queryOutput.getRecords();
-
-      if(indexRecords == null || indexRecords.isEmpty())
+      if(CollectionUtils.nullSafeIsEmpty(indexRecords))
       {
          LOG.info("No enabled quickSearchIndex rows found; nothing to basepull");
+         purgeOldRuns();
          return;
       }
 
-      ////////////////////////////////////////////////////
-      // 3. Process each enabled index row              //
-      ////////////////////////////////////////////////////
+      Integer tablesIndexed = 0;
       for(QRecord indexRecord : indexRecords)
       {
          QuickSearchIndex index = new QuickSearchIndex()
@@ -113,188 +134,319 @@ public class BasepullIndexStep extends AbstractIndexingStep
             .withBasepullTimestampField(indexRecord.getValueString("basepullTimestampField"))
             .withLastBasepullTime(indexRecord.getValueInstant("lastBasepullTime"));
 
+         QuickSearchableTableConfig tableConfig = getTableConfig(index.getTableName());
+         detectDrift(indexRecord, tableConfig);
+
          if(!isDueForBasepull(index))
          {
-            LOG.info("Index not due for basepull; skipping",
-               "tableName", index.getTableName(),
-               "lastBasepullTime", index.getLastBasepullTime());
+            LOG.debug("Index not due for basepull; skipping", logPair("tableName", index.getTableName()), logPair("lastBasepullTime", index.getLastBasepullTime()));
             continue;
          }
 
          try
          {
             basepullIndex(index);
+            tablesIndexed++;
          }
          catch(Exception e)
          {
-            LOG.warn("Error during basepull for table; continuing with next table",
-               e, "tableName", index.getTableName());
+            LOG.warn("Error during basepull for table; continuing with next table", e, logPair("tableName", index.getTableName()));
+            failures.add(index.getTableName() + ": " + e.getMessage());
          }
       }
+
+      purgeOldRuns();
+      output.addValue("tablesIndexed", tablesIndexed);
+
+      if(!failures.isEmpty())
+      {
+         throw (new QException("Basepull failed for " + failures.size() + " item(s): " + String.join("; ", failures)));
+      }
    }
 
 
 
    /*******************************************************************************
-    ** Determine whether this index is due for a basepull run.
-    **
-    ** Returns true when lastBasepullTime is null (never run), or when the elapsed
-    ** time since the last run has exceeded the configured interval.
-    **
-    ** @param index the QuickSearchIndex row to evaluate
-    ** @return true if a basepull should be executed for this index
+    ** Due when never run, or when the interval has elapsed since the last run.
     *******************************************************************************/
    boolean isDueForBasepull(QuickSearchIndex index)
    {
       Instant lastBasepullTime = index.getLastBasepullTime();
-
-      if(lastBasepullTime == null)
+      Integer intervalMinutes  = index.getBasepullIntervalMinutes();
+      if(lastBasepullTime == null || intervalMinutes == null)
       {
          return (true);
       }
-
-      Integer intervalMinutes = index.getBasepullIntervalMinutes();
-      if(intervalMinutes == null)
-      {
-         return (true);
-      }
-
       return (lastBasepullTime.plus(intervalMinutes, ChronoUnit.MINUTES).isBefore(Instant.now()));
    }
 
 
 
    /*******************************************************************************
-    ** Execute a basepull indexing run for one QuickSearchIndex row.
-    **
-    ** Steps:
-    ** 1. Resolve table config and client.
-    ** 2. Create a run record (status=RUNNING).
-    ** 3. Build an optional QQueryFilter using lastBasepullTime.
-    ** 4. Paginate through the source table, building and indexing documents.
-    ** 5. Update lastBasepullTime on the QuickSearchIndex row.
-    ** 6. Complete the run record with final status and counts.
-    **
-    ** @param index the QuickSearchIndex row driving this run
-    ** @throws QException if an error occurs that cannot be handled locally
+    ** One table's basepull run.
     *******************************************************************************/
    void basepullIndex(QuickSearchIndex index) throws QException
    {
-      String                   tableName   = index.getTableName();
-      QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
-      QuickSearchOpenSearchClient client    = getClient();
-      QuickSearchQBitConfig    config      = getConfig();
+      String                      tableName   = index.getTableName();
+      QuickSearchableTableConfig  tableConfig = getTableConfig(tableName);
+      QuickSearchOpenSearchClient client      = getClient();
+      QuickSearchQBitConfig       config      = getConfig();
 
-      QuickSearchIndexRun run = createRunRecord(index.getId(), "BASEPULL");
+      QuickSearchIndexRun run = createRunRecord(index.getId(), RUN_TYPE);
 
-      Integer totalProcessed = 0;
-      Integer totalIndexed   = 0;
-      Integer totalErrors    = 0;
-      String  errorMessage   = null;
+      if(tableConfig == null)
+      {
+         String message = "Table is no longer configured for Quick Search; disable or remove its quickSearchIndex row";
+         completeRunRecord(run, RUN_FAILED, 0, 0, 1, message);
+         throw (new QException(tableName + ": " + message));
+      }
+
+      Instant runStart = Instant.now();
 
       try
       {
-         ///////////////////////////////////////////////////////////////////
-         // Build filter: timestamp field > lastBasepullTime (if set)     //
-         ///////////////////////////////////////////////////////////////////
-         QQueryFilter filter = null;
-
-         if(index.getLastBasepullTime() != null && tableConfig != null
-            && tableConfig.getBasepullTimestampField() != null)
+         String timestampField = tableConfig.getBasepullTimestampField();
+         if(timestampField == null && index.getLastBasepullTime() != null)
          {
-            filter = new QQueryFilter()
-               .withCriteria(new QFilterCriteria(
-                  tableConfig.getBasepullTimestampField(),
-                  QCriteriaOperator.GREATER_THAN,
-                  index.getLastBasepullTime()));
+            ////////////////////////////////////////////////////////////////////////
+            // no change-detection field: only the first run indexes everything;  //
+            // later changes are caught by reconcile                              //
+            ////////////////////////////////////////////////////////////////////////
+            completeRunRecord(run, RUN_COMPLETED, 0, 0, 0, null);
+            return;
          }
 
-         Integer sourceBatchSize = config.getSourceBatchSize();
-         Integer bulkBatchSize   = config.getBulkBatchSize();
-
-         ///////////////////////////////////////////////////////////////////
-         // Paginate through the source table                             //
-         ///////////////////////////////////////////////////////////////////
-         int offset = 0;
-
-         while(true)
+         List<QFilterCriteria> criteria = new ArrayList<>();
+         if(timestampField != null && index.getLastBasepullTime() != null)
          {
-            List<QRecord> batch = querySourceTableBatch(tableName, filter, sourceBatchSize, offset);
-
-            if(batch.isEmpty())
-            {
-               break;
-            }
-
-            ////////////////////////////////////////////////////
-            // Build documents for this batch                 //
-            ////////////////////////////////////////////////////
-            List<OpenSearchDocument> documents = new ArrayList<>();
-
-            for(QRecord record : batch)
-            {
-               if(tableConfig != null)
-               {
-                  OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
-                  documents.add(doc);
-               }
-            }
-
-            documents.removeIf(doc -> doc == null);
-
-            ////////////////////////////////////////////////////
-            // Bulk-index the batch                           //
-            ////////////////////////////////////////////////////
-            if(!documents.isEmpty())
-            {
-               BulkIndexResult result = client.indexDocuments(documents, bulkBatchSize);
-               totalIndexed   = totalIndexed + result.getSuccessCount();
-               totalErrors    = totalErrors + result.getFailureCount();
-
-               if(!result.getErrors().isEmpty() && errorMessage == null)
-               {
-                  errorMessage = result.getErrors().get(0);
-               }
-            }
-
-            totalProcessed = totalProcessed + batch.size();
-            offset         = offset + batch.size();
+            int     overlap = config.getBasepullOverlapSeconds() == null ? 0 : config.getBasepullOverlapSeconds();
+            Instant since   = index.getLastBasepullTime().minusSeconds(overlap);
+            criteria.add(new QFilterCriteria(timestampField, QCriteriaOperator.GREATER_THAN_OR_EQUALS, since));
          }
 
-         ////////////////////////////////////////////////////
-         // Update lastBasepullTime on the index row       //
-         // only when no indexing errors occurred           //
-         ////////////////////////////////////////////////////
-         if(totalErrors.equals(0))
+         IndexCounts counts = indexAllRecords(client, tableConfig, criteria, null, run);
+
+         if(counts.errors() == 0)
          {
-            QRecord updateRecord = new QRecord()
-               .withValue("id", index.getId())
-               .withValue("lastBasepullTime", Instant.now());
-
-            UpdateInput updateInput = new UpdateInput();
-            updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
-            updateInput.setRecords(List.of(updateRecord));
-
-            new UpdateAction().execute(updateInput);
+            Map<String, Serializable> values = new HashMap<>();
+            values.put("lastBasepullTime", runStart);
+            Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
+            if(documentCount != null)
+            {
+               values.put("documentCount", documentCount);
+            }
+            updateIndexRow(index.getId(), values);
          }
 
-         ////////////////////////////////////////////////////
-         // Complete the run record                        //
-         ////////////////////////////////////////////////////
-         String finalStatus = (totalErrors > 0) ? "FAILED" : "COMPLETED";
-         completeRunRecord(run, finalStatus, totalProcessed, totalIndexed, totalErrors, errorMessage);
+         String status = counts.errors() > 0 ? RUN_FAILED : RUN_COMPLETED;
+         completeRunRecord(run, status, counts.processed(), counts.indexed(), counts.errors(), counts.firstError());
 
-         LOG.info("Basepull complete",
-            "tableName", tableName,
-            "totalProcessed", totalProcessed,
-            "totalIndexed", totalIndexed,
-            "totalErrors", totalErrors);
+         LOG.info("Basepull complete", logPair("tableName", tableName), logPair("processed", counts.processed()), logPair("indexed", counts.indexed()),
+            logPair("skippedStale", counts.skipped()), logPair("errors", counts.errors()));
+
+         if(counts.errors() > 0)
+         {
+            throw (new QException("Basepull indexed " + counts.indexed() + " of " + counts.processed() + " records for table [" + tableName + "]; first error: " + counts.firstError()));
+         }
+      }
+      catch(QException e)
+      {
+         if(!RUN_FAILED.equals(run.getStatus()))
+         {
+            completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
+            run.withStatus(RUN_FAILED);
+         }
+         throw e;
       }
       catch(Exception e)
       {
-         LOG.warn("Basepull failed for table", e, "tableName", tableName);
-         completeRunRecord(run, "FAILED", totalProcessed, totalIndexed, totalErrors, e.getMessage());
-         throw e;
+         completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
+         throw (new QException("Basepull failed for table [" + tableName + "]: " + e.getMessage(), e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Replay PENDING failed real-time events: re-read and index records for
+    ** INDEX events, delete documents for DELETE events. A DELETE whose record
+    ** exists again (re-created with the same key since) keeps its document and
+    ** counts as replayed. Successes are removed; failures bump attempts and
+    ** become EXHAUSTED at the limit.
+    *******************************************************************************/
+   void replayFailedEvents(RunBackendStepOutput output) throws QException
+   {
+      QuickSearchQBitConfig config          = getConfig();
+      String                failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(failedEventTable);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_PENDING))
+         .withOrderBy(new QFilterOrderBy("id", true))
+         .withLimit(config.getSourceBatchSize()));
+      List<QRecord> events = new QueryAction().execute(queryInput).getRecords();
+
+      if(CollectionUtils.nullSafeIsEmpty(events))
+      {
+         output.addValue("failedEventsReplayed", 0);
+         return;
+      }
+
+      QuickSearchOpenSearchClient client = getClient();
+
+      Map<String, List<QRecord>> indexByTable  = new LinkedHashMap<>();
+      Map<String, List<QRecord>> deleteByTable = new LinkedHashMap<>();
+      for(QRecord event : events)
+      {
+         Map<String, List<QRecord>> target = "DELETE".equals(event.getValueString("action")) ? deleteByTable : indexByTable;
+         target.computeIfAbsent(event.getValueString("tableName"), k -> new ArrayList<>()).add(event);
+      }
+
+      List<Serializable> succeeded = new ArrayList<>();
+      List<QRecord>      failed    = new ArrayList<>();
+
+      for(Map.Entry<String, List<QRecord>> entry : deleteByTable.entrySet())
+      {
+         List<QRecord> toDelete = entry.getValue();
+         try
+         {
+            Set<String> existingIds = queryExistingRecordIds(entry.getKey(), entry.getValue());
+            toDelete = new ArrayList<>();
+            for(QRecord event : entry.getValue())
+            {
+               if(existingIds.contains(event.getValueString("recordId")))
+               {
+                  succeeded.add(event.getValue("id"));
+               }
+               else
+               {
+                  toDelete.add(event);
+               }
+            }
+
+            if(!toDelete.isEmpty())
+            {
+               List<String> documentIds = new ArrayList<>();
+               for(QRecord event : toDelete)
+               {
+                  documentIds.add(OpenSearchDocument.buildDocumentId(entry.getKey(), event.getValueString("recordId")));
+               }
+               BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
+               markReplayOutcome(toDelete, result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
+            }
+         }
+         catch(Exception e)
+         {
+            markReplayOutcome(toDelete, false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
+         }
+      }
+
+      for(Map.Entry<String, List<QRecord>> entry : indexByTable.entrySet())
+      {
+         String                     tableName   = entry.getKey();
+         QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
+         if(tableConfig == null)
+         {
+            markReplayOutcome(entry.getValue(), false, List.of("table is no longer configured"), succeeded, failed);
+            continue;
+         }
+
+         try
+         {
+            String             primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
+            List<Serializable> ids             = new ArrayList<>();
+            for(QRecord event : entry.getValue())
+            {
+               ids.add(event.getValueString("recordId"));
+            }
+
+            QueryInput sourceQuery = new QueryInput();
+            sourceQuery.setTableName(tableName);
+            sourceQuery.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, ids)));
+            sourceQuery.setShouldGenerateDisplayValues(true);
+            sourceQuery.setShouldTranslatePossibleValues(true);
+            List<QRecord> records = CollectionUtils.nonNullList(new QueryAction().execute(sourceQuery).getRecords());
+
+            List<OpenSearchDocument> documents = new ArrayList<>();
+            List<String>             foundIds  = new ArrayList<>();
+            for(QRecord record : records)
+            {
+               OpenSearchDocument document = IndexingUtils.buildDocument(record, tableConfig);
+               if(document != null)
+               {
+                  documents.add(document);
+                  foundIds.add(document.getRecordId());
+               }
+            }
+
+            ///////////////////////////////////////////////////////////////////////
+            // a record deleted since the event was recorded has no document to //
+            // index; remove any stale document for it instead                  //
+            ///////////////////////////////////////////////////////////////////////
+            List<String> goneDocumentIds = new ArrayList<>();
+            for(Serializable id : ids)
+            {
+               if(!foundIds.contains(String.valueOf(id)))
+               {
+                  goneDocumentIds.add(OpenSearchDocument.buildDocumentId(tableName, String.valueOf(id)));
+               }
+            }
+
+            BulkIndexResult result = client.indexDocuments(documents, config.getBulkBatchSize());
+            if(!goneDocumentIds.isEmpty())
+            {
+               BulkIndexResult deleteResult = client.deleteDocuments(goneDocumentIds, config.getBulkBatchSize());
+               deleteResult.getErrors().forEach(result::addFailure);
+            }
+            markReplayOutcome(entry.getValue(), result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
+         }
+         catch(Exception e)
+         {
+            markReplayOutcome(entry.getValue(), false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
+         }
+      }
+
+      if(!succeeded.isEmpty())
+      {
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(failedEventTable);
+         deleteInput.setPrimaryKeys(succeeded);
+         new DeleteAction().execute(deleteInput);
+      }
+
+      if(!failed.isEmpty())
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(failedEventTable);
+         updateInput.setRecords(failed);
+         new UpdateAction().execute(updateInput);
+      }
+
+      LOG.info("Replayed failed real-time events", logPair("succeeded", succeeded.size()), logPair("stillFailing", failed.size()));
+      output.addValue("failedEventsReplayed", succeeded.size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void markReplayOutcome(List<QRecord> events, boolean success, List<String> errors, List<Serializable> succeeded, List<QRecord> failed)
+   {
+      for(QRecord event : events)
+      {
+         if(success)
+         {
+            succeeded.add(event.getValue("id"));
+            continue;
+         }
+
+         int attempts = (event.getValueInteger("attempts") == null ? 0 : event.getValueInteger("attempts")) + 1;
+         failed.add(new QRecord()
+            .withValue("id", event.getValue("id"))
+            .withValue("attempts", attempts)
+            .withValue("errorMessage", truncate(errors == null || errors.isEmpty() ? "replay failed" : errors.get(0), 4000))
+            .withValue("status", attempts >= MAX_REPLAY_ATTEMPTS ? QuickSearchFailedEvent.STATUS_EXHAUSTED : QuickSearchFailedEvent.STATUS_PENDING));
       }
    }
 

src/main/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtils.java

@@ -18,26 +18,38 @@ package com.kingsrook.qbits.quicksearch.processes;
 
 
 import java.time.Instant;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
 import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** Static utility class for text normalization and OpenSearch document building.
+ ** Text normalization and OpenSearch document building.
  **
- ** Provides helpers used during the indexing pipeline to normalize field values
- ** into searchable text and to assemble OpenSearchDocument instances from
- ** QRecord data.
+ ** Indexed text uses a record's display value when the query generated one
+ ** (possible-value labels, formatted dates) and the raw value otherwise. Every
+ ** value is stored as a string (fieldValues are mapped as text), truncated to
+ ** the table's maxFieldLength when set.
  *******************************************************************************/
 public class IndexingUtils
 {
+   private static final QLogger LOG = QLogger.getLogger(IndexingUtils.class);
+
+
 
    /*******************************************************************************
-    ** Private constructor to prevent instantiation of this utility class.
+    ** Private constructor to prevent instantiation.
     *******************************************************************************/
    private IndexingUtils()
    {
@@ -46,14 +58,7 @@ public class IndexingUtils
 
 
    /*******************************************************************************
-    ** Normalize text for search indexing.
-    **
-    ** Converts to lowercase, trims leading/trailing whitespace, and collapses
-    ** runs of internal whitespace to a single space. Returns empty string for
-    ** null or blank input.
-    **
-    ** @param text the raw text to normalize
-    ** @return the normalized text, never null
+    ** Trim and collapse whitespace. Lower-casing is left to the analyzers.
     *******************************************************************************/
    public static String normalizeSearchText(String text)
    {
@@ -61,39 +66,39 @@ public class IndexingUtils
       {
          return ("");
       }
-
-      return (text.toLowerCase().trim().replaceAll("\\s+", " "));
+      return (text.trim().replaceAll("\\s+", " "));
    }
 
 
 
    /*******************************************************************************
-    ** Build a concatenated searchable text string from selected record fields.
-    **
-    ** Iterates over each field name in {@code fields}, retrieves the string value
-    ** from the record, and appends it to the result. Fields with a null or blank
-    ** value are skipped. When {@code includeLabels} maps a field name to
-    ** {@code true}, the value is prefixed with {@code "fieldName: "}.
-    **
-    ** @param record        the QRecord to read field values from
-    ** @param fields        ordered list of field names to include
-    ** @param includeLabels map of field name to whether its label should be prefixed
-    ** @return the concatenated text, trimmed; empty string if fields is null or empty
+    ** Concatenate the display values of the given fields. A field whose
+    ** includeLabels entry is true is prefixed with the field's label (or name).
     *******************************************************************************/
    public static String buildSearchableText(QRecord record, List<String> fields, Map<String, Boolean> includeLabels)
+   {
+      return (buildSearchableText(record, fields, includeLabels, null, null));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String buildSearchableText(QRecord record, List<String> fields, Map<String, Boolean> includeLabels, QTableMetaData table, Integer maxFieldLength)
    {
       if(fields == null || fields.isEmpty())
       {
          return ("");
       }
 
-      StringBuilder sb = new StringBuilder();
+      Map<String, Boolean> labels = includeLabels == null ? Collections.emptyMap() : includeLabels;
+      StringBuilder        sb     = new StringBuilder();
 
       for(String fieldName : fields)
       {
-         String value = record.getValueString(fieldName);
-
-         if(value == null || value.isBlank())
+         String value = displayValue(record, fieldName, maxFieldLength);
+         if(value == null)
          {
             continue;
          }
@@ -103,9 +108,11 @@ public class IndexingUtils
             sb.append(" ");
          }
 
-         if(Boolean.TRUE.equals(includeLabels.get(fieldName)))
+         if(Boolean.TRUE.equals(labels.get(fieldName)))
          {
-            sb.append(fieldName).append(": ");
+            String label = (table != null && table.getFields().containsKey(fieldName) && StringUtils.hasContent(table.getField(fieldName).getLabel()))
+               ? table.getField(fieldName).getLabel() : fieldName;
+            sb.append(label).append(": ");
          }
 
          sb.append(value);
@@ -117,42 +124,105 @@ public class IndexingUtils
 
 
    /*******************************************************************************
-    ** Build an OpenSearchDocument from a QRecord.
-    **
-    ** Delegates to {@link #buildSearchableText} for the searchable text, reads the
-    ** record ID from {@code primaryKeyField}, and populates {@code fieldValues} with
-    ** only those entries where the record's value is non-null.
-    **
-    ** @param record          the source QRecord
-    ** @param tableName       the QQQ table name used as the document's sourceTable
-    ** @param primaryKeyField the field name whose value becomes the document's recordId
-    ** @param fields          ordered list of field names for searchable text
-    ** @param fieldWeights    map of field name to search weight (reserved for future use)
-    ** @param includeLabels   map of field name to whether its label should be prefixed
-    ** @return a fully populated OpenSearchDocument with indexedAt set to now
+    ** Display value when present, else the raw value as a string; blank is null.
+    *******************************************************************************/
+   static String displayValue(QRecord record, String fieldName, Integer maxFieldLength)
+   {
+      String value = record.getDisplayValue(fieldName);
+      if(!StringUtils.hasContent(value))
+      {
+         value = record.getValueString(fieldName);
+      }
+      if(!StringUtils.hasContent(value))
+      {
+         return (null);
+      }
+      if(maxFieldLength != null && value.length() > maxFieldLength)
+      {
+         value = value.substring(0, maxFieldLength);
+      }
+      return (value);
+   }
+
+
+
+   /*******************************************************************************
+    ** Build a document from explicit field lists (no table config).
     *******************************************************************************/
    public static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List<String> fields, Map<String, Integer> fieldWeights, Map<String, Boolean> includeLabels)
    {
-      String searchableText = buildSearchableText(record, fields, includeLabels);
+      return (buildDocument(record, tableName, primaryKeyField, fields, includeLabels, lookupTable(tableName), null, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build a document for a configured table: display values, label fallback,
+    ** truncation, and an external version from the basepull timestamp field.
+    ** Returns null when the record has no primary key.
+    *******************************************************************************/
+   public static OpenSearchDocument buildDocument(QRecord record, QuickSearchableTableConfig tableConfig)
+   {
+      QTableMetaData     table = lookupTable(tableConfig.getTableName());
+      OpenSearchDocument doc   = buildDocument(record, tableConfig.getTableName(), tableConfig.getPrimaryKeyField(), tableConfig.getSearchableFields(),
+         tableConfig.getFieldIncludeLabels(), table, tableConfig.getMaxFieldLength(), tableConfig.getBasepullTimestampField());
+
+      if(doc != null && tableConfig.getRecordLabelFormat() != null && tableConfig.getRecordLabelFields() != null)
+      {
+         doc.withRecordLabel(formatLabel(record, tableConfig, doc.getRecordLabel()));
+      }
+
+      return (doc);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List<String> fields, Map<String, Boolean> includeLabels,
+      QTableMetaData table, Integer maxFieldLength, String timestampField)
+   {
       String recordId = record.getValueString(primaryKeyField);
       if(recordId == null)
       {
          return (null);
       }
-      String recordLabel = record.getRecordLabel();
 
       Map<String, Object> fieldValues = new HashMap<>();
-
-      if(fields != null)
+      for(String fieldName : fields == null ? List.<String>of() : fields)
       {
-         for(String fieldName : fields)
+         String value = displayValue(record, fieldName, maxFieldLength);
+         if(value != null)
          {
-            Object value = record.getValue(fieldName);
+            fieldValues.put(fieldName, value);
+         }
+      }
 
-            if(value != null)
-            {
-               fieldValues.put(fieldName, value);
-            }
+      String recordLabel = record.getRecordLabel();
+      if(!StringUtils.hasContent(recordLabel) && table != null)
+      {
+         try
+         {
+            recordLabel = QValueFormatter.formatRecordLabel(table, record);
+         }
+         catch(Exception e)
+         {
+            LOG.debug("Could not format record label", e, logPair("tableName", tableName));
+         }
+      }
+
+      Long version = null;
+      if(timestampField != null)
+      {
+         try
+         {
+            Instant timestamp = record.getValueInstant(timestampField);
+            version = timestamp == null ? null : timestamp.toEpochMilli();
+         }
+         catch(Exception e)
+         {
+            version = null;
          }
       }
 
@@ -160,47 +230,45 @@ public class IndexingUtils
          .withSourceTable(tableName)
          .withRecordId(recordId)
          .withRecordLabel(recordLabel)
-         .withSearchableText(searchableText)
+         .withSearchableText(buildSearchableText(record, fields, includeLabels, table, maxFieldLength))
          .withIndexedAt(Instant.now())
-         .withFieldValues(fieldValues));
+         .withFieldValues(fieldValues)
+         .withVersion(version));
    }
 
 
 
    /*******************************************************************************
-    ** Build an OpenSearchDocument from a QRecord using a QuickSearchableTableConfig.
-    **
-    ** Delegates to the existing buildDocument overload for core document
-    ** construction. When recordLabelFormat is set on the table config, overrides
-    ** the record label by formatting the specified field values using
-    ** String.format. Falls back to QRecord.getRecordLabel() when no format is set.
-    **
-    ** @param record      the source QRecord
-    ** @param tableConfig the table configuration containing fields, weights, and label format
-    ** @return a fully populated OpenSearchDocument, or null if the primary key is null
+    ** Apply recordLabelFormat; a bad format or null value never breaks indexing.
     *******************************************************************************/
-   public static OpenSearchDocument buildDocument(QRecord record, QuickSearchableTableConfig tableConfig)
+   private static String formatLabel(QRecord record, QuickSearchableTableConfig tableConfig, String fallback)
    {
-      OpenSearchDocument doc = buildDocument(
-         record,
-         tableConfig.getTableName(),
-         tableConfig.getPrimaryKeyField(),
-         tableConfig.getSearchableFields(),
-         tableConfig.getFieldWeights(),
-         tableConfig.getFieldIncludeLabels() != null ? tableConfig.getFieldIncludeLabels() : java.util.Collections.emptyMap());
-
-      if(doc != null && tableConfig.getRecordLabelFormat() != null && tableConfig.getRecordLabelFields() != null)
+      try
       {
          Object[] labelValues = new Object[tableConfig.getRecordLabelFields().size()];
-         for(int i = 0; i < tableConfig.getRecordLabelFields().size(); i++)
+         for(int i = 0; i < labelValues.length; i++)
          {
-            Object v = record.getValue(tableConfig.getRecordLabelFields().get(i));
-            labelValues[i] = v != null ? v : "";
+            String value = displayValue(record, tableConfig.getRecordLabelFields().get(i), null);
+            labelValues[i] = value == null ? "" : value;
          }
-         doc.withRecordLabel(String.format(tableConfig.getRecordLabelFormat(), labelValues));
+         return (String.format(tableConfig.getRecordLabelFormat(), labelValues));
       }
+      catch(Exception e)
+      {
+         LOG.warn("recordLabelFormat could not be applied; using default label", e, logPair("tableName", tableConfig.getTableName()));
+         return (fallback);
+      }
+   }
 
-      return (doc);
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData lookupTable(String tableName)
+   {
+      QInstance qInstance = QContext.getQInstance();
+      return (qInstance == null || tableName == null ? null : qInstance.getTable(tableName));
    }
 
 }

src/main/java/com/kingsrook/qbits/quicksearch/processes/ReconcileIndexStep.java

@@ -0,0 +1,221 @@
+/*
+ * Copyright 2024 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.processes;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
+import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Reconcile the index with its source tables without a search blackout.
+ **
+ ** For each enabled table (or the one named by the optional "tableName"
+ ** input): re-index every source record in primary-key order, then, if every
+ ** document indexed, refresh and delete the table's documents this run did not
+ ** touch (indexedAt before the run started). When no table filter is given,
+ ** documents of tables that are no longer configured are removed too.
+ **
+ ** Used by the full reindex of a single table as well, with its own run type.
+ *******************************************************************************/
+public class ReconcileIndexStep extends AbstractIndexingStep
+{
+   private static final QLogger LOG = QLogger.getLogger(ReconcileIndexStep.class);
+
+   public static final String RUN_TYPE = "RECONCILE";
+
+
+
+   /*******************************************************************************
+    ** Totals from reconciling one table.
+    *******************************************************************************/
+   public record ReconcileCounts(Integer recordsIndexed, Long documentsRemoved)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+   {
+      String tableNameFilter = input.getValueString("tableName");
+      boolean allTables      = tableNameFilter == null || tableNameFilter.isBlank();
+
+      Integer recordsIndexed   = 0;
+      Long    documentsRemoved = 0L;
+      Integer tablesSkipped    = 0;
+
+      for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
+      {
+         if(!allTables && !tableNameFilter.equals(tableConfig.getTableName()))
+         {
+            continue;
+         }
+
+         ensureIndexRowExists(tableConfig.getTableName(), tableConfig);
+         QRecord row = queryIndexRow(tableConfig.getTableName());
+         if(!isRowEnabled(row))
+         {
+            LOG.info("Table is disabled; skipping reconcile", logPair("tableName", tableConfig.getTableName()));
+            tablesSkipped++;
+            continue;
+         }
+
+         ReconcileCounts counts = reconcileTable(tableConfig, RUN_TYPE);
+         recordsIndexed   = recordsIndexed + counts.recordsIndexed();
+         documentsRemoved = documentsRemoved + counts.documentsRemoved();
+      }
+
+      Long orphanDocumentsRemoved = 0L;
+      if(allTables)
+      {
+         List<String> configured = new ArrayList<>();
+         getDiscoveredTables().forEach(tc -> configured.add(tc.getTableName()));
+         orphanDocumentsRemoved = getClient().deleteDocumentsForTablesNotIn(configured);
+         if(orphanDocumentsRemoved > 0)
+         {
+            LOG.info("Removed documents of tables that are no longer configured", logPair("count", orphanDocumentsRemoved));
+         }
+      }
+
+      purgeOldRuns();
+
+      output.addValue("recordsIndexed", recordsIndexed);
+      output.addValue("documentsRemoved", documentsRemoved);
+      output.addValue("orphanDocumentsRemoved", orphanDocumentsRemoved);
+      output.addValue("tablesSkipped", tablesSkipped);
+   }
+
+
+
+   /*******************************************************************************
+    ** Reconcile one table, recording the run with the given run type.
+    *******************************************************************************/
+   ReconcileCounts reconcileTable(QuickSearchableTableConfig tableConfig, String runType) throws QException
+   {
+      String                      tableName = tableConfig.getTableName();
+      QuickSearchOpenSearchClient client    = getClient();
+
+      Integer             indexId = queryIndexId(tableName);
+      QuickSearchIndexRun run     = createRunRecord(indexId, runType);
+
+      try
+      {
+         Instant     reconcileStart = Instant.now();
+         IndexCounts counts         = indexAllRecords(client, tableConfig, List.of(), null, run);
+
+         Long documentsRemoved = 0L;
+         if(counts.errors() == 0)
+         {
+            client.refreshIndex();
+            documentsRemoved = client.deleteDocumentsIndexedBefore(tableName, reconcileStart);
+
+            Map<String, Serializable> values = new HashMap<>();
+            values.put("lastReconcileTime", Instant.now());
+            values.put("lastBasepullTime", reconcileStart);
+            values.put("recordCount", counts.processed());
+            Integer documentCount = safeCount(client, tableName);
+            if(documentCount != null)
+            {
+               values.put("documentCount", documentCount);
+            }
+            values.put("searchableFieldsJson", buildSearchableFieldsJson(tableConfig));
+            ////////////////////////////////////////////////////////////////////
+            // a full reindex running alongside owns the status until it ends //
+            ////////////////////////////////////////////////////////////////////
+            if(!STATUS_REBUILDING.equals(queryIndexRow(tableName).getValueString("status")))
+            {
+               values.put("status", STATUS_ACTIVE);
+            }
+            if(ReconcileIndexStep.RUN_TYPE.equals(runType) == false)
+            {
+               values.put("lastFullReindexTime", Instant.now());
+            }
+            updateIndexRow(indexId, values);
+         }
+         else
+         {
+            LOG.warn("Reconcile had indexing errors; stale documents were not removed", logPair("tableName", tableName), logPair("errorCount", counts.errors()));
+         }
+
+         String status = counts.errors() == 0 ? RUN_COMPLETED : RUN_FAILED;
+         completeRunRecord(run, status, counts.processed(), counts.indexed(), counts.errors(), counts.firstError());
+
+         LOG.info("Reconcile complete", logPair("tableName", tableName), logPair("processed", counts.processed()), logPair("indexed", counts.indexed()),
+            logPair("skippedStale", counts.skipped()), logPair("errors", counts.errors()), logPair("documentsRemoved", documentsRemoved));
+
+         if(counts.errors() > 0)
+         {
+            throw (new QException("Reconcile indexed " + counts.indexed() + " of " + counts.processed() + " records for table [" + tableName + "]; first error: " + counts.firstError()));
+         }
+
+         return (new ReconcileCounts(counts.indexed(), documentsRemoved));
+      }
+      catch(QException e)
+      {
+         if(!RUN_FAILED.equals(run.getStatus()))
+         {
+            completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
+            run.withStatus(RUN_FAILED);
+         }
+         throw e;
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Reconcile failed", e, logPair("tableName", tableName));
+         completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
+         throw new QException("Reconcile failed for table [" + tableName + "]: " + e.getMessage(), e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Number of documents indexed for the table, or null when the count fails
+    ** (logged), so callers keep the previous documentCount.
+    *******************************************************************************/
+   static Integer safeCount(QuickSearchOpenSearchClient client, String tableName)
+   {
+      try
+      {
+         Long count = client.countDocumentsForTable(tableName);
+         return (count == null ? null : count.intValue());
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not count indexed documents for table; keeping the previous documentCount", e, logPair("tableName", tableName));
+         return (null);
+      }
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/processes/diagnostics/AbstractDiagnosticToolStep.java

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.processes.diagnostics;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
+
+
+/*******************************************************************************
+ ** Base of the diagnostic tool steps. Each tool's process has the steps
+ ** PREPARE (backend), FORM or CONFIRM (frontend: fields and an HTML block),
+ ** RUN (backend) and, for one-shot tools, RESULT (frontend: HTML). One step
+ ** class serves a tool's backend steps, switching on the step name.
+ ** Looping tools (playground, analyzer, inspect) end RUN by pointing the
+ ** process back at PREPARE, so the next step is FORM again with the results
+ ** below the fields: change the input, run again, without restarting.
+ ** HTML goes in the process value "{step}.html", which the HTML component of
+ ** that frontend step renders.
+ *******************************************************************************/
+public abstract class AbstractDiagnosticToolStep implements BackendStep
+{
+   public static final String STEP_PREPARE = "prepare";
+   public static final String STEP_FORM    = "form";
+   public static final String STEP_CONFIRM = "confirm";
+   public static final String STEP_RUN     = "run";
+   public static final String STEP_RESULT  = "result";
+
+
+
+   /*******************************************************************************
+    ** One record a tool works on.
+    *******************************************************************************/
+   public record Target(String tableName, String recordId)
+   {
+      /*******************************************************************************
+       ** "table:recordId"
+       *******************************************************************************/
+      @Override
+      public String toString()
+      {
+         return (tableName + ":" + recordId);
+      }
+
+
+
+      /*******************************************************************************
+       ** Parse "table:recordId" (null when malformed).
+       *******************************************************************************/
+      public static Target parse(String value)
+      {
+         int colon = value == null ? -1 : value.indexOf(':');
+         return (colon <= 0 || colon == value.length() - 1 ? null : new Target(value.substring(0, colon), value.substring(colon + 1)));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Set the HTML shown by a frontend step.
+    *******************************************************************************/
+   protected static void setHtml(RunBackendStepOutput output, String frontendStepName, String html)
+   {
+      output.addValue(frontendStepName + ".html", html);
+   }
+
+
+
+   /*******************************************************************************
+    ** After RUN, show FORM again (the step after PREPARE).
+    *******************************************************************************/
+   protected static void loopBackToForm(RunBackendStepOutput output)
+   {
+      output.setOverrideLastStepName(STEP_PREPARE);
+   }
+
+
+
+   /*******************************************************************************
+    ** A failure shown in the step's HTML instead of ending the process, so the
+    ** user can change the input and run again.
+    *******************************************************************************/
+   protected static String failureHtml(String what, Exception e)
+   {
+      return (new DiagnosticsHtml().open().status(DiagnosticsHtml.Level.FAIL, what + ": " + e.getMessage()).close().toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** The records the process was started on, as targets: for a source table,
+    ** its primary keys; for the failed event table, each row's table and
+    ** record id (only tables Quick Search indexes). Read from the records the
+    ** backend loaded into the process state (never from client values), so
+    ** every step works on exactly the records the user selected and may read.
+    ** At most max; more is a user-facing error.
+    *******************************************************************************/
+   protected static List<Target> targetsFromInput(RunBackendStepInput input, QuickSearchRuntime runtime, int max) throws QException
+   {
+      QProcessMetaData process   = QContext.getQInstance().getProcess(input.getProcessName());
+      String           tableName = process == null ? null : process.getTableName();
+      QTableMetaData   table     = tableName == null ? null : QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw (new QUserFacingException("This tool runs on records of a table."));
+      }
+
+      List<QRecord> records = CollectionUtils.nonNullList(input.getRecords());
+      if(records.isEmpty())
+      {
+         throw (new QUserFacingException("Select at least one record."));
+      }
+      if(records.size() > max)
+      {
+         throw (new QUserFacingException("Select at most " + max + " records (" + records.size() + "+ selected)."));
+      }
+
+      boolean      failedEvents = tableName.equals(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
+      Set<Target>  targets      = new LinkedHashSet<>();
+      for(QRecord record : records)
+      {
+         String targetTable = failedEvents ? record.getValueString("tableName") : tableName;
+         String recordId    = failedEvents ? record.getValueString("recordId") : record.getValueString(table.getPrimaryKeyField());
+         if(StringUtils.hasContent(targetTable) && StringUtils.hasContent(recordId) && runtime.getTableConfig(targetTable) != null)
+         {
+            targets.add(new Target(targetTable, recordId));
+         }
+      }
+      return (new ArrayList<>(targets));
+   }
+
+}

src/main/java/com/kingsrook/qbits/quicksearch/processes/diagnostics/AnalyzeTextStep.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.quicksearch.processes.diagnostics;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
+import com.kingsrook.qbits.quicksearch.diagnostics.AnalyzerTester;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
+import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
+import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
+
+
+/*******************************************************************************
+ ** Steps of the Test Search Analyzer process (a table action on Quick Search
+ ** Index): run the index's analyzers on text, and check a query against it.
+ ** Loops back to the form.
+ *******************************************************************************/
+public class AnalyzeTextStep extends AbstractDiagnosticToolStep
+{
+   public static final String FIELD_TEXT  = "analyzeText";
+   public static final String FIELD_QUERY = "analyzeQuery";
+
+
+
+   /*******************************************************************************
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+   {
+      QuickSearchRuntime runtime = DiagnosticsSupport.requireRuntime();
+      if(STEP_PREPARE.equals(input.getStepName()))
+      {
+         setHtml(output, STEP_FORM, new DiagnosticsHtml().open()
+            .paragraph("Enter text as it would be indexed (a field value) and, optionally, a search term. You get the tokens each of the index's analyzers produces "
+               + "and, with a term, whether every term token is among the text's indexed tokens: that is how a document's searchable text matches. Nothing is changed.")
+            .close().toString());
+         return;
+      }
+
+      loopBackToForm(output);
+      long start = System.currentTimeMillis();
+      try
+      {
+         DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_ANALYZE_TEXT);
+      }
+      catch(QUserFacingException e)
+      {
+         setHtml(output, STEP_FORM, failureHtml("Not run", e));
+         return;
+      }
+
+      String query = input.getValueString(FIELD_QUERY);
+      try
+      {
+         setHtml(output, STEP_FORM, AnalyzerTester.render(new AnalyzerTester().analyze(runtime, input.getValueString(FIELD_TEXT), query)));
+         DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_ANALYZE_TEXT, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, null, query, start, null, null);
+      }
+      catch(Exception e)
+      {
+         setHtml(output, STEP_FORM, failureHtml("Analyzing failed (is OpenSearch reachable and the index created?)", e));
+         DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_ANALYZE_TEXT, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, null, query, start, e, null);
+      }
+   }
+
+}