Compare commits

..
Author SHA1 Message Date
james.maes ceac89aee3 Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#92) from deps/align-gitea into develop
build / maven (push) Successful in 8m59s
maven / build (push) Successful in 8m59s
2026-11-07 18:31:07 +00:00
james.maesandClaude Opus 5.5 75a8d55da2 build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)
build / maven (push) Successful in 13m48s
maven / build (push) Successful in 9m43s
pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.15 -> 4.1.0-RC.26
pom.xml: removed Central snapshot repository
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 18:04:20 +00:00
james.maes 75d6f7f489 Merge pull request 'ci: pin shared workflow to v1' (#91) from ci/pin-v1-develop into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (push) Successful in 12s
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Successful in 11s
QRun-IO/ci-policy: native / native-result (push) Successful in 9s
build / maven (push) Successful in 10m38s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 10m38s
2026-11-06 16:09:45 +00:00
james.maesandClaude Opus 5.5 2e4cbd1537 ci: pin shared workflow to v1
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (pull_request) Successful in 8s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 6s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 6s
build / maven (push) Successful in 7m48s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 7m48s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 14:12:44 +00:00
james.maes 31536a2472 Merge pull request 'ci: use shared ci-workflows Maven workflow' (#90) from ci/ci-workflows into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 4s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 6s
build / maven (push) Successful in 7m24s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow d20a578c7d818bc7ca32a2ac2c675da4e4f27385] (push) Successful in 6s
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 7m24s
2026-11-05 23:17:40 +00:00
CI 1f39169f46 ci: rerun shared workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Failing after 3s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 1s
build / maven (push) Successful in 7m27s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 7m27s
2026-11-05 16:26:16 -06:00
james.maesandClaude Opus 5.5 35bf7d7a46 ci: use shared ci-workflows Maven workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Failing after 1s
QRun-IO/ci-policy: native / native-result (pull_request) Failing after 1s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 2s
build / maven (push) Failing after 2m54s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Failing after 2m54s
Replace the platform-ci release callers with one caller of
KofTwentyTwo/ci-workflows maven.yaml (qqq-orb port).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:07:34 +00:00
forge-recovery e3aa05e3f9 ci: enroll shared Gitea lifecycle
qrun-release / postpublish (push) Successful in 5m18s
Shared Java Maven CI with Docker / verify (push) Successful in 8m16s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 3s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 19s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 22s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 8s
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 44m32s
Shared Java Maven CI with Docker / gate (push) Successful in 15s
qrun-release / release (push) Successful in 44m46s
Shared QRun source version / version (push) Successful in 5m18s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 7m48s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 20s
2026-11-02 22:20:37 +00:00
forge-recovery 2fc62b584e ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 9s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 27s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 21m52s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 6s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 8m11s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 15s
2026-11-02 21:33:51 +00:00
forge-recoveryandCodex 32644c0d8f ci: enroll shared Gitea release-candidate lifecycle
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 6s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 6s
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 16s
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 7m0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 23s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 1h7m45s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 20:18:51 +00:00
219a028162 ci: add the isolated postpublish job to the QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 15s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
Shared Java Maven CI with Docker / verify (push) Canceled after 0s
qrun-release / postpublish (push) Canceled after 0s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 15s
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
Shared Java Maven CI with Docker / gate (push) Canceled after 0s
qrun-release / release (push) Canceled after 2h31m26s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 8m6s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 10s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 3h8m5s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 15:52:03 +00:00
8627c5d144 ci: move the QRun release caller to the thin v2 contract
Shared Java Maven CI with Docker / secrets (push) Successful in 24s
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 4s
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Canceled after 29m21s
Shared Java Maven CI with Docker / verify (push) Canceled after 0s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 22s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Canceled after 0s
Shared Java Maven CI with Docker / gate (push) Canceled after 0s
qrun-release / release (push) Canceled after 29m33s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 06:19:54 +00:00
james.maesandCodex bd50e1f2d9 ci: adopt managed orb 0.8 patch track (#81)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 15:27:44 -05:00
james.maesandCodex d42ccc178c ci: adopt shared orb 0.7.18 (#80)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:46:04 -05:00
james.maesandCodex 98c545ad4f ci: adopt shared orb 0.7.17 (#79)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:10:11 -05:00
james.maesandCodex 3372dc9f68 ci: centralize QBit source scanning (#72)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 10:44:14 -05:00
james.maesandCodex d67dec8448 ci: adopt central orb 0.7.12 and stable gates (#71)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 08:57:06 -05:00
james.maesandCodex fd34b1c6c8 ci: centralize rc release completion in qqq orb (#70)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 02:29:01 -05:00
james.maesandCodex 00148af810 ci: adopt shared orb 0.7.11 (#69)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 01:03:35 -05:00
james.maes 15a564258b Merge pull request #68 from QRun-IO/feature/central-ci
ci: adopt verified central QQQ orb 0.7.1
2026-10-20 18:47:45 -05:00
james.maesandCodex 4437720546 ci: adopt verified central QQQ orb 0.7.2
Co-Authored-By: Codex <noreply@openai.com>
2026-10-20 18:42:39 -05:00
james.maes e4bf215310 Merge pull request #67 from QRun-IO/feature/single-node-replicas
fix(opensearch): size index replicas to the cluster so a single node stays green
2026-10-16 11:51:24 -05:00
james.maesandClaude Opus 5.5 d76391f6a4 fix(opensearch): size index replicas to the cluster so a single node stays green
The index was created without number_of_replicas, so OpenSearch's default
of 1 left an unassignable replica on single-node clusters: the alias was
yellow and quick_search.opensearch reported DEGRADED.

- New config indexReplicas (default null = automatic:
  min(maxAutoReplicas, data nodes - 1)) and maxAutoReplicas (default 1).
  Zero config gives 0 replicas on one node and 1 on larger clusters.
- New physical indexes get the target; at startup existing indexes are
  reconciled with PUT /<index>/_settings when the value differs
  (idempotent, logged, failure tolerant, never blocks startup). Indexes
  with auto_expand_replicas and OpenSearch Serverless are left alone.
- Status snapshot and quick_search.opensearch health details gain
  unassignedReplicaShards, targetReplicas and replicaSettingsFailed.
  Yellow on a single node with every primary active is UP with a note;
  yellow with an inactive primary or on multiple nodes stays DEGRADED,
  red stays DOWN.
- Unit tests (stub server), health/status tests and a Testcontainers
  integration test; OpenAPI fragment regenerated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 11:43:13 -05:00
james.maes fd0322b8dc Merge pull request #65 from QRun-IO/ci/release-rc-secrets-inherit
ci(release-rc): pass secrets so the shared workflow can sign the RC tag
2026-10-16 08:04:16 -05:00
james.maes 3efff6de95 Merge pull request #66 from QRun-IO/fix/system-session-instance
fix: startup and diagnostics system sessions carry the instance
2026-10-16 01:29:33 -05:00
james.maesandClaude Opus 5.5 d4d5029691 fix: build startup and diagnostics system sessions with the instance
primeConfig created QSystemUserSession() before QContext.init, so
on a thread without a context (startup on [main]) the session had no
instance and no all-access keys, and kof22 logged "QInstance was not
supplied" once per pod. DiagnosticEventRecorder did the same.
Both now pass the instance.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 01:24:24 -05:00
james.maesandClaude Opus 5.5 3de57f534b ci(release-rc): pass secrets to the shared workflow so it can sign the RC tag
QRun-IO/.github#2 adds a tag-and-release job that signs vX with the
QRUN_TAG_SIGNING_KEY secret after Maven Central confirms the artifact.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 16:21:59 -05:00
james.maes 8ef66f6efe Merge pull request #63 from QRun-IO/feature/ui-api-1.6
feat(ui-api): extension for QQQ UI API 1.6.0 (operationIds, BasicErrorResponseV1, QQQ 4.1.0-RC.15)
2026-10-15 16:13:09 -05:00
james.maes 1705c73b06 Merge remote-tracking branch 'origin/develop' into feature/ui-api-1.6
# Conflicts:
#	CHANGELOG.md
#	pom.xml
2026-10-15 16:01:17 -05:00
james.maes ae5c00c589 Merge pull request #64 from QRun-IO/docs/rc11-release-note-correction
docs(changelog): release note correction for 1.1.0-RC.11
2026-10-15 14:34:50 -05:00
james.maesandClaude Opus 5.5 87768d3dd3 docs(changelog): release note correction for 1.1.0-RC.11
The v1.1.0-RC.11 tag points at 6028a66, but the RC.11 artifact on Maven
Central was built from 2e2501d (#61 only). Cluster caches (#62) first ship
in 1.1.0-RC.12.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 14:34:35 -05:00
james.maes 832c628298 Merge pull request #62 from QRun-IO/feature/cluster-caches
Clear the Quick Search config cache on every node (ClusterCaches)
2026-10-15 14:11:18 -05:00
james.maesandClaude Opus 5.5 b4c7ba4697 build: QQQ 4.1.0-RC.14 (ships ClusterCaches)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 14:05:04 -05:00
james.maesandClaude Opus 5.5 6af1362668 chore: target QQQ 4.1.0-RC.15 (UI API 1.6.0)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:27:06 -05:00
james.maesandClaude Opus 5.5 af9eefe311 feat(cluster): clear the Quick Search config cache on every node
Register the runtime config cache (config store snapshot and settings,
per-table enabled flags) with QQQ ClusterCaches as
qbit-quick-search.config. invalidateConfig() now clears locally and
publishes a cluster invalidation, so a config edit on one node reaches
every node at once; the existing TTLs stay as the safety net. The
warn-once table set in QuickSearchFieldRules is registered node-local.

Builds against QQQ 4.1.0-SNAPSHOT (qqq.snapshot.version) until the QQQ
RC with ClusterCaches is published; a release must pin that RC.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:10:35 -05:00
james.maesandClaude Opus 5.5 a9550d82b4 feat(ui-api): extension for QQQ UI API 1.6.0 (operationIds, BasicErrorResponseV1, QQQ 4.1.0-RC.14)
Regenerated against UI API 1.6.0: every operation has an operationId and
the standard error responses (BasicErrorResponseV1: error, code, message,
fieldErrors, traceId); the extension's own error schema is kept but
deprecated; hand-written error bodies go through QJavalinUtils so they carry
code/message/traceId. Extension MINOR bumped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 11:48:12 -05:00
james.maes 3f7e67df4b Merge pull request #61 from QRun-IO/feature/quiet-index-bookkeeping-audits
fix(audit): do not record-audit indexing bookkeeping
2026-10-15 07:52:26 -05:00
james.maesandClaude Opus 5.5 a6887f7ea2 fix(audit): do not record-audit indexing bookkeeping
Basepull, full reindex and real-time indexing update runtime state on the
quick_search_index row (status, lastBasepullTime, documentCount, last run,
realTimeErrorCount), and every startup re-records the priming outcome on the
settings row. Each of those writes produced a FIELD audit ("Record was Edited
during process: Update search index"), about 2,800 audit rows in 6 hours on
kof22 production. These writes now set omitDmlAudit; admin edits to the
index, field and settings tables keep their audits. Tests prove both sides.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 07:47:23 -05:00
james.maes bce0f7ff53 Merge pull request #60 from QRun-IO/fix/help-bundle-version
fix(help): help bundle version comes from the build, not a hardcoded SNAPSHOT
2026-10-14 22:07:01 -05:00
james.maesandClaude Opus 5.5 e9e3e59225 fix(help): the help bundle carries the artifact version (Maven-filtered) instead of a hardcoded SNAPSHOT
On release branches the bundle's version and the producer's differed, so the
bundle test that asserts they match failed, and released bundles recorded
SNAPSHOT as their provenance. help/*.yaml is now filtered:
version: ${project.version}.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 22:00:05 -05:00
james.maes 1d1b8a4b7b Merge pull request #57 from QRun-IO/feature/health-checks
feat(health): register quick search checks with core central health
2026-10-14 21:27:00 -05:00
james.maesandClaude Opus 5.5 2fc2485fd8 Merge origin/develop into feature/health-checks; pin QQQ 4.1.0-RC.8
RC.8 ships the central health kernel (QRun-IO/qqq#1207).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 21:18:17 -05:00
james.maes e4d6d279f2 Merge pull request #59 from QRun-IO/feature/help-bundle
feat(help): qbit-quick-search help bundle via QQQ help bundles, with coverage gate
2026-10-14 20:45:30 -05:00
james.maesandClaude Opus 5.5 be27514a58 feat(help): ship the qbit-quick-search help bundle through QQQ help bundles, with coverage gate
Host-neutral help (deployment-specific facts removed), registered with
HelpBundles, gated by HelpBundleAssertions. QQQ 4.1.0-RC.7.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 20:37:08 -05:00
james.maes 4503eddb6e Merge pull request #58 from QRun-IO/feature/tables-populated-defaults
Every quick_search table explains itself; guardrail and primer tests
2026-10-14 17:19:51 -05:00
james.maesandClaude Opus 5.5 d8cab84387 feat(help): empty-state help for every quick_search table, guardrail and primer tests
Adds help/quick-search.yaml (QQQ help bundle) applied by QuickSearchHelp, a
tables guardrail test, and a seeded-defaults test proving the primer fills
the config tables on a fresh backend.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 16:52:39 -05:00
james.maes 912015f2fc Merge pull request #56 from QRun-IO/feature/typeahead-suggest
feat(api): typeahead suggest endpoint for the global search box
2026-10-14 13:54:22 -05:00
james.maes 45d063adfc Merge pull request #40 from QRun-IO/feature/record-search-details
feat(search): scores, highlights, totals and paging for core record search
2026-10-14 13:54:14 -05:00
james.maesandClaude Opus 5.5 2cde433c35 docs: quick search health checks
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 13:39:30 -05:00
james.maesandClaude Opus 5.5 6111d2fdf2 feat(health): register quick search checks with core central health
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 13:39:29 -05:00
james.maesandClaude Opus 5.5 99e7e0806a feat(health): deadline-bounded status reads for health checks
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 13:39:28 -05:00
james.maesandClaude Sonnet 5.5 f9d80c1cda docs: typeahead suggest endpoint (README, CHANGELOG, API host notes)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-14 13:14:53 -05:00
james.maesandClaude Sonnet 5.5 06bc7da07d feat(api): typeahead suggest operation on the quick-search UI API extension
GET /qqq/v1/qbit/quick-search/suggest: prefix matches grouped per table with label and subtitle, reusing the permission-, lock- and PHI-aware QuickSearchAction.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-14 13:14:52 -05:00
james.maesandClaude Sonnet 5.5 2fd3a1ad38 feat(search): single-pass option for bounded per-table searches
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-14 13:14:51 -05:00
james.maesandClaude Sonnet 5.5 fe57092891 Merge origin/develop into feature/record-search-details
Resolve the record-search provider (keep score/highlight/total hits and the failure diagnostics), CHANGELOG and README; requires QQQ 4.1.0-RC.4+ (develop is on RC.6).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-14 13:01:39 -05:00
james.maes 4753174e1b Merge pull request #55 from QRun-IO/feature/exclude-classified-fields
feat(index): never index PII/PHI-classified or secured fields (secured data §15)
2026-10-14 11:23:44 -05:00
james.maes 50bf08f32a Merge remote-tracking branch 'origin/develop' into feature/exclude-classified-fields 2026-10-14 11:13:34 -05:00
james.maesandClaude Opus 5.5 024cda7b08 feat(index): never index PII/PHI-classified or secured fields
Secured-data spec section 15: Quick Search must never put classified
values in the shared search index, snippets or results.

QuickSearchFieldRules decides it in one place on QQQ 4.1.0-RC.6:
isProtected is true for fields classified PII or PHI (field, else the
table's classification), fields with a field security lock, sensitive
fields and masked/password types. Such fields are never indexable;
explicit configs drop them with a warning (never the value); a table
whose primary key is protected (for example a PHI table) is not indexed
at all. Record labels built from protected fields fall back, the
version field is skipped when protected, and results from tables with
protected fields are rebuilt from allowed values only, so documents
indexed before this change cannot leak through labels or snippets.
Tables with only harmless non-indexable fields (booleans, blobs) keep
their labels and are not result-filtered.

A canary test proves a PHI and a PII marker never reach a captured
index request, result or snippet. RC.7 follow-up: also honour core
ClassifiedValues.isClassified (qqq#1182) and FieldAccessHelper lock
tiers (qqq#1172).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 11:13:22 -05:00
james.maes 81f9917e10 Merge pull request #54 from QRun-IO/fix/realtime-error-count
fix(status): count real-time indexing failures (realTimeErrorCount)
2026-10-14 10:50:25 -05:00
james.maes bb556d9b0c Merge pull request #52 from QRun-IO/feature/orb-0.6.9
ci: qqq-orb 0.6.9 (refresh SNAPSHOT dependencies)
2026-10-14 10:46:23 -05:00
james.maesandClaude Opus 5.5 1a4a29fa2b fix(status): count real-time indexing failures
realTimeErrorCount was only ever written as 0, so the status screen,
diagnostics and the status API always said there were no real-time
errors. When the change listener records failed events it now adds
their number to the table's count (after commit, serialized in the JVM,
never failing the user's write). A successful basepull or full reindex
resets it; the field's help text says so.

Fixes #53

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 10:42:35 -05:00
james.maesandClaude Opus 5.5 7bf8f6c804 ci: qqq-orb 0.6.9 (refresh SNAPSHOT dependencies)
Moves qqq-orb@0.6.8 to qqq-orb@0.6.9, which passes --update-snapshots so cached SNAPSHOT dependencies are re-checked on every build (QRun-IO/qqq-orb#22).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 10:07:38 -05:00
james.maes dbaf14a9b5 Merge pull request #51 from QRun-IO/feature/arch-status-api
feat(api): QuickSearchStatus API + spec'd status endpoint (#50)
2026-10-14 07:22:23 -05:00
james.maesandClaude Opus 5.5 2ccf082493 build: qbit-build-parent 2.1.0 with architecture guardrails enforced
Enforce the qbit-bom#10 guardrails (G1, G3, G4, G7) and run tests in
strict raw-route mode (G2). All pass on the released QQQ 4.1.0-RC.6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 07:15:19 -05:00
james.maesandClaude Opus 5.5 a099a1d1b5 feat(api): QuickSearchStatus API and spec'd GET /qbit/quick-search/status
Hosts no longer need QuickSearchOpenSearchClient or a copy of this qbit's
diagnostics. The new public api package holds:
- QuickSearchStatus, an immutable snapshot: overall state, cluster
  reachability, per-table index counts, backlog, last sync, last error
  and checkedAt;
- QuickSearchStatusService.get(), bounded and cached for five seconds,
  built on the existing diagnostics;
- QuickSearchApi, with the app name and home href, so hosts link to the
  Quick Search home instead of duplicating it.

The same data is served as UI API extension operation
GET /qqq/v1/qbit/quick-search/status (getQuickSearchStatus), gated by the
admin permission. The fragment is committed and checked by a contract
test. The core baseline becomes 4.1.0-RC.6.

Closes #50 (architecture review QRun-IO/qqq#1150, finding F10).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:55:28 -05:00
james.maes e6f6b66b93 Merge pull request #49 from QRun-IO/feature/release-rc-automation
ci: automate RC releases; hotfix/* pushes no longer publish
2026-10-13 14:43:30 -05:00
james.maesandClaude Opus 5.5 7719f63db2 ci: automate RC releases; hotfix/* pushes no longer publish
Removes publish_hotfix_release: a hotfix/* branch push published a GA
release to Maven Central with no tag or review (QRun-IO/qbit-cdn#12,
QRun-IO/qqq#1121). hotfix/* branches now run test_only; a hotfix
release publishes from a reviewed vX.Y.Z tag through publish_release.
Adds .github/workflows/release-rc.yml (reusable QRun-IO/.github
qbit-release-rc): a manual dispatch, or merging a develop PR labelled
release:rc, merges develop into the release line and the release/* push
runs the existing CircleCI RC publish.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 14:37:28 -05:00
james.maes 923aec8e78 Merge pull request #47 from QRun-IO/feature/quick-search-data-driven-config
feat: data-driven Quick Search config, valid Add Search Index, global pause, cancel (1.1.0-RC.3)
2026-10-12 16:10:32 -05:00
james.maes c9f0614e07 Merge branch 'develop' into feature/quick-search-data-driven-config 2026-10-12 16:05:38 -05:00
james.maes 0804c29e9a Merge pull request #48 from QRun-IO/feature/ga-only-release-tags
ci: publish only GA tags (vX.Y.Z), never RC tags
2026-10-12 16:02:23 -05:00
james.maesandClaude Opus 5.5 2719445ef2 ci: publish only GA tags (vX.Y.Z), never RC tags
An RC tag matched /v.*/ and the orb republished it as the next RC.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 15:58:40 -05:00
james.maesandClaude Opus 5.5 ee76e1e3c7 fix: record screens of the Quick Search tables, process inputs in Next
- quick_search_index_run and quick_search_failed_event are system
  history: Create and Edit are off (Delete stays), with named sections,
  record labels and pickers (run type and status, event status and
  action, and the table picker for failed events' tableName).
- The index record lists its Recent Runs (read-only child list).
- quick_search_validation_run.reportJson uses a JSON code editor.
- Basepull, Full Reindex and Reconcile are table actions on Quick Search
  Index, not app (nav) children. Their table input is searchTableName
  (tableName still accepted; a Quick Search table name is ignored), so
  running them from the table no longer limits them to quick_search_index.
- Process input steps carry HELP_TEXT and EDIT_FORM components, so the
  Next UI renders their fields; Add Search Index's table input is
  searchTableName for the same reason.
- The Search Diagnostics widget shows the pause, priming and pending
  changes as banners; screenshots under docs/images/data-driven-config.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 15:55:24 -05:00
james.maesandClaude Opus 5.5 27a170f208 feat: data-driven Quick Search config, valid create screen, global pause, cancel
The searchable tables and fields are now standard QQQ records:
quick_search_index (the registry; new source and ownerEdited columns) and
the new quick_search_index_field (weight, role, analyzer, snippet and
label flags). Indexing, search, the record-search provider and the
diagnostics read the effective config from them through
QuickSearchConfigStore (30 s cache, dropped by the tables' customizers),
so changes need no redeploy.

Code (@QuickSearchable, searchableTables) only seeds the records:
QuickSearchConfigPrimer adds missing tables and fields at startup and
queues their builds, updates rows nobody edited, never overwrites rows an
admin edited (ownerEdited) and never re-adds what an admin removed
(quick_search_config_tombstone). Basepull builds queued tables
(PENDING_BUILD, BUILDING) and recreates a missing index; an empty index
is rebuilt. Without the config tables the code config is used as before.

The generic Create and Delete of quick_search_index are replaced by
streamed-ETL processes with review steps: Add Search Index (eligible
readable tables only, estimated record count, default fields, sensitive
and hidden fields never indexed), Apply Search Index Changes (previews
field changes, then rebuilds), Disable / Remove Search Index (delete or
keep documents), Cancel Index Run (cooperative flag between batches; runs
end CANCELLED with counts) and Pause / Resume Indexing (global pause in
quick_search_settings; search keeps serving; changes are deferred and
replayed). Field roles shape documents (TITLE label, SUBTITLE, DISPLAY
values, KEYWORD and EXACT outside the snippet) and WORD, ENGLISH and EXACT
analyzers get their own document objects and mapping templates.

Config tables are audited at field level and permission-gated; the
processes need quick_search_admin. Upgrade notes and DDL:
docs/MIGRATION-1.1.md, "Data-driven configuration".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 15:30:47 -05:00
james.maes 471dcbc1e7 Merge pull request #42 from QRun-IO/feature/quick-search-snake-table-names
feat!: snake_case table, PVS and permission names for 1.1.0-RC.3 (owner naming rule)
2026-10-12 15:22:36 -05:00
james.maes 1e8f46ec6f Merge branch 'develop' into feature/quick-search-snake-table-names 2026-10-12 15:22:22 -05:00
james.maes 9ae4e1445a Merge pull request #41 from QRun-IO/feature/developer-panel
feat: Quick Search record developer panel provider
2026-10-12 15:22:01 -05:00
james.maesandClaude Opus 5.5 53416f5163 feat!: snake_case table, PVS and permission names (owner naming rule)
Owner rule (2026-10-12): "table names should always be lowercase and use
_, and qbits should always be prefixed with their name". For 1.1.0-RC.3,
metadata only (the QBit ships no DDL):

- tables: quick_search_index, quick_search_index_run,
  quick_search_failed_event, quick_search_diagnostic_event,
  quick_search_validation_run (were camelCase)
- possible-value sources: quick_search_index, quick_search_searchable_table
- permission bases: quick_search_admin, quick_search_debug; table
  permissions follow the table names
- applyTablePrefix: a host prefix is snake_cased and joined with "_"
  (myApp -> my_app_quick_search_index); processes, widgets and apps keep
  applyPrefix
- @Deprecated LEGACY_* constants keep the old names for upgrade scripts
- docs/MIGRATION-1.1.md: upgrade steps, ALTER TABLE for camelCase
  databases, UPDATEs for audit_table, saved views and reports, permission
  grants; no reindex needed
- QuickSearchTableNamesTest covers new names, legacy constants and prefixing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 14:13:00 -05:00
james.maesandClaude Opus 5.5 5c8320bf29 feat(search): scores, highlights, totals and paging for core record search
QuickSearchRecordSearchProvider now fills the QQQ 4.1.0-RC.4 record
search contract: each hit's relevance score, a plain-text highlight of
the matching fieldValues.* fields (OpenSearch highlighting with
private-use markers, converted to text plus match ranges), each table's
total hits (totalCountByTable, counted to 10,000), and the request
offset (from). RECENT sort keeps relevance order: the index holds no
record modify dates. New client method searchRecordsPerTable;
searchRecordIdsPerTable delegates to it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 05:49:19 -05:00
138 changed files with 18174 additions and 498 deletions

No files matched your search

+11 -68
View File
@@ -1,71 +1,14 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.8
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
qqq-orb: kingsrook/qqq-orb@0.8
workflows:
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*|hotfix\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/v.*/]
publish_hotfix_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: hotfix
filters:
branches:
only: [/hotfix\/.*/]
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
+7
View File
@@ -0,0 +1,7 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
+23
View File
@@ -0,0 +1,23 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+74
View File
@@ -0,0 +1,74 @@
name: Release candidate
'on':
workflow_dispatch:
inputs:
dry_run:
description: Plan a cut without changing release state
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
+16
View File
@@ -0,0 +1,16 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
+2 -2
View File
@@ -31,13 +31,13 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quick_search_failed_event`.
- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
- `metadata/*` - Component producers and the table/process helpers.
**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
**Tables Produced:** `quick_search_index`, `quick_search_index_run`, `quick_search_failed_event` (plus a PVS over `quick_search_index`).
**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
+37 -2
View File
@@ -2,26 +2,61 @@
All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
## Release note correction (1.1.0-RC.11)
1.1.0-RC.11 (Maven Central, 2026-10-15 13:05Z) was built from release/1.1 `2e2501d` and contains #61 only. The `v1.1.0-RC.11` tag mistakenly points at `6028a66`; use `2e2501d` as RC.11's source. Cluster caches (#62) first ship in 1.1.0-RC.12.
## [Unreleased]
Requires QQQ 4.1.0-RC.4 or later (the developer panel contract, QRun-IO/qqq#1076).
Requires QQQ 4.1.0-RC.15 or later (central health checks, help bundles, the spec-managed UI API extension contract, and `ClusterCaches` for multi-node deployments).
### Fixed
- Single-node clusters: the Quick Search index no longer asks for a replica it cannot place, so a one-node OpenSearch stays green instead of yellow and `quick_search.opensearch` is no longer DEGRADED. New `indexReplicas` (default null: automatic, `min(maxAutoReplicas, data nodes - 1)`) and `maxAutoReplicas` (default 1) config; zero config gives 0 replicas on one node and 1 on larger clusters. The target is set on every new physical index and reconciled on existing ones at startup with `PUT /<index>/_settings` when it differs (idempotent, logged; a failure is logged and reported, never blocks startup; indexes with `auto_expand_replicas` and OpenSearch Serverless are left alone). The status snapshot and the `quick_search.opensearch` health details gain `unassignedReplicaShards`, `targetReplicas` and `replicaSettingsFailed`; yellow on a single node with every primary active (only replicas unassigned) is reported UP with a note. Yellow with an unassigned primary, or on a multi-node cluster, stays DEGRADED; red stays DOWN.
- Priming config records at startup, and recording diagnostic events, build their system session with the instance, so it has its all-access keys. Startup logged "QInstance was not supplied when creating a QSystemUserSession" once per node.
- Multi-node deployments: a Quick Search config change (an index row, its fields or the settings row, from the admin UI or a config process) now clears the cached config on every node, not only the node that made the change. The runtime's config cache (the config store snapshot and settings, and the per-table enabled flags) is registered with QQQ `ClusterCaches` as `qbit-quick-search.config`, and `QuickSearchRuntime.invalidateConfig()` publishes a cluster invalidation. The 30 s and 60 s TTLs remain as the safety net. `QuickSearchFieldRules`' warn-once table set is registered as node-local. `QuickSearchClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation.
- Indexing bookkeeping is no longer record-audited: the `quick_search_index` runtime fields written by basepull, full reindex and real-time indexing (status, last basepull time, document count, last run, real-time error count) and the settings row's priming outcome (`lastPrimedAt`, summary) are written with `omitDmlAudit`. On a production host these wrote about 470 "Record was Edited during process: Update search index" audit rows an hour. Admin edits to an index, its fields or the settings are still audited.
- Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write.
### Added
- UI API extension quick-search 1.1.0 (requires QQQ 4.1.0-RC.15, UI API 1.6.0): operationIds, standard error responses with BasicErrorResponseV1, extension tags, and retained deprecated QuickSearchErrorV1 schema. Existing operationIds and success payloads are preserved.
- Central health checks `quick_search.opensearch`, `quick_search.backlog` and the dynamic `quick_search.index.<indexName>` supplier, all SEARCH/DEPENDENCY checks every 60 seconds. Reuses cached diagnostics with deadline-bounded waits and safe count/flag details. Includes registration, state, deadline and credential-sentinel tests.
- Host-neutral `help/qbit-quick-search.yaml` bundle, registered through QQQ help bundles, with coverage and no-help-table tests. Replaces the temporary inline help loader; hosts can override entries with a HOST bundle.
- Every `quick_search_*` table and the Quick Search Admin app carry help that says what the table is for and how its rows get created (`help/quick-search.yaml`, QQQ help-bundle format, applied by `QuickSearchHelp` as a stand-in for core help bundles; host help is never overridden). `QuickSearchTablesGuardrailTest` fails when a table is unused by runtime code, neither primed nor listed as starting empty, or has no such help; `QuickSearchSeededDefaultsTest` proves priming populates the config tables on a fresh backend, is idempotent and keeps admin edits.
- Session-only typeahead: `GET /qqq/v1/qbit/quick-search/suggest` (`getQuickSearchSuggestions`) and `QuickSearchSuggestService.suggest(...)`. Reuses the permission-aware search action with one bounded candidate batch per table; groups expose only record IDs, filtered labels/subtitles, `hasMore` and `truncated`. Defaults to 5 items across 8 tables (maximum 10/20); short terms and unavailable search return empty groups. The existing UI extension and generated contract now carry both status and suggestions.
- Supported host status API (`api.QuickSearchStatusService.get()`), immutable status values, and `QuickSearchApi` home navigation constants (issue #50/F10). Permission-checked, bounded diagnostics refreshes share a short per-runtime cache and reuse saved validation counts. New spec-managed `GET /qqq/v1/qbit/quick-search/status` (`getQuickSearchStatus`) with committed OpenAPI fragment and strict HTTP/contract tests. RC.6 requires the default `/qbit/quick-search` prefix; `/quick-search` is not permitted by core. Existing routes and wire shapes are unchanged.
- Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`.
- **Data-driven configuration.** The searchable tables and fields are records: `quick_search_index` (the table registry, new columns `source` and `ownerEdited`) and the new `quick_search_index_field` (weight, role `DISPLAY`/`TITLE`/`SUBTITLE`/`BODY`/`KEYWORD`/`EXACT`, analyzer `PREFIX`/`WORD`/`ENGLISH`/`EXACT`, snippet and label flags). Indexing, search, the record-search provider and the diagnostics read the effective config from them (cached 30 s, dropped on change), so changes need no redeploy. Code (`@QuickSearchable`, `searchableTables`) only seeds them; rows an admin edited are never overwritten, and removals are remembered in the new `quick_search_config_tombstone`. Precedence and DDL: `docs/MIGRATION-1.1.md`, "Data-driven configuration".
- **Self-priming.** New code-declared tables and fields get their rows at startup and are built by the next basepull (`PENDING_BUILD`, `BUILDING`); a missing index is recreated and rebuilt, an empty one rebuilt; existing rows are migrated without a rebuild. The Search Diagnostics widget reports it; `QuickSearchStatus` gives hosts the same banner.
- Admin processes with review and result steps (record or table actions, never in the nav): Add Search Index (the valid create: eligible-table picker, estimated record count, default fields), Apply Search Index Changes (previews field changes, then rebuilds), Disable Search Index and Remove Search Index (delete or keep documents), Cancel Index Run (cooperative cancel between batches; runs end `CANCELLED` with counts), Pause Indexing and Resume Indexing (global pause in the new `quick_search_settings`; search keeps serving, changes are deferred and replayed).
- `QuickSearchResult.subtitle` and `displayValues` (from `SUBTITLE` and `DISPLAY` fields); document objects `fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`, added to existing indexes' mapping at startup.
- Possible-value sources `quick_search_eligible_table`, `quick_search_indexable_field`, `quick_search_timestamp_field`, `quick_search_field_role`, `quick_search_field_analyzer`, `quick_search_documents_choice`, `quick_search_config_source`; the join `quick_search_index_join_quick_search_index_field` and the Searchable Fields widget on the index record. Config tables are audited at field level.
### Removed
- None for central health integration; the status API, diagnostics widget and developer panel are retained.
### Changed
- Distinguish protected fields from fields that simply cannot be searched. Boolean, blob and hidden fields no longer trigger result rebuilding or erase record labels. Labels are suppressed only when their source fields are protected; unprotected hidden primary keys and timestamps remain usable.
- Exclude PII/PHI-classified fields (including table defaults) and field-security-locked fields everywhere Quick Search indexes or displays values. Configuration rejects protected fields; code defaults drop them with one warning per table. Search and diagnostics omit protected values from older documents. Rebuild existing indexes after upgrading or changing classification metadata. Uses QQQ 4.1.0-RC.6; no dependency changes.
- `quick_search_index`: the generic Create and Delete are off (`TABLE_INSERT`, `TABLE_DELETE`); use Add Search Index and Remove Search Index. `quick_search_searchable_table` is now a custom source over the effective tables. `quick_search_index_run` gains `cancelRequested` and `cancelRequestedBy`.
- Hidden and `isSensitive` fields are never indexed (previously hidden and password only).
- Record screens (record-screen audit): `quick_search_index_run` and `quick_search_failed_event` are written by the QBit only, so Create and Edit are off (Delete stays, for clearing); both get named field sections, record labels and pickers (`quick_search_run_type`, `quick_search_run_status`, `quick_search_failed_event_status`, `quick_search_failed_event_action`, and the table picker for `tableName`). The index record lists its Recent Runs (join `quick_search_index_join_quick_search_index_run`). `quick_search_validation_run.reportJson` uses a JSON code editor. The Basepull, Full Reindex and Reconcile processes run from Quick Search Index's Actions and are no longer app (nav) children; their table input is `searchTableName` (`tableName` still works for API callers, and a Quick Search table name is ignored), so running them from the table no longer limits them to `quick_search_index`. Process input steps now carry `HELP_TEXT` and `EDIT_FORM` components, so the Next UI renders their fields.
- **Snake_case names (owner naming rule, 2026-10-12), for 1.1.0-RC.3.** The tables are now `quick_search_index`, `quick_search_index_run`, `quick_search_failed_event`, `quick_search_diagnostic_event` and `quick_search_validation_run` (they were camelCase). The possible-value sources are `quick_search_index` and `quick_search_searchable_table`. The permission bases are `quick_search_admin` and `quick_search_debug`, and table permissions follow the table names. A table-name prefix is now converted to snake_case and joined with `_`. The old names remain as `@Deprecated` `LEGACY_*` constants. This is metadata only: the QBit ships no DDL. Upgrade steps, with SQL for camelCase databases and for stored references (audit table names, saved views, permission grants), are in `docs/MIGRATION-1.1.md` under "Snake_case names (1.1.0-RC.3)".
## [1.1.0-RC.1] - 2026-10-11
First release candidate of 1.1, published to Maven Central from `release/1.1`. It contains every change listed under 1.1.0 and 1.0.0 below (1.0.0 has not been released as GA). Requires QQQ 4.1.0-RC.2 or later. Upgrade notes: `docs/MIGRATION-1.1.md`.
## [1.1.0] - Unreleased
Requires QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI; built against 4.1.0-RC.2. See `docs/MIGRATION-1.1.md`.
Requires QQQ 4.1.0-RC.4 or later (the record-search provider SPI arrived in 4.1.0-RC.2; its scores, highlights, totals and paging in 4.1.0-RC.4); built against 4.1.0-RC.4. See `docs/MIGRATION-1.1.md`.
### Added
- `QuickSearchRecordSearchProvider`: serves QQQ core record search (`RecordSearchAction`, `POST /qqq/v1/search`, the Next UI global search) from OpenSearch for the tables Quick Search indexes. One `_msearch` request with a per-table top-N search (same query, AND semantics and field weights as `QuickSearchAction`); returns record ids only, which core re-reads through `QueryAction` (permissions, record locks, labels). Audit finding F-11, decision D1.
- `serveCoreRecordSearch` (default `true`): registers the provider and gives indexed tables core `searchFields` (visible string and integer searchable fields) when they have none. Host `searchFields` and a host-registered provider are never replaced.
- `recordSearchTimeoutMillis` (default 2500): wall-clock limit on the OpenSearch request; on timeout or any failure core falls back to its own search, and the provider claims no tables for 30 seconds.
- `QuickSearchOpenSearchClient.searchRecordIdsPerTable`.
- Record search details for QQQ 4.1.0-RC.4's record-search contract: the provider passes each hit's relevance `score`, a plain-text highlight (match ranges, from OpenSearch highlighting of `fieldValues.*` with private-use markers) and each table's total hits (`totalCountByTable`, counted to 10,000), and honors the request `offset` (`from`). `sort: RECENT` keeps relevance order (the index has no record modify dates). New `QuickSearchOpenSearchClient.searchRecordsPerTable`; `searchRecordIdsPerTable` now delegates to it. This requires QQQ 4.1.0-RC.4 or later.
- Diagnostic tools (#35), on by default (`enableDiagnosticTools`): Search Playground, Validate Search Index (schedulable with `validationCronExpression`; reports kept in the new `quickSearchValidationRun` table), Test Search Analyzer, and on every indexed table and on `quickSearchFailedEvent` rows the record actions Inspect Search Document (database vs index, and why a record does or does not match a term) and Reindex in Search. All are table or record actions, gated by the new `debugPermissionRules` (`quickSearchDebug.hasAccess`; Reindex in Search uses `adminPermissionRules`), rate-limited per user (`diagnosticRunsPerMinute`), and recorded in the new `quickSearchDiagnosticEvent` table. Settings: `enableRecordDiagnosticActions`, `validationCronExpression`, `validationCronTimeZoneId`, `validationSampleSize`.
- Search failure diagnostics: a failed or timed-out OpenSearch request behind core record search or `QuickSearchAction` is recorded as a `SEARCH_FAILED` diagnostic event (tables, duration, error, server; never the term); the runtime counts failures and backoff fallbacks. The Search Diagnostics widget (`quickSearchDiagnostics`, on the admin app home) shows the provider's state, recent failures and the latest validation.
- `QuickSearchDiagnosticsOperations`: read-only probe search (named clauses, field highlights, request and response JSON), explain, document reads, `_analyze`, document sampling, and raw cluster, mapping and settings reads.
+4 -2
View File
@@ -31,13 +31,15 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quick_search_failed_event`.
- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
- `metadata/*` - Component producers and the table/process helpers.
**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
**Tables Produced:** `quick_search_index`, `quick_search_index_field`, `quick_search_settings`, `quick_search_config_tombstone`, `quick_search_index_run`, `quick_search_failed_event`, `quick_search_diagnostic_event`, `quick_search_validation_run` (plus PVSs).
**Data-driven config (`config/`):** the records are the source of truth. `QuickSearchConfigStore` builds the effective `QuickSearchableTableConfig`s from `quick_search_index` + `quick_search_index_field` (30 s cache, dropped by the tables' customizers); `QuickSearchRuntime.getDiscoveredTables()/getTableConfig()` return them, `getCodeTables()` the code defaults. `QuickSearchConfigPrimer` seeds records from code (never over `ownerEdited` rows or tombstones). QBit writes to config tables go through `QuickSearchConfigWrites.asSystem` so customizers do not treat them as owner edits. Admin processes live in `processes/config/` (streamed ETL with review steps).
**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
+224 -19
View File
@@ -8,20 +8,20 @@
Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the 1.1 line builds against 4.1.0-RC.4 from the next release candidate on.
Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the current build requires QQQ 4.1.0-RC.7 for help bundles.
## What it does
| Capability | How |
|---|---|
| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quickSearchFailedEvent` and replayed by the scheduled basepull. |
| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quick_search_failed_event` and replayed by the scheduled basepull. |
| Basepull catch-up | Scheduled process (`schedulerName` plus `basepullRepeatSeconds`) that re-indexes rows changed since the previous run's start, with a configurable overlap, paging by primary key. |
| Reconcile | Scheduled or on-demand process that re-indexes everything in place, then removes documents whose source row is gone, including documents of tables no longer configured. No search blackout. |
| Full rebuild | Indexes every table into a fresh physical index and atomically swaps the alias; the old index keeps serving until the new one is complete. Also upgrades an index created by an older release. |
| Core search | `QuickSearchRecordSearchProvider` serves QQQ's `RecordSearchAction` (`POST /qqq/v1/search`, the Next UI global search box) from OpenSearch for the indexed tables, with a timeout and an automatic fallback to core's own search. |
| Search | `QuickSearchAction`: AND semantics, edge-ngram typeahead with a plain search analyzer, per-field weights, highlights, `tableNames` and `limitPerTable`, bounded paging, `tableLabel` on results. Only tables the session may read are searched; hits are re-read through `QueryAction` so record security locks apply. |
| Connections | HTTP or HTTPS, basic auth, AWS IAM SigV4, custom CA or trust store, mTLS, timeouts, pool size, a transport customizer SPI for anything else. Secrets by `${env.X}` reference. |
| Operations | `quickSearchIndex` (per-table status, counts, last run, last error), `quickSearchIndexRun` (history, purged after `runHistoryRetentionDays`), `quickSearchFailedEvent`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
| Operations | `quick_search_index` (per-table status, counts, last run, last error), `quick_search_index_run` (history, purged after `runHistoryRetentionDays`), `quick_search_failed_event`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
## Supported platforms
@@ -129,12 +129,12 @@ Search runs as the current `QContext` session: tables the session cannot read ar
QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v1/search` and used by the Next UI's global search box): a case-insensitive contains-query over each table's `searchFields`. With `serveCoreRecordSearch` (default `true`) the producer registers `QuickSearchRecordSearchProvider` on the instance, and core asks it to search every table Quick Search indexes. The API and the UI do not change.
- **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quickSearchIndex`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights); each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order. Core re-reads them by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, and records deleted since they were indexed drop out.
- **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quick_search_index`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights), skipping the request's `offset`; each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order, each with its score and a highlight of the matching `fieldValues.*` fields (plain text plus match ranges), and the table's total hits (counted to 10,000). Core re-reads the ids by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, records deleted since they were indexed drop out, and only highlights of fields the user can see (where core's own highlighting found no match) reach the response, with `tables[].totalCount` and `hasMore`. The index holds no record modify dates, so `sort: RECENT` gets relevance order for indexed tables.
- **Failure.** The OpenSearch request is cancelled after `recordSearchTimeoutMillis` (default 2500). Any failure (timeout, cluster down, index missing) makes core search those tables with its own contains-query instead, so search degrades but never fails. After a failure, Quick Search claims no tables for 30 seconds, so a down cluster does not add a timeout to every search.
- **Opting out.** `withServeCoreRecordSearch(false)` leaves core search and `searchFields` untouched. If the host already registered another record-search provider, the QBit keeps it and logs a warning (an instance has one provider).
`QuickSearchAction` remains the richer API (scores, highlights, pagination, totals).
`QuickSearchAction` remains the direct API (all hits in one ranked list, `searchableText` highlights, pagination across tables).
### Processes
@@ -146,6 +146,18 @@ QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v
Without `schedulerName` the processes exist but nothing runs them; the QBit logs a warning at startup.
### Managing searchable tables (data-driven, 1.1.0-RC.3)
Which tables and fields are indexed is data: `quick_search_index` (one row per table) and `quick_search_index_field` (its fields: weight, role, analyzer, snippet and label flags), edited in the Quick Search admin app with the standard screens and these record actions (each with a review step; none is in the nav):
- **Add Search Index** (Quick Search Index, Actions): pick a readable, eligible table, review the estimated record count and the fields it starts with, save; the first build is queued.
- **Searchable Fields** on the index record: add, edit or remove fields; then **Apply Search Index Changes** previews the changes and rebuilds.
- **Disable Search Index** and **Remove Search Index**: stop indexing at once and delete or keep the documents. Removals are remembered, so code defaults do not bring them back.
- **Cancel Index Run** (Quick Search Index Run): stops a running reindex after its current batch (`CANCELLED`, with counts).
- **Pause Indexing** and **Resume Indexing** (Quick Search Settings): pause all indexing while search keeps serving; changes are indexed after resume.
`@QuickSearchable` entities and `searchableTables` only seed these records (self-priming at startup: new tables and fields are added and built automatically; rows an admin edited are never overwritten). The precedence rules, the DDL for the new tables and the upgrade steps are in `docs/MIGRATION-1.1.md`, "Data-driven configuration".
### Scheduling
The schedules run only when the host starts through QQQ's `QApplicationLauncher` (`com.kingsrook.qqq.middleware.javalin.QApplicationLauncher`, in `qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` (because the QBit's processes are scheduled), then the instance's runtime services. A host that starts only a server, or builds the `QInstance` itself, gets the processes but nothing runs them on a schedule, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
@@ -162,12 +174,12 @@ config.withSchedulerName("quartz")
### Diagnostic tools
With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate the search and debug it from the admin UI. They are table and record actions (never left-nav entries), gated by permissions, recorded in `quickSearchDiagnosticEvent`, limited to `diagnosticRunsPerMinute` runs per user and tool, and read-only except Reindex in Search.
With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate the search and debug it from the admin UI. They are table and record actions (never left-nav entries), gated by permissions, recorded in `quick_search_diagnostic_event`, limited to `diagnosticRunsPerMinute` runs per user and tool, and read-only except Reindex in Search.
| Tool | Where | What it does |
|---|---|---|
| Search Playground | Quick Search Index, Actions (select rows to pre-fill their tables) | Runs a term the way the global search does, as you: the path each table takes (OpenSearch provider, core database search, or not searched) and why; the provider's own request, timed; the global search results you would see; per indexed table, the production query with scores, highlights, matched fields, which hits you can read and which the global search shows, the query DSL and the raw response (hits you cannot read are redacted); the term's tokens; and `explain` for a hit (`table:id`). The form stays open, so you can change the term and run again. |
| Validate Search Index | Quick Search Index, Actions; also scheduled by `validationCronExpression` | Cluster and index health, the alias, leftover physical indexes from interrupted rebuilds, the live mapping (version, field types, analyzers, the `fieldValues` template) and analysis settings, then per table: database rows vs documents, a sample of records checked present and current in the index, a random sample of documents checked to still have a record, the index row's status and last run, and waiting failed events. Each finding is OK, INFO, WARN or FAIL with what to do. The report is saved as a `quickSearchValidationRun` row (HTML for the record view, JSON for tools). Database reads run as the system user so counts compare like with like; the report holds counts, ids and field names, never record values. One validation runs at a time per server. |
| Validate Search Index | Quick Search Index, Actions; also scheduled by `validationCronExpression` | Cluster and index health, the alias, leftover physical indexes from interrupted rebuilds, the live mapping (version, field types, analyzers, the `fieldValues` template) and analysis settings, then per table: database rows vs documents, a sample of records checked present and current in the index, a random sample of documents checked to still have a record, the index row's status and last run, and waiting failed events. Each finding is OK, INFO, WARN or FAIL with what to do. The report is saved as a `quick_search_validation_run` row (HTML for the record view, JSON for tools). Database reads run as the system user so counts compare like with like; the report holds counts, ids and field names, never record values. One validation runs at a time per server. |
| Test Search Analyzer | Quick Search Index, Actions | `_analyze` on the live index: the tokens of a text as indexed (edge n-grams of 2 to 20 characters) and as a query, and, given a term, whether every term token is among the text's indexed tokens. |
| Inspect Search Document | Every indexed table's records (Actions), and Quick Search Failed Event rows | The record's document in the index next to the document indexing would write from the database now, field by field (current, stale or missing), with versions and the raw document. Enter a term to see why the record does or does not match it: term length, table enabled, read permission, document present and current, each term token found or not (and why: one character, longer than 20, not in the text, or only in the database's newer values), `explain`, and the record's rank against the number of ids the global search takes. |
| Reindex in Search | Every indexed table's records, and Quick Search Failed Event rows | Rebuild the selected records' documents from the database now (versioned writes, so a newer document is never overwritten); a record that no longer exists has its document removed; a record you cannot read is skipped. Asks to confirm. Needs the admin permission. |
@@ -175,7 +187,7 @@ With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate th
Search failures are recorded too: when the provider's OpenSearch request fails or times out (and core falls back to its own search), or `QuickSearchAction`'s search fails, a `SEARCH_FAILED` event keeps the tables, duration, error class and message, and the server. The user's term is never stored, only its length. At most 120 events are written per server per minute.
Permissions: the read-only tools and the widget use `debugPermissionRules` (default `quickSearchDebug.hasAccess`); Reindex in Search uses `adminPermissionRules` (`quickSearchAdmin.hasAccess`); the two history tables use `tablePermissionRules` (grant `.read` only: rows are written by the QBit). Impersonation ("search as user X") is deliberately not offered: a fabricated session would not carry the real user's security key values, so its results would mislead.
Permissions: the read-only tools and the widget use `debugPermissionRules` (default `quick_search_debug.hasAccess`); Reindex in Search uses `adminPermissionRules` (`quick_search_admin.hasAccess`); the two history tables use `tablePermissionRules` (grant `.read` only: rows are written by the QBit). Impersonation ("search as user X") is deliberately not offered: a fabricated session would not carry the real user's security key values, so its results would mislead.
### Record developer panel
@@ -215,18 +227,20 @@ The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned
| `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
| `fullReindexStaleMinutes` | 30 | A running full reindex that shows no sign of life for this long is treated as stopped: it no longer blocks a new one, and basepull recovers its tables. Keep it well above the time one page of `sourceBatchSize` records takes; null means the default |
| `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
| `indexReplicas` | null (automatic) | Index `number_of_replicas`; null means `min(maxAutoReplicas, data nodes - 1)`, so a single node gets 0 and stays green. Set on new indexes, reconciled on existing ones at startup |
| `maxAutoReplicas` | 1 | Most replicas automatic sizing asks for |
| `maxFieldLength` | 10000 | Truncate each indexed value |
| `maxSearchLimit` | 100 | Largest page size |
| `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
| `serveCoreRecordSearch` | `true` | Register `QuickSearchRecordSearchProvider` so core `POST /qqq/v1/search` uses OpenSearch for indexed tables, and give those tables `searchFields` when they have none |
| `recordSearchTimeoutMillis` | 2500 | Wall-clock limit on the OpenSearch request behind a core search; past it, core falls back to its own search |
| `enableDiagnosticTools` | `true` | Add the diagnostic tools, the `quickSearchDiagnosticEvent` and `quickSearchValidationRun` tables and the diagnostics widget |
| `enableDiagnosticTools` | `true` | Add the diagnostic tools, the `quick_search_diagnostic_event` and `quick_search_validation_run` tables and the diagnostics widget |
| `enableRecordDiagnosticActions` | `true` | With the tools on, add Inspect Search Document and Reindex in Search to every indexed table |
| `validationCronExpression`, `validationCronTimeZoneId` | none | Schedule Validate Search Index; same rules as the reconcile schedule (a cron-capable scheduler such as Quartz, a valid Quartz expression, a time zone ID) |
| `validationSampleSize` | 50 | Records and documents sampled per table by validation (at most 1000) |
| `diagnosticRunsPerMinute` | 30 | Per-user, per-tool limit on diagnostic runs |
| `debugPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchDebug` | Rules for the read-only diagnostic tools and the widget |
| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchAdmin` | Rules for the app, the maintenance processes and Reindex in Search |
| `debugPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quick_search_debug` | Rules for the read-only diagnostic tools and the widget |
| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quick_search_admin` | Rules for the app, the maintenance processes and Reindex in Search |
| `tablePermissionRules` | `READ_WRITE_PERMISSIONS` | Rules for the operational tables |
| `tableMetaDataCustomizer` | | Applied to every produced table (backend details, for example) |
| `searchableEntityClasses`, `searchableTables` | at least one | What to index |
@@ -236,11 +250,20 @@ The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned
## Operational tables
- `quickSearchIndex`: one row per table with `enabled` (honoured by indexing and search, cached for one minute), `basepullIntervalMinutes`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`, `NEEDS_REINDEX` when the configured fields changed since the row was created, or `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
- `quickSearchIndexRun`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status`, counts, `errorMessage`.
- `quickSearchDiagnosticEvent` (with `enableDiagnosticTools`): `eventType` (`SEARCH_FAILED`, `TOOL_RUN`, `REINDEX`, `RATE_LIMITED`), `source`, `outcome`, `userName`, `tableNames`, `searchTerm` (tool runs only), `durationMillis`, `errorClass`, `errorMessage` (text), `detail` (text), `hostName`, `createDate`. Purged after `runHistoryRetentionDays`.
- `quickSearchValidationRun` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
- `quickSearchFailedEvent`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
- `quick_search_index`: one row per table with `source` (`CODE` or `OWNER`), `ownerEdited`, `enabled` (honoured by indexing and search; a change applies at once), `basepullIntervalMinutes`, `basepullTimestampField`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`; `PENDING_BUILD` when a build is queued and `BUILDING` while the basepull builds it; `NEEDS_REINDEX` when the fields changed since the last build; `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
- `quick_search_index_field`: the searchable fields of each index row: `fieldName`, `weight`, `role` (`DISPLAY`, `TITLE`, `SUBTITLE`, `BODY`, `KEYWORD`, `EXACT`), `analyzer` (`PREFIX`, `WORD`, `ENGLISH`, `EXACT`; blank: by role), `includeInSnippet`, `includeLabel`, `sortOrder`, `source`, `ownerEdited`. Unique on (`quickSearchIndexId`, `fieldName`).
- `quick_search_settings`: one row: `indexingPaused` (with `pausedAt`, `pausedBy`, `pauseReason`) and the last self-priming report (`lastPrimedAt`, `lastPrimingSummary`).
- `quick_search_config_tombstone`: tables (blank `fieldName`) and fields an admin removed; priming never re-adds them.
- `quick_search_index_run`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status` (`RUNNING`, `COMPLETED`, `FAILED`, `CANCELLED`), counts, `errorMessage`, `cancelRequested`, `cancelRequestedBy`.
- `quick_search_diagnostic_event` (with `enableDiagnosticTools`): `eventType` (`SEARCH_FAILED`, `TOOL_RUN`, `REINDEX`, `RATE_LIMITED`), `source`, `outcome`, `userName`, `tableNames`, `searchTerm` (tool runs only), `durationMillis`, `errorClass`, `errorMessage` (text), `detail` (text), `hostName`, `createDate`. Purged after `runHistoryRetentionDays`.
- `quick_search_validation_run` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
- `quick_search_failed_event`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
### Starter defaults and empty-state help
This qbit ships no DDL and no seed changelog: startup self-priming (`QuickSearchConfigPrimer`) is its seed mechanism. On a fresh database it writes a `quick_search_index` row and `quick_search_index_field` rows for every table the code declares searchable, and the single `quick_search_settings` row (id 1). `quick_search_config_tombstone` and the five log and history tables start empty by design (rows appear when an admin removes a table or field, or when the qbit runs).
Real-time error counts on Quick Search Index show failures since the last successful full reindex or basepull. Each failed event increases the count; deferred changes while indexing is paused do not. Counter updates are best effort and serialized across listeners in one JVM; concurrent application nodes can still race because QQQ has no arithmetic update API.
## Extending
@@ -254,7 +277,7 @@ CI=true ./mvnw verify # plus Testcontainers integration te
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
```
Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.4 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports QQQ 4.1.0-RC.8 through the default-active `qqq-snapshot` profile ahead of the parent BOM (override only with a compatible released version using `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
## Upgrading from 1.0
@@ -262,7 +285,7 @@ See [docs/MIGRATION-1.1.md](docs/MIGRATION-1.1.md). In short: 1.1 needs the QQQ
## Upgrading from 0.x
See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, grant `quickSearchAdmin.hasAccess` and the table permissions, add the new operational table columns and `quickSearchFailedEvent`, and run a full reindex once so the index gets the new mapping.
See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, grant `quick_search_admin.hasAccess` and the table permissions, add the new operational table columns and `quick_search_failed_event`, and run a full reindex once so the index gets the new mapping.
## Adopting in another app
@@ -273,3 +296,185 @@ See also the [QQQ adoption checklist](https://github.com/QRun-IO/qqq/blob/develo
## License
Apache License, Version 2.0. See [LICENSE](LICENSE).
## UI API for hosts
The `quick-search` extension registers both status and suggestions in the committed
[OpenAPI fragment](src/main/resources/openapi/qqq-v1/quick-search.yaml).
### Typeahead suggestions
`GET /qqq/v1/qbit/quick-search/suggest` (`getQuickSearchSuggestions`) requires a
signed-in session. Quick Search admin permission is not required. Java hosts use
`QuickSearchSuggestService.suggest(q, limitPerTable, maxTables, tables)` in the caller's
`QContext`.
| Query parameter | Meaning |
| --- | --- |
| `q` | Required text, trimmed with consecutive whitespace collapsed. Fewer than 2 characters returns empty groups; more than 100 returns HTTP 400. |
| `limitPerTable` | Integer 1..10; default 5. |
| `maxTables` | Integer 1..20; default 8. |
| `tables` | Optional comma-separated table metadata names. Unknown, disabled and unreadable tables are omitted; an empty filter returns no groups. |
```json
{
"query": "blue wid",
"groups": [{
"tableName": "product",
"tableLabel": "Products",
"hasMore": true,
"items": [{"recordId": "42", "label": "Blue Widget", "subtitle": "Small"}]
}],
"truncated": false
}
```
The existing edge-ngram query supports the last word's prefix. Groups are ordered
by their best accessible score, with per-table relevance order preserved. Labels
fall back to `tableLabel #recordId`; subtitles are nullable and come only from the
already-filtered result. Scores, highlights and display values are never returned.
The existing action enforces table READ permissions, enabled-table selection,
configured record security locks (on by default), PHI/PII exclusions and protected
primary keys.
Suggestions make one bounded OpenSearch request per eligible table, fetching at
most `2 * (limitPerTable + 1)` candidates under record locks (also subject to the
host's `maxSearchLimit`). Locked rows can leave a group partially filled.
`hasMore` confirms an extra accessible hit in this batch; false does not prove
there are no further matches. `truncated` means more tables had accessible hits
than `maxTables`. Missing runtime/client or unavailable search returns empty groups.
Missing `q`, invalid integers and out-of-range limits return HTTP 400.
### Administrative status
Requires QQQ **4.1.0-RC.6**. Hosts should use the supported API rather than reach into
`QuickSearchRuntime`, `QuickSearchOpenSearchClient`, diagnostic renderers, or qbit tables:
```java
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatusService;
// Runs in the host request's QContext, with the caller's QSession.
QuickSearchStatus status = QuickSearchStatusService.get();
String homeHref = status.homeHref();
```
UI clients use **`GET /qqq/v1/qbit/quick-search/status`**, operationId
`getQuickSearchStatus`, extension `quick-search` version `1.1.0` (requires QQQ `4.1.0-RC.15`, UI API `1.6.0`). The endpoint returns
that same immutable value as JSON; it requires the Quick Search app's configured
`adminPermissionRules` (default `quick_search_admin.hasAccess`). Permission is checked
before every read, including cache hits. No additional source-table permissions are
required: like index validation, this is an administrator's aggregate view of the index.
The committed contract is [quick-search.yaml](src/main/resources/openapi/qqq-v1/quick-search.yaml).
The default core namespace is intentional: RC.6 permits `/qbit/quick-search` but rejects
`/quick-search`, which is not an established legacy prefix. There were no existing
Quick Search UI API routes to preserve. Core source is unchanged.
- `state`: `OK`, `DEGRADED` (including paused indexing, backlog, incomplete diagnostics
or a refresh timeout), `DOWN` (unavailable health or red health), or `DISABLED`
(qbit absent or every persisted index disabled). Pausing indexing does not disable search.
- `clusterReachable`, `clusterHealth`, `nodes`, `activeShards`, `unassignedShards` come
from the existing diagnostics health operation. Null means unknown/not probed.
- `indexes` contains up to 100 persisted index rows. `documentCount` and `sourceCount`
come from the latest saved `IndexValidator` report, with `countsCheckedAt`. Without
matching validation, only the last indexing run's document count is available and
its observation time is unknown. These are **saved counts, not live counts**. A
missing source count must not be displayed as zero or “in sync.”
- `pending` includes `PENDING` and `AWAITING_REINDEX` failed events; `exhausted` counts
failed events that gave up. At most 1000 backlog rows are read; `backlogTruncated`
makes both counts lower bounds. `tablesTruncated` indicates incomplete index rows.
These counters do not measure an external publisher's queue.
- `lastSuccessfulSync` is the newest successful indexing watermark among the included
tables (basepull is start-anchored), not proof that all tables are current.
`lastError` is the newest available indexing/runtime search error or snapshot failure.
- `checkedAt` is snapshot completion time. A per-runtime five-second cache bounds
polling cost; each refresh uses `DiagnosticsRateLimiter`'s per-user `STATUS` key.
Rate-limit refusal uses the existing diagnostics error (HTTP 400). A refresh waits
at most two seconds and retains at most one in-flight worker, even if a backend is
stuck. Runtime shutdown stops the worker. No source counts, record samples, full
validation, or per-table OpenSearch calls run on this polling path.
Link to `status.homeHref()` instead of copying the Quick Search home. It includes the
configured metadata prefix. `QuickSearchApi.APP_NAME` (`quickSearchAdmin`) and
`QuickSearchApi.HOME_HREF` (`/app/quickSearchAdmin`) are available for default-name
navigation. Keep detailed runs, failed events, validation and operator tools in the
qbit's own home. [kof22 migration mapping](docs/STATUS-API-HOST-MIGRATION.md).
## Health checks
The QBit producer registers these checks with QQQ's central health registry. All use
component `quick_search`, category `SEARCH`, a 60-second interval and a 5-second timeout.
| Name | Kind | Interval | Reports |
| --- | --- | --- | --- |
| `quick_search.opensearch` | DEPENDENCY | 60s | Cluster reachability and colour: red is DOWN, yellow is DEGRADED, disabled is UNKNOWN. |
| `quick_search.backlog` | DEPENDENCY | 60s | DEGRADED for any exhausted events, more than 100 pending events, or paused indexing. |
| `quick_search.index.<indexName>` | DEPENDENCY | 60s | One check per persisted index row; non-active status, count mismatch, last error or real-time errors are DEGRADED. Disabled indexes are UNKNOWN. |
Index names come from the row's table name, normalized to lowercase health identifiers.
The supplier refreshes the set through core's normal discovery cycle. Checks reuse the
five-second status cache and saved validation counts, with waits bounded by the runner's
deadline. Incomplete measurements never become zero; indexes absent from the bounded
100-row status snapshot report UNKNOWN. Backlog truncation reports DEGRADED.
Details contain counts, timestamps and flags, never stored error text or credentials.
The status endpoint, diagnostics widget and record developer panel remain available.
This integration requires QQQ 4.1.0-RC.8 or later, which includes the central health SPI. The POM pins that release. For local offline verification:
```powershell
./mvnw.cmd -o verify
```
## Classified data
Quick Search excludes fields classified as **PII** or **PHI**, including fields
that inherit the table's classification. It also excludes hidden, sensitive,
password/masked fields and every field with a `FieldSecurityLock`; a shared
search index cannot enforce access to individual fields for each user. PUBLIC,
CONFIDENTIAL and unclassified fields remain eligible under the usual field rules.
The same rule applies to annotation and code configuration, admin field choices,
real-time writes, basepull, rebuilds, replay, labels, snippets, analyzer copies and
result display fields. An admin trying to add a protected field gets an error.
Protected fields in existing or explicit code configuration are dropped, with one
warning per table metadata lifetime; warnings never contain field values. A
protected primary key prevents indexing the record or sending its ID for deletion.
Protection is separate from search eligibility: boolean, blob, hidden and other
unsupported fields alone do not trigger result filtering or remove record labels.
Labels are omitted when their source fields are protected; unrelated protected
fields do not erase safe labels. Unprotected hidden keys and timestamps can still
provide record IDs and document versions.
After upgrading, or after changing a field's classification or security metadata,
run **Full Reindex** across all indexed tables to remove previously indexed values.
Until then, tables containing protected fields are searched through allowed field
values, and results are rebuilt without old combined labels or highlights.
Diagnostic views also omit protected values and the raw response for these tables.
This does not purge old OpenSearch backups or retained physical indexes; apply your
host's retention policy to those copies.
This release builds against QQQ **4.1.0-RC.7**. Support for the host's
`AuditValueProtector` classification (`ClassifiedValues`, qqq#1182) and
`FieldAccessHelper.getLockTier` (qqq#1172) is deferred until the qbit moves to RC.7.
## In-app help
The producer registers [help/qbit-quick-search.yaml](src/main/resources/help/qbit-quick-search.yaml)
with QQQ's help registry (requires QQQ 4.1.0-RC.7 or later). The bundle covers
Quick Search's own app, tables, actions, widgets and possible-value sources.
Registration is metadata-only and works without a `helpContent` table. Hosts
with help enabled load help content and call `HelpBundles.syncOnStartup(qInstance)`
after producing all metadata; QQQ guards sync when the help table is absent.
To customize wording or document deployment schedules, register your own
`layer: HOST` bundle with the same entry keys and roles using
`HelpBundles.register(qInstance, "help/my-host.yaml")`. HOST text takes precedence
over QBIT text; edits made by users are preserved by sync. Host-specific record
actions (Inspect Search Document and Reindex in Search on host tables) belong in
the host bundle. Keep the bundle's literal version aligned with the POM revision.
`QuickSearchTablesGuardrailTest.helpBundleIsComplete` uses `HelpBundleAssertions`
to check owned objects, non-trivial fields, summary length and live references.
The same test class checks registration without the help subsystem.
+203 -2
View File
@@ -33,7 +33,208 @@ restores 1.0 behaviour: no provider, no `searchFields` changes.
The QBit now adds admin tools to validate and debug search (see the README, "Diagnostic tools"). They are on by default.
- **Two new operational tables**, on `backendName` like the others: `quickSearchDiagnosticEvent` and `quickSearchValidationRun` (columns in the README, "Operational tables"). Create them in the host's schema, with the naming the host applies to the other operational tables (`tableMetaDataCustomizer` applies to them too). Until they exist, writing events and saving reports fails quietly (logged at warn) and the tools still run.
- **Permissions.** Grant `quickSearchDebug.hasAccess` to the people who may use the read-only tools and see the widget, `quickSearchAdmin.hasAccess` for Reindex in Search, and `.read` on the two new tables (the QBit writes them; users never do).
- **Two new operational tables**, on `backendName` like the others: `quick_search_diagnostic_event` and `quick_search_validation_run` (columns in the README, "Operational tables"). Create them in the host's schema, with the naming the host applies to the other operational tables (`tableMetaDataCustomizer` applies to them too). Until they exist, writing events and saving reports fails quietly (logged at warn) and the tools still run.
- **Permissions.** Grant `quick_search_debug.hasAccess` to the people who may use the read-only tools and see the widget, `quick_search_admin.hasAccess` for Reindex in Search, and `.read` on the two new tables (the QBit writes them; users never do).
- **New processes** have a `tableName` (Quick Search Index, Quick Search Failed Event, and each indexed table), so they appear as table and record actions. They are not added to the admin app's children; a host that strips processes from its nav needs no change.
- **Opting out.** `withEnableDiagnosticTools(false)` adds none of it; `withEnableRecordDiagnosticActions(false)` keeps the tools on Quick Search Index but adds nothing to the indexed tables.
## Snake_case names (1.1.0-RC.3)
Owner naming rule (2026-10-12): table names are lowercase snake_case, and a QBit's tables are prefixed with its full name. RC.3 renames the QQQ metadata names. The QBit ships no DDL, so nothing is dropped or rewritten; what you change depends on how your database tables are named today.
| Kind | Before (RC.1, RC.2) | From RC.3 |
|---|---|---|
| Table | `quickSearchIndex` | `quick_search_index` |
| Table | `quickSearchIndexRun` | `quick_search_index_run` |
| Table | `quickSearchFailedEvent` | `quick_search_failed_event` |
| Table | `quickSearchDiagnosticEvent` | `quick_search_diagnostic_event` |
| Table | `quickSearchValidationRun` | `quick_search_validation_run` |
| Possible-value source | `quickSearchIndex`, `quickSearchSearchableTable` | `quick_search_index`, `quick_search_searchable_table` |
| Table permissions | `quickSearchIndex.read`, `quickSearchIndexRun.read`, ... | `quick_search_index.read`, `quick_search_index_run.read`, ... |
| Admin permission | `quickSearchAdmin.hasAccess` | `quick_search_admin.hasAccess` |
| Debug permission | `quickSearchDebug.hasAccess` | `quick_search_debug.hasAccess` |
| Record actions on failed events | `quickSearchInspect_quickSearchFailedEvent`, `quickSearchReindexRecords_quickSearchFailedEvent` | `quickSearchInspect_quick_search_failed_event`, `quickSearchReindexRecords_quick_search_failed_event` |
Unchanged: Java class names, field names (`quickSearchIndexId`, `tableName`), process, widget and app names (`quickSearchFullReindex`, `quickSearchDiagnostics`, `quickSearchAdmin` app), the OpenSearch index, and the developer panel id.
**With a table-name prefix** (`withTableNamePrefix` or the producer namespace), table and possible-value source names are now the prefix in snake_case joined with `_`: prefix `myApp_` or `myApp` gives `my_app_quick_search_index`, where RC.2 gave `myApp_quickSearchIndex`. Process, widget and app names keep the prefix as given.
**The old names stay available** as `@Deprecated` constants for upgrade scripts: `QuickSearchIndex.LEGACY_TABLE_NAME` (and the same on the other four entities), `QuickSearchSearchableTablePossibleValueSourceProducer.LEGACY_NAME`, and `QuickSearchQBitConfig.LEGACY_ADMIN_PERMISSION_BASE_NAME` / `LEGACY_DEBUG_PERMISSION_BASE_NAME`. They will be removed in 2.0.
### Database tables
- **Your tables are already `quick_search_*`** (for example you set the RDBMS backend name with `inferBackendName` or a table customizer): no DDL. You can drop the customizer, since the QQQ name now equals the database name.
- **Your tables are camelCase** (`"quickSearchIndex"`): rename them, in a maintenance window, before starting RC.3. PostgreSQL example:
```sql
ALTER TABLE "quickSearchIndex" RENAME TO quick_search_index;
ALTER TABLE "quickSearchIndexRun" RENAME TO quick_search_index_run;
ALTER TABLE "quickSearchFailedEvent" RENAME TO quick_search_failed_event;
ALTER TABLE "quickSearchDiagnosticEvent" RENAME TO quick_search_diagnostic_event;
ALTER TABLE "quickSearchValidationRun" RENAME TO quick_search_validation_run;
```
Rename the matching sequences, indexes and constraints the same way if your tooling depends on their names. Data is untouched. If you cannot rename yet, point each table at its old name with an `RDBMSTableBackendDetails` table name in a table customizer.
### Stored references
QQQ stores table, process and permission names in your own data. Update them in the same window (PostgreSQL examples; adjust schema and column names to your instance):
```sql
-- audit history: audit_table.name holds the QQQ table name
UPDATE audit_table SET name = 'quick_search_index' WHERE name = 'quickSearchIndex';
UPDATE audit_table SET name = 'quick_search_index_run' WHERE name = 'quickSearchIndexRun';
UPDATE audit_table SET name = 'quick_search_failed_event' WHERE name = 'quickSearchFailedEvent';
UPDATE audit_table SET name = 'quick_search_diagnostic_event' WHERE name = 'quickSearchDiagnosticEvent';
UPDATE audit_table SET name = 'quick_search_validation_run' WHERE name = 'quickSearchValidationRun';
-- saved views and saved reports
UPDATE saved_view SET table_name = 'quick_search_index' WHERE table_name = 'quickSearchIndex';
UPDATE saved_report SET table_name = 'quick_search_index' WHERE table_name = 'quickSearchIndex';
-- ... and the same for the other four tables
```
- **Permission grants.** Wherever your roles live (QQQ role tables, an identity provider's group attributes, code), replace `quickSearchAdmin.hasAccess` with `quick_search_admin.hasAccess`, `quickSearchDebug.hasAccess` with `quick_search_debug.hasAccess`, and `quickSearchIndex.read` / `.edit` (and the other tables' permissions) with the `quick_search_*` names. Users without the new grants lose the admin app and tools until they are granted.
- **Bookmarks and links** to `/app/quickSearchIndex/...` move to `/app/quick_search_index/...`.
- **Index rows and failed events** store *your* table names (`quick_search_index.table_name`, `quick_search_failed_event.table_name`), not the QBit's, so they need no change for this rename. The OpenSearch documents are keyed by your tables too, so **no reindex is needed**. If you rename your own tables at the same time, update those two columns and run a full reindex.
### Checklist
1. Back up the database.
2. Rename the tables (only if they are camelCase) and update the stored references above.
3. Update permission grants.
4. Upgrade to RC.3 and start; `startupMode` `FAIL_FAST` reports any table the backend cannot find.
5. Open Admin › Quick Search and run Validate Search Index.
## Data-driven configuration (1.1.0-RC.3)
From RC.3, which tables and fields Quick Search indexes is **data**: standard QQQ tables an admin edits, secured by permissions and audited. Code (`@QuickSearchable` / `@QuickSearchField` entities and `searchableTables`) still works, but it only **seeds** the records. Changes take effect without a redeploy.
### New tables and columns
| Table | One row per | Notes |
|---|---|---|
| `quick_search_index` (existing) | searchable table | New columns `source` (`CODE` or `OWNER`) and `owner_edited`. `basepull_timestamp_field` is now editable. The generic Create and Delete screens are off: use Add Search Index and Remove Search Index. |
| `quick_search_index_field` | searchable field of an index | `field_name`, `weight` (1-1000), `role` (`DISPLAY` T1 display, `TITLE`, `SUBTITLE`, `BODY`, `KEYWORD`, `EXACT`), `analyzer` (blank, `PREFIX`, `WORD`, `ENGLISH`, `EXACT`), `include_in_snippet` (blank: by role), `include_label`, `sort_order`, `source`, `owner_edited`. Unique on (`quick_search_index_id`, `field_name`). Edited from the index record's Searchable Fields list. |
| `quick_search_settings` | one row (id 1) | The global pause (`indexing_paused`, `paused_at`, `paused_by`, `pause_reason`) and the last self-priming report. Created by the QBit. |
| `quick_search_config_tombstone` | table or field an admin removed | Priming never adds back what a tombstone names. Delete the tombstone to let code seed it again. |
| `quick_search_index_run` (existing) | run | New columns `cancel_requested` and `cancel_requested_by` (Cancel Index Run). |
The config tables are audited at field level (QQQ's `audit` tables; without them the audit is skipped with a log line) and use `tablePermissionRules` (by default `quick_search_index_field.read`, `.edit`, and so on). The new processes use `adminPermissionRules` (`quick_search_admin.hasAccess`).
### Precedence: code versus data
1. **The records are the source of truth** once the config tables exist and priming has run. Indexing (real-time, basepull, reconcile, full reindex), search, the core record-search provider and the diagnostic tools read the effective config from them. It is cached for 30 seconds and dropped at once on the server where a config row changes, so an edit takes effect immediately there and within 30 seconds on other servers.
2. **Code seeds; it never overrides an owner.** At startup (and on each basepull until it succeeds), priming:
- adds a row (source `CODE`, `enabled` per `enabledByDefault`) and its field rows for every code-declared table that has none, and queues its first build;
- updates the interval and timestamp field of rows nobody edited (`owner_edited` false); `enabled` is set only when a row is created;
- adds code-declared fields that have no row, and updates the weight and label flag of field rows nobody edited; either change queues a rebuild of that table;
- never touches a row or field an admin added or edited (`owner_edited` true), and never re-adds anything a tombstone names;
- never deletes anything: removing an annotation leaves the rows for the admin to remove.
3. **Removals are remembered.** Remove Search Index, and deleting a field row, write a tombstone, so code does not bring them back.
4. **Without the config tables** (before you add the DDL below), or until priming succeeds, Quick Search uses the code-declared tables exactly as before RC.3. So the upgrade can be staged.
### Self-priming
- A new code-declared table gets its rows on startup and is built by the next basepull (status `PENDING_BUILD`, then `BUILDING`, then `ACTIVE`; the run is recorded as `FULL_REINDEX`). No manual step is needed.
- A new install, or an index that was deleted, is recreated by the basepull, and the tables that had documents in it are rebuilt. An index that is empty while a table last counted documents in it is rebuilt too.
- The Search Diagnostics widget on the Quick Search home reports it ("New: 2 tables added and building (...)"), and `QuickSearchStatus.read(runtime).bannerHtml()` gives the same lines for a host's own home widget.
- Existing rows from RC.2 or earlier get their field rows on the first start, with **no rebuild** when the index was built with exactly the code's fields.
### Admin actions
All are standard streamed-ETL processes with a review step and a result step, run from the table's Actions (never from the nav):
| Action | On | What it does |
|---|---|---|
| Add Search Index | Quick Search Index (no selection) | Pick a table you can read that is not indexed and is not internal or secret; set the interval, timestamp field and enabled flag. The review shows the estimated record count, the fields it starts with (record-label fields as titles, then text fields) and what is never indexed (hidden and `isSensitive` fields). Saving queues the first build (or runs it at once when no scheduler runs the basepull). |
| Searchable Fields (child list on the index record) | Quick Search Index Field | Add, edit or remove fields with the standard screens; a field must exist, be indexable and not sensitive, and not be listed twice. The change applies to search and new writes at once and marks the index `NEEDS_REINDEX`. |
| Apply Search Index Changes | Quick Search Index (selected) | The review previews the field changes since the last build and the records the rebuild reads; executing rebuilds the table. |
| Disable Search Index, Remove Search Index | Quick Search Index (selected) | Stop indexing at once. The input step's "Search documents" choice (delete now, or keep) is restated on the review step. Remove also deletes the field rows and pending failed events and leaves a tombstone. Enabling a disabled index again (edit Enabled) queues a build. |
| Cancel Index Run | Quick Search Index Run (selected `RUNNING` runs) | Sets a cooperative cancel flag that the run checks between batches; it ends `CANCELLED` with its counts. A cancelled all-tables rebuild keeps the previous index; a cancelled queued build waits for Apply Search Index Changes. |
| Pause Indexing, Resume Indexing | Quick Search Settings | Pauses all indexing (real-time and scheduled; a running rebuild stops after its batch) while search keeps serving the current index. Changes made while paused are kept (`DEFERRED` failed events) and indexed by the basepull after Resume. The home shows "Paused since ... by ...". |
Roles and analyzers: `TITLE` fields make the hit's label, `SUBTITLE` its subtitle (`QuickSearchResult.subtitle`), `DISPLAY` fields its short details (`QuickSearchResult.displayValues`); `KEYWORD` and `EXACT` fields stay out of the highlighted snippet and match on their own. The `WORD`, `ENGLISH` and `EXACT` analyzers use new document objects (`fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`); their mapping is added to an existing index at startup (no mapping-version change), and documents get them as they are rebuilt. Tables of plain fields produce the same documents and queries as before.
### DDL
The QBit ships no DDL. PostgreSQL, with the snake_case column names QQQ infers (`QInstanceEnricher.setInferredFieldBackendNames`); use the field names as written if your backend maps fields to camelCase columns. Liquibase formatted SQL:
```sql
-- liquibase formatted sql
-- changeset quick-search:1.1.0-RC.3-1
ALTER TABLE quick_search_index ADD COLUMN IF NOT EXISTS source VARCHAR(20);
ALTER TABLE quick_search_index ADD COLUMN IF NOT EXISTS owner_edited BOOLEAN;
ALTER TABLE quick_search_index_run ADD COLUMN IF NOT EXISTS cancel_requested BOOLEAN;
ALTER TABLE quick_search_index_run ADD COLUMN IF NOT EXISTS cancel_requested_by VARCHAR(250);
-- changeset quick-search:1.1.0-RC.3-2
CREATE TABLE IF NOT EXISTS quick_search_index_field (
id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
quick_search_index_id INTEGER NOT NULL,
field_name VARCHAR(255) NOT NULL,
weight INTEGER,
role VARCHAR(20),
analyzer VARCHAR(20),
include_in_snippet BOOLEAN,
include_label BOOLEAN,
sort_order INTEGER,
source VARCHAR(20),
owner_edited BOOLEAN,
create_date TIMESTAMP WITHOUT TIME ZONE,
modify_date TIMESTAMP WITHOUT TIME ZONE,
CONSTRAINT pk_quick_search_index_field PRIMARY KEY (id),
CONSTRAINT uq_quick_search_index_field UNIQUE (quick_search_index_id, field_name)
);
-- changeset quick-search:1.1.0-RC.3-3
CREATE TABLE IF NOT EXISTS quick_search_settings (
id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
indexing_paused BOOLEAN,
paused_at TIMESTAMP WITHOUT TIME ZONE,
paused_by VARCHAR(250),
pause_reason VARCHAR(500),
last_primed_at TIMESTAMP WITHOUT TIME ZONE,
last_priming_summary TEXT,
last_priming_new_tables INTEGER,
create_date TIMESTAMP WITHOUT TIME ZONE,
modify_date TIMESTAMP WITHOUT TIME ZONE,
CONSTRAINT pk_quick_search_settings PRIMARY KEY (id)
);
-- changeset quick-search:1.1.0-RC.3-4
CREATE TABLE IF NOT EXISTS quick_search_config_tombstone (
id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
table_name VARCHAR(255) NOT NULL,
field_name VARCHAR(255),
removed_by VARCHAR(250),
reason VARCHAR(500),
create_date TIMESTAMP WITHOUT TIME ZONE,
CONSTRAINT pk_quick_search_config_tombstone PRIMARY KEY (id)
);
CREATE INDEX IF NOT EXISTS idx_quick_search_config_tombstone_table ON quick_search_config_tombstone (table_name);
```
MySQL: use `INT AUTO_INCREMENT` for the ids, `DATETIME` for the timestamps and `TINYINT(1)` for the booleans, and drop `IF NOT EXISTS` from `ADD COLUMN`.
### Behaviour changes to be aware of
- `quick_search_index` no longer offers Create or Delete (`TABLE_INSERT` and `TABLE_DELETE` are off), so the API cannot insert or delete rows either. Rows come from priming and Add Search Index, and go through Remove Search Index.
- Basepull builds queued tables (`PENDING_BUILD`) itself and recreates a missing index. `NEEDS_REINDEX` still waits for an admin (Apply Search Index Changes, or Rebuild).
- With the config tables, a table that is not in the instance is not part of the effective config, and `ensureIndexRowExists` no longer creates rows (priming does, honouring tombstones).
- Hidden and `isSensitive` fields are never indexed, whether declared in code or added as rows.
- `quick_search_index_run` and `quick_search_failed_event` offer no Create or Edit (Delete stays). The Basepull, Full Reindex and Reconcile processes are table actions on Quick Search Index rather than app children; their table input is now `searchTableName` (API callers may keep sending `tableName`).
- `quick_search_searchable_table` is now a custom possible-value source over the effective tables (it was an enum of the code-declared ones).
- Tables added at runtime get core `searchFields` (for the global search) when they have none, but the per-table Inspect Search Document and Reindex in Search record actions appear only after the next restart.
### Table help (no action needed)
The qbit now sets bundle help (`help/quick-search.yaml`) on the Quick Search Admin app and each `quick_search_*` table, so an empty table says what it is for and how its rows get created. It fills only empty `help` slots; help a host already set on these tables is kept. There is no DDL change and no host include.
### Checklist for data-driven configuration
1. Add the DDL above (it is additive).
2. Grant `quick_search_index_field`, `quick_search_settings` and `quick_search_config_tombstone` permissions (read, and edit where admins should change them) next to the existing Quick Search grants.
3. Upgrade and start. The Search Diagnostics widget should not say the searchable tables come from code, and shows the first priming's summary.
4. Optionally move code-declared search config into the records and delete the annotations: the rows stay and are now the owner's.
-31
View File
@@ -1,31 +0,0 @@
# Session State
Updated: 2026-10-06
## Context
`1.0.0-RC.2` is released (2026-10-06). It fixes the five pre-GA findings from the PR #10 review, each on its own branch and PR into `develop`:
| Issue | PR | Merge commit | Fix |
|---|---|---|---|
| #15 | #18 | `2398e62` | Transport: warn on `hostnameVerification=false`, blank `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected |
| #17 | #21 | `e9ec54a` | `safeCount` logs at warn and keeps the previous `documentCount` |
| #16 | #19 | `9941a70` | Search `offset` counts readable results under record locks; bounded scan; 10,000 window cap |
| #14 | #20 | `dce0f26` | Deletes during a full reindex captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap |
| #13 | #22 | `c5c41d4` | Listener re-read failure and missing AWS SDK (`LinkageError`) recorded as failed events |
`develop` (`c5c41d4`) was merged into `release/1.0` as `b7d366c` (only difference from develop: `<revision>1.0.0-RC.1</revision>`). The orb's `publish_release_candidate` workflow succeeded on `b7d366c` (CircleCI build 82) and pushed `c11b2c2` ("Bump version to 1.0.0-RC.2 [skip ci]"). Maven Central serves the jar, pom, sources, javadoc and signatures for `com.kingsrook.qbits:qbit-quick-search:1.0.0-RC.2`.
## Release mechanics (qqq-orb 0.6.8, confirmed in its source)
- RCs publish artifacts only: no git tag and no GitHub release. The GitHub release comes when the release branch merges to `main` and a `vX.Y.Z` tag is pushed.
- Never push a `vX.Y.Z-RC.n` tag: it matches the GA job's `v*` filter, maps back to `release/X.Y.Z`, and would publish a stray `RC.n+1`.
- The next push to `release/1.0` publishes `1.0.0-RC.3`. Keep the release branch's `<revision>` on merge.
- `develop` requires one approving review and the PR author cannot approve their own PR; James merges with `gh pr merge <n> --squash --admin`. The agent cannot run `gh pr merge`, and the auto-mode classifier also refused a background loop that polled PR merge state.
- GitHub SSH times out from this machine; push over HTTPS with `git -c credential.helper='!gh auth git-credential' push https://github.com/QRun-IO/qbit-quick-search.git <ref>`.
## Next
1. qqq-all and Voyage pin `1.0.0-RC.2` and report back; their feedback decides RC.3 or GA.
2. Remaining "Before 1.0.0 GA" items in `docs/TODO.md`: stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review), and the `qbit-build-parent` 2.1.0 re-pin.
3. GA: merge `release/1.0` to `main` and tag `v1.0.0`.
+79
View File
@@ -0,0 +1,79 @@
# Quick Search host migration (kof22 #164)
Issue #50 / architecture finding F10. Paths below are under
`Website-Backend/src/main/java/com/kof22/website/metadata/adminhome/`;
line numbers refer to the local clone inspected on 2026-10-13. No host files were changed.
| Existing host code | Replacement |
| --- | --- |
| `LiveAdminHomeProbes.java:340` `searchConfigured()` | Use `QuickSearchApi.APP_NAME` for the default-name app lookup; for an authorized snapshot use `QuickSearchStatusService.get()` and its state/homeHref. `DISABLED` also includes all indexes disabled, so it is not exactly “qbit absent.” |
| `LiveAdminHomeProbes.java:351` `searchCluster()` | One `QuickSearchStatusService.get()`; map `clusterReachable`, `clusterHealth`, `nodes`, `activeShards`, `unassignedShards`. Remove direct cluster HTTP/auth/CA handling. |
| `LiveAdminHomeProbes.java:386` `searchDocumentCounts(...)` | Reuse the same snapshot's `indexes()` and each row's `tableName()` / `documentCount()` / `sourceCount()` / `countsCheckedAt()`. Remove `QuickSearchOpenSearchClient` access and relabel counts as saved diagnostics, not live. Preserve null as unknown. |
| `QuickSearchHomeWidgets.java:118` `widgets()` | Stop adding the host's duplicate Quick Search home widgets. Link to `status.homeHref()` (or default `QuickSearchApi.HOME_HREF`). Keep the qbit-provided home/diagnostics widget. |
| `QuickSearchHomeWidgets.java:133` `indexFacts()`, `:160` `summary()` and `:196` `syncTile()` | For a compact host overview, use the same `QuickSearchStatusService.get()` value: `state`, `indexes`, `pending`, `exhausted`, `lastSuccessfulSync`, `lastError`. Remove SQL aggregates and live source counts; use saved count timestamps and truncation flags. |
| `QuickSearchHomeWidgets.java:267` `runs()`, `:315` `failedEvents()` | Replace duplicate detailed views with the home link; drill down using the qbit's own Index Run / Failed Event tables from that home. The summary API intentionally does not export detailed histories. |
| `AdminOperationsWidgets.java:209` `quickSearch()` | Render `status.indexes()` for the compact admin overview, and use `status.homeHref()` for its footer link. Remove qbit-table SQL and source-table counts. |
| `AdminOperationsWidgets.java:343` `searchFooter()` | Use `clusterHealth`, `pending`, `exhausted`, `lastSuccessfulSync`, `lastError`, and `checkedAt` from the same snapshot. Detailed recent-run information belongs in the qbit home. |
Call with the existing request's `QContext`/`QSession`; do not elevate a host user's
session to bypass `adminPermissionRules`. Cache hits still check permission.
## UI contract
| Method | Path | operationId |
| --- | --- | --- |
| GET | `/qqq/v1/qbit/quick-search/status` | `getQuickSearchStatus` |
| GET | `/qqq/v1/qbit/quick-search/suggest` | `getQuickSearchSuggestions` |
Suggestions require only a signed-in session, with table READ permissions and
configured record locks applied to results. Do not gate the typeahead UI on the
status endpoint's admin permission. Send `q`, optionally `limitPerTable` (default 5,
maximum 10), `maxTables` (default 8, maximum 20), and comma-separated `tables`.
The response contains `query`, relevance-ordered `groups` and `truncated`; each
group has `tableName`, `tableLabel`, `hasMore` and `items` of `recordId`, `label`,
and nullable `subtitle`. See the [API details](../README.md#typeahead-suggestions)
for normalization, bounded-batch semantics and empty responses during outages.
Extension: `quick-search`, version `1.1.0`; core dependency: `4.1.0-RC.15` (UI API `1.6.0`).
The requested `/quick-search` prefix cannot validate on the supplied RC.6 baseline:
`MiddlewareSpecExtensionsMetaData.ESTABLISHED_PREFIXES` only admits esb, alerts,
assistant, and sql-workbench. We use the default namespace without modifying core.
The suggestions change adds a second endpoint and its schemas; the status
method, path, request and response are unchanged. Next UI `src/` contains no calls to a
Quick Search-specific endpoint today; its core `/qqq/v1/search` integration is unchanged.
Vendor `src/main/resources/openapi/qqq-v1/quick-search.yaml` for clients.
Regenerate the fragment with:
```powershell
./mvnw.cmd -B -Dtest=QuickSearchApiContractTest -Dquicksearch.openapi.update=true test
./mvnw.cmd -B -Dtest=QuickSearchApiContractTest test
```
The fragment covers both registered specs using the committed dependency version.
## Suggestions verification (2026-10-14)
The two-spec contract regenerates and passes without the update flag. Final
`CI=true ./mvnw.cmd -B verify` passes 495 unit/guardrail tests, including strict
HTTP checks for reader access and invalid suggestion parameters. All 9 integration
classes fail their Docker prerequisite because this sandbox cannot access Docker's
named pipe; the 3 new suggest integration tests compile but remain unexecuted.
The configured JaCoCo gates pass when invoked separately with
`./mvnw.cmd -B jacoco:report jacoco:check@check` (74.38% instruction, 91.88% class).
New API files and changed tests pass repository Checkstyle; the full source check
still reports 446 existing violations.
## Original status verification and limits
`./mvnw.cmd -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT` passes
468 unit tests and JaCoCo, including strict HTTP server startup, permission checks,
OpenAPI contract consistency, cached/disabled/degraded/down states, timeout recovery,
bounded backlog/table lists and prefixed QInstance isolation. All 41 Docker integration
tests skip locally because Docker is unavailable.
The separate Checkstyle check is **not green**: the existing checked-in rules report
471 main-source violations at HEAD and 438 in this worktree, with zero new violations.
New source files and the two new tests pass those rules. No repository-wide header
rewrite or unrelated style cleanup was included.
+36
View File
@@ -1,5 +1,32 @@
# TODO
## UI API 1.6 migration (2026-10-15)
- [x] Bump `quick-search` extension to 1.1.0, retain deprecated `QuickSearchErrorV1`, and use core `BasicErrorResponseV1` for errors; preserve operation IDs and success payloads.
- [x] Regenerate the two-operation contract with `-Dquicksearch.openapi.update=true`, then pass the contract test without regeneration; no dangling references or separate baseline were found.
- [x] Set the release BOM default to QQQ `4.1.0-RC.15`; validate offline against the locally installed `4.1.99-SNAPSHOT` using `./mvnw.cmd -o test -Dqqq.snapshot.version=4.1.99-SNAPSHOT -Denforcer.skipRules=requireReleaseDeps` (540 tests, no failures/errors/skips). The command-line exception is needed only because the requested local core is a SNAPSHOT; other guardrails remain enabled.
- [x] Verify strengthened HTTP error-body assertions: 8 status/HTTP tests passed after the full suite.
- [ ] Run Docker integration tests through `verify` in CI; this migration ran the requested `test` lifecycle.
## In-app help bundle (2026-10-14)
- [x] Pin QQQ RC.7, register the host-neutral QBIT bundle, replace the temporary loader, document HOST overrides, and add coverage/no-help-table checks.
- [ ] Host adoption: enable QQQ help startup sync and keep deployment schedules and host-table record actions in HOST bundles.
## Real-time error counter
- [x] Count failed real-time events after commit, serialize concurrent listeners within a JVM, tolerate counter-write failures, and reset after successful basepull/full reindex; add regression tests and field/documentation help.
## Architecture status API (#50 / F10)
- [x] Supported Java status API, bounded cache/rate limiting, administrator authorization, home navigation, spec endpoint and committed fragment.
- [x] Contract and strict HTTP tests; host migration mapping for kof22#164.
- [ ] kof22#164: replace internal probes/duplicate home using `docs/STATUS-API-HOST-MIGRATION.md`.
- [ ] Next UI: vendor the additive Quick Search status fragment when adopting RC.6.
- [ ] Run Docker integration tests in CI; local Docker is unavailable.
- [ ] Resolve the pre-existing repository Checkstyle baseline (438 remaining main-source violations, down from 471 at HEAD); status API adds none.
## Before tagging 1.0.0
- [ ] qqq-all and Voyage adopt `1.0.0-RC.2` (items below) and report back.
@@ -43,3 +70,12 @@
- [x] Status assessment, deep-dive audit (25 blockers), PRD (2026-10-04).
- [x] 4.1.0-RC.1 compatibility verified (2026-10-04).
- [x] All 25 audit blockers implemented and tested on the release branch (2026-10-04).
## Secured-data section 15 (2026-10-14)
- [x] Review fix: separate protection from indexability; preserve safe labels, ordinary results and unprotected hidden keys/timestamps, with regression tests.
- [x] Exclude PII/PHI, hidden, sensitive, password and field-locked values from indexing and results on RC.6.
- [x] Add canary coverage for real-time indexing, basepull and stale results; refuse classified config fields.
- [ ] On the QQQ RC.7 upgrade, honour `ClassifiedValues.isClassified(table, field)` (qqq#1182) and `FieldAccessHelper.getLockTier` (qqq#1172).
- [ ] Deployment: rebuild existing indexes and apply host retention policy to old physical indexes/backups containing previously indexed values.
Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 73 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 97 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 123 KiB

+12 -14
View File
@@ -18,6 +18,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
<revision>1.1.0-SNAPSHOT</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
@@ -30,7 +33,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.0.0</version>
<version>2.2.1</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -70,6 +73,11 @@
</scm>
<dependencies>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-javalin</artifactId>
</dependency>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
@@ -163,6 +171,7 @@
<directory>src/main/resources</directory>
<filtering>true</filtering>
<includes>
<include>help/*.yaml</include>
<include>quick-search.properties</include>
</includes>
</resource>
@@ -170,6 +179,7 @@
<directory>src/main/resources</directory>
<filtering>false</filtering>
<excludes>
<exclude>help/*.yaml</exclude>
<exclude>quick-search.properties</exclude>
</excludes>
</resource>
@@ -274,7 +284,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -287,18 +297,6 @@
</dependency>
</dependencies>
</dependencyManagement>
<repositories>
<repository>
<id>sonatype-snapshots</id>
<url>https://central.sonatype.com/repository/maven-snapshots/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</profile>
</profiles>
</project>
@@ -68,13 +68,20 @@ import org.quartz.CronExpression;
*******************************************************************************/
public class QuickSearchQBitConfig implements QBitConfig
{
public static final String DEFAULT_ADMIN_PERMISSION_BASE_NAME = "quickSearchAdmin";
public static final String DEFAULT_ADMIN_PERMISSION_BASE_NAME = "quick_search_admin";
public static final String DEFAULT_BASEPULL_TIMESTAMP_FIELD = "modifyDate";
public static final int DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS = 2500;
public static final String DEFAULT_DEBUG_PERMISSION_BASE_NAME = "quickSearchDebug";
public static final String DEFAULT_DEBUG_PERMISSION_BASE_NAME = "quick_search_debug";
/** Admin permission base name before 1.1.0-RC.3 (camelCase). */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_ADMIN_PERMISSION_BASE_NAME = "quickSearchAdmin";
/** Debug permission base name before 1.1.0-RC.3 (camelCase). */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_DEBUG_PERMISSION_BASE_NAME = "quickSearchDebug";
public static final int DEFAULT_VALIDATION_SAMPLE_SIZE = 50;
public static final int MAX_VALIDATION_SAMPLE_SIZE = 1000;
public static final int DEFAULT_DIAGNOSTIC_RUNS_PER_MINUTE = 30;
public static final int DEFAULT_MAX_AUTO_REPLICAS = 1;
///////////////////////////////////////////////////////////////////////////////
// a live full reindex refreshes its run record after every page; a page is //
@@ -107,6 +114,8 @@ public class QuickSearchQBitConfig implements QBitConfig
private Integer responseTimeoutMillis = 60000;
private Integer maxConnections = 30;
private Integer maxBulkRequestBytes = 5 * 1024 * 1024;
private Integer indexReplicas;
private Integer maxAutoReplicas = DEFAULT_MAX_AUTO_REPLICAS;
private QCodeReference transportCustomizer;
private QuickSearchStartupMode startupMode = QuickSearchStartupMode.FAIL_FAST;
private Boolean enableRealTimeIndexing = true;
@@ -437,6 +446,16 @@ public class QuickSearchQBitConfig implements QBitConfig
{
errors.add("basepullOverlapSeconds must not be negative for QuickSearchQBit");
}
if(indexReplicas != null && indexReplicas < 0)
{
errors.add("indexReplicas must not be negative for QuickSearchQBit");
}
if(maxAutoReplicas != null && maxAutoReplicas < 0)
{
errors.add("maxAutoReplicas must not be negative for QuickSearchQBit");
}
}
@@ -687,7 +706,35 @@ public class QuickSearchQBitConfig implements QBitConfig
/*******************************************************************************
** Apply the table-name prefix, if any.
** Apply the table-name prefix to a table or possible-value source name.
** Table names are lowercase snake_case (owner naming rule, 2026-10-12), so a
** prefix is converted to snake_case and joined with "_": prefix "acme" and
** quick_search_index give acme_quick_search_index.
*******************************************************************************/
public String applyTablePrefix(String name)
{
if(StringUtils.hasContent(tableNamePrefix))
{
String prefix = toSnakeCase(tableNamePrefix);
return (prefix.endsWith("_") ? prefix : prefix + "_") + name;
}
return name;
}
/*******************************************************************************
** Convert a camelCase name to lowercase snake_case.
*******************************************************************************/
static String toSnakeCase(String name)
{
return name.replaceAll("([a-z0-9])([A-Z])", "$1_$2").replace('-', '_').toLowerCase();
}
/*******************************************************************************
** Apply the name prefix, if any, to a process, widget or app name.
*******************************************************************************/
public String applyPrefix(String name)
{
@@ -701,21 +748,21 @@ public class QuickSearchQBitConfig implements QBitConfig
/*******************************************************************************
** Name of the produced quickSearchIndex table.
** Name of the produced quick_search_index table.
*******************************************************************************/
public String getQuickSearchIndexTableName()
{
return applyPrefix(QuickSearchIndex.TABLE_NAME);
return applyTablePrefix(QuickSearchIndex.TABLE_NAME);
}
/*******************************************************************************
** Name of the produced quickSearchIndexRun table.
** Name of the produced quick_search_index_run table.
*******************************************************************************/
public String getQuickSearchIndexRunTableName()
{
return applyPrefix(QuickSearchIndexRun.TABLE_NAME);
return applyTablePrefix(QuickSearchIndexRun.TABLE_NAME);
}
@@ -1461,6 +1508,73 @@ public class QuickSearchQBitConfig implements QBitConfig
/*******************************************************************************
** Getter for indexReplicas: number_of_replicas for the Quick Search index.
** Null (the default) means automatic: min(maxAutoReplicas, data nodes - 1),
** so a single-node cluster gets 0 replicas and stays green. Applied when a
** physical index is created and reconciled on existing indexes at startup.
** Ignored for OpenSearch Serverless (awsServiceName aoss).
*******************************************************************************/
public Integer getIndexReplicas()
{
return (this.indexReplicas);
}
/*******************************************************************************
** Setter for indexReplicas
*******************************************************************************/
public void setIndexReplicas(Integer indexReplicas)
{
this.indexReplicas = indexReplicas;
}
/*******************************************************************************
** Fluent setter for indexReplicas
*******************************************************************************/
public QuickSearchQBitConfig withIndexReplicas(Integer indexReplicas)
{
this.indexReplicas = indexReplicas;
return (this);
}
/*******************************************************************************
** Getter for maxAutoReplicas: the most replicas automatic mode (indexReplicas
** null) asks for; it never asks for more than data nodes - 1. Default 1.
*******************************************************************************/
public Integer getMaxAutoReplicas()
{
return (this.maxAutoReplicas);
}
/*******************************************************************************
** Setter for maxAutoReplicas
*******************************************************************************/
public void setMaxAutoReplicas(Integer maxAutoReplicas)
{
this.maxAutoReplicas = maxAutoReplicas;
}
/*******************************************************************************
** Fluent setter for maxAutoReplicas
*******************************************************************************/
public QuickSearchQBitConfig withMaxAutoReplicas(Integer maxAutoReplicas)
{
this.maxAutoReplicas = maxAutoReplicas;
return (this);
}
/*******************************************************************************
** Getter for transportCustomizer: QCodeReference to an OpenSearchTransportCustomizer
*******************************************************************************/
@@ -1818,7 +1932,7 @@ public class QuickSearchQBitConfig implements QBitConfig
/*******************************************************************************
** Getter for runHistoryRetentionDays: delete quickSearchIndexRun rows older than this many days; null keeps them forever
** Getter for runHistoryRetentionDays: delete quick_search_index_run rows older than this many days; null keeps them forever
*******************************************************************************/
public Integer getRunHistoryRetentionDays()
{
@@ -2458,7 +2572,7 @@ public class QuickSearchQBitConfig implements QBitConfig
/*******************************************************************************
** Getter for debugPermissionRules: permission rules for the read-only diagnostic tools and the diagnostics widget (default: quickSearchDebug.hasAccess)
** Getter for debugPermissionRules: permission rules for the read-only diagnostic tools and the diagnostics widget (default: quick_search_debug.hasAccess)
*******************************************************************************/
public QPermissionRules getDebugPermissionRules()
{
@@ -25,6 +25,16 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.health.QuickSearchHealthChecks;
import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
@@ -32,19 +42,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -215,6 +220,11 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
MetaDataProducerMultiOutput output = QBitMetaDataProducer.super.produce(qInstance);
output.addSelfToInstance(qInstance);
//////////////////////////////////////////////////////////////////////////
// Metadata only; core guards startup sync when helpContent is absent. //
//////////////////////////////////////////////////////////////////////////
HelpBundles.register(qInstance, "help/qbit-quick-search.yaml");
return (new MetaDataProducerMultiOutput());
}
@@ -254,6 +264,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
QuickSearchRuntime runtime = new QuickSearchRuntime(qBitConfig);
runtime.setDiscoveredTables(discoveredTables);
qBitConfig.setRuntime(runtime);
QuickSearchHealthChecks.register(qInstance);
if(Boolean.TRUE.equals(qBitConfig.getEnableRealTimeIndexing()))
{
@@ -262,6 +273,9 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
qInstance.withRuntimeService(new QCodeReference(QuickSearchRuntimeService.class));
MiddlewareSpecExtensionsMetaData.ofOrWithNew(qInstance)
.withExtension(QuickSearchApi.extension());
addRecordDiagnosticActions(output, qInstance, discoveredTables);
if(Boolean.TRUE.equals(qBitConfig.getServeCoreRecordSearch()))
@@ -347,7 +361,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
** never replaced. A table that is not in the instance yet, uses backend
** variants, or has no such field is skipped with a warning.
*******************************************************************************/
static void defaultCoreSearchFields(QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
public static void defaultCoreSearchFields(QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
{
for(QuickSearchableTableConfig tableConfig : tableConfigs)
{
@@ -375,8 +389,8 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
for(String fieldName : CollectionUtils.nonNullList(tableConfig.getSearchableFields()))
{
QFieldMetaData field = table.getFields() == null ? null : table.getFields().get(fieldName);
if(field != null && !Boolean.TRUE.equals(field.getIsHidden()) && field.getType() != null
&& (field.getType().isStringLike() || field.getType().isIntegral()) && !field.getType().needsMasked()
if(QuickSearchFieldRules.isIndexable(table, fieldName)
&& (field.getType().isStringLike() || field.getType().isIntegral())
&& !searchFields.contains(fieldName))
{
searchFields.add(fieldName);
@@ -461,7 +475,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
{
primaryKeyField = table.getPrimaryKeyField();
basepullTimestampField = resolveTimestampField(table, searchableFields, basepullTimestampField);
removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
removeExcludedFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
}
tables.add(new QuickSearchableTableConfig()
@@ -518,7 +532,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
warnIncrementalBasepullDisabled(table, basepullTimestampField);
basepullTimestampField = null;
}
removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
removeExcludedFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
}
Integer basepullInterval = stc.getBasepullIntervalMinutes();
@@ -598,23 +612,22 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
/*******************************************************************************
** Hidden and password fields are never indexed: their values would leak
** through searchable text and highlights.
** Classified, locked, hidden, sensitive and password fields are excluded:
** would leak through searchable text and highlights.
*******************************************************************************/
private static void removeHiddenFields(QTableMetaData table, List<String> searchableFields, Map<String, Integer> fieldWeights, Map<String, Boolean> fieldIncludeLabels)
private static void removeExcludedFields(QTableMetaData table, List<String> searchableFields, Map<String, Integer> fieldWeights, Map<String, Boolean> fieldIncludeLabels)
{
List<String> hidden = new ArrayList<>();
for(String fieldName : searchableFields)
{
QFieldMetaData field = table.getFields().get(fieldName);
if(field != null && (Boolean.TRUE.equals(field.getIsHidden()) || (field.getType() != null && field.getType().needsMasked())))
if(!QuickSearchFieldRules.isIndexable(table, fieldName))
{
hidden.add(fieldName);
}
}
if(!hidden.isEmpty())
{
LOG.warn("Hidden and password fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
QuickSearchFieldRules.warnExcludedFields(table);
searchableFields.removeAll(hidden);
hidden.forEach(fieldWeights::remove);
hidden.forEach(fieldIncludeLabels::remove);
@@ -23,10 +23,20 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigStore;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticEventRecorder;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsRateLimiter;
import com.kingsrook.qbits.quicksearch.diagnostics.StatusCache;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -37,11 +47,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticEventRecorder;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsRateLimiter;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -55,6 +61,11 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** QBitMetaData, so steps, listeners and actions resolve it from QContext with
** {@link #get()} rather than from JVM-global statics. Two QInstances in one
** JVM therefore no longer share state.
**
** The tables declared in code (annotations, searchableTables) are kept as
** the code defaults; getDiscoveredTables() and getTableConfig() return the
** effective configs, which come from the config records once they are
** primed (see QuickSearchConfigStore for the precedence).
*******************************************************************************/
public class QuickSearchRuntime
{
@@ -62,12 +73,27 @@ public class QuickSearchRuntime
private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
/*******************************************************************************
** Cluster cache name for every runtime's config cache (the config store's
** snapshot and settings, and the enabled flags). A config row edited on
** one node clears it on every node; the TTLs remain the safety net.
*******************************************************************************/
public static final String CONFIG_CACHE_NAME = "qbit-quick-search.config";
private static final Set<QuickSearchRuntime> RUNTIMES = ClusterCaches.shared(CONFIG_CACHE_NAME,
Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())),
QuickSearchRuntime::invalidateAllLocal, null);
public static final long RECORD_SEARCH_BACKOFF_MILLIS = 30_000L;
private final StatusCache statusCache = new StatusCache();
private final QuickSearchQBitConfig config;
private List<QuickSearchableTableConfig> discoveredTables = new ArrayList<>();
private Map<String, QuickSearchableTableConfig> tablesByName = new LinkedHashMap<>();
private List<QuickSearchableTableConfig> codeTables = new ArrayList<>();
private Map<String, QuickSearchableTableConfig> codeTablesByName = new LinkedHashMap<>();
private final QuickSearchConfigStore configStore;
private volatile QuickSearchOpenSearchClient client;
private volatile IndexEventPublisher publisher;
@@ -75,7 +101,8 @@ public class QuickSearchRuntime
private volatile boolean closed = false;
private volatile long recordSearchBackoffUntilMillis = 0L;
private final Map<String, CachedFlag> enabledCache = new ConcurrentHashMap<>();
private final Map<String, CachedFlag> enabledCache = new ConcurrentHashMap<>();
private final Map<String, Long> buildFailures = new ConcurrentHashMap<>();
private volatile DiagnosticEventRecorder diagnosticEventRecorder;
private final DiagnosticsRateLimiter diagnosticsRateLimiter = new DiagnosticsRateLimiter();
@@ -84,10 +111,21 @@ public class QuickSearchRuntime
private final AtomicLong backoffDeclinedClaims = new AtomicLong();
private final Instant startedAt = Instant.now();
private volatile Instant lastRecordSearchFailureTime;
private volatile QInstance startInstance;
private volatile String lastRecordSearchFailureMessage;
/*******************************************************************************
** Instance-owned bounded administrative status cache.
*******************************************************************************/
public StatusCache getStatusCache()
{
return (statusCache);
}
/*******************************************************************************
** A cached enabled flag with the time it was read.
*******************************************************************************/
@@ -102,7 +140,9 @@ public class QuickSearchRuntime
*******************************************************************************/
public QuickSearchRuntime(QuickSearchQBitConfig config)
{
this.config = config;
this.config = config;
this.configStore = new QuickSearchConfigStore(this);
RUNTIMES.add(this);
}
@@ -159,36 +199,120 @@ public class QuickSearchRuntime
/*******************************************************************************
** Getter for discoveredTables (never null)
** The effective searchable tables (never null): from the config records
** once primed, else the code-declared ones.
*******************************************************************************/
public List<QuickSearchableTableConfig> getDiscoveredTables()
{
return (Collections.unmodifiableList(this.discoveredTables));
return (Collections.unmodifiableList(configStore.getTables()));
}
/*******************************************************************************
** Replace the discovered tables.
** Replace the code-declared tables (the defaults priming seeds).
*******************************************************************************/
public void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
{
this.discoveredTables = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
this.tablesByName = new LinkedHashMap<>();
for(QuickSearchableTableConfig tableConfig : this.discoveredTables)
this.codeTables = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
this.codeTablesByName = new LinkedHashMap<>();
for(QuickSearchableTableConfig tableConfig : this.codeTables)
{
this.tablesByName.put(tableConfig.getTableName(), tableConfig);
this.codeTablesByName.put(tableConfig.getTableName(), tableConfig);
}
configStore.invalidate();
}
/*******************************************************************************
** Find the config for one table, or null.
** The tables declared in code (annotations and searchableTables), never null.
*******************************************************************************/
public List<QuickSearchableTableConfig> getCodeTables()
{
return (Collections.unmodifiableList(this.codeTables));
}
/*******************************************************************************
** One code-declared table's config, or null.
*******************************************************************************/
public QuickSearchableTableConfig getCodeTableConfig(String tableName)
{
return (tableName == null ? null : codeTablesByName.get(tableName));
}
/*******************************************************************************
** Find the effective config for one table, or null when it is not indexed.
*******************************************************************************/
public QuickSearchableTableConfig getTableConfig(String tableName)
{
return (tableName == null ? null : tablesByName.get(tableName));
return (configStore.getTable(tableName));
}
/*******************************************************************************
** The store of the effective config records.
*******************************************************************************/
public QuickSearchConfigStore getConfigStore()
{
return (configStore);
}
/*******************************************************************************
** Whether an admin has paused all indexing (search keeps serving).
*******************************************************************************/
public boolean isIndexingPaused()
{
return (configStore.isIndexingPaused());
}
/*******************************************************************************
** Drop every cached config (a config row changed): on this node now, and
** on the other nodes through the cluster bus.
*******************************************************************************/
public void invalidateConfig()
{
invalidateConfigLocal();
ClusterCaches.invalidate(CONFIG_CACHE_NAME);
}
/*******************************************************************************
** Drop this runtime's cached config on this node only.
*******************************************************************************/
void invalidateConfigLocal()
{
configStore.invalidate();
enabledCache.clear();
}
/*******************************************************************************
** Drop the cached config of every live runtime on this node (a cluster
** invalidation arrived).
*******************************************************************************/
private static void invalidateAllLocal()
{
List<QuickSearchRuntime> runtimes;
synchronized(RUNTIMES)
{
runtimes = new ArrayList<>(RUNTIMES);
}
for(QuickSearchRuntime runtime : runtimes)
{
runtime.invalidateConfigLocal();
}
}
@@ -273,7 +397,7 @@ public class QuickSearchRuntime
}
else
{
publisher = new SynchronousIndexEventPublisher(getClient(), discoveredTables, config.getBulkBatchSize());
publisher = new SynchronousIndexEventPublisher(getClient(), this::getTableConfig, config.getBulkBatchSize());
}
}
return (publisher);
@@ -284,16 +408,21 @@ public class QuickSearchRuntime
/*******************************************************************************
** Whether a table is enabled for indexing and search, per its
** quickSearchIndex row. A table with no row yet is enabled. Results are
** quick_search_index row. A table with no row yet is enabled. Results are
** cached for one minute; a failed lookup is treated as enabled so indexing
** does not stop because the operational table is unreachable.
*******************************************************************************/
public boolean isTableEnabled(String tableName)
{
if(tableName == null || !tablesByName.containsKey(tableName))
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
if(tableConfig == null)
{
return (false);
}
if(tableConfig.getEnabled() != null)
{
return (tableConfig.getEnabled());
}
long now = System.currentTimeMillis();
CachedFlag cached = enabledCache.get(tableName);
@@ -309,6 +438,46 @@ public class QuickSearchRuntime
/*******************************************************************************
** When a queued build of the table last failed on this server (0: never).
*******************************************************************************/
public long getLastBuildFailureMillis(String tableName)
{
return (buildFailures.getOrDefault(tableName, 0L));
}
/*******************************************************************************
** Remember that a queued build of the table failed now.
*******************************************************************************/
public void recordBuildFailure(String tableName)
{
buildFailures.put(tableName, System.currentTimeMillis());
}
/*******************************************************************************
** Forget a table's build failure (it built).
*******************************************************************************/
public void clearBuildFailure(String tableName)
{
buildFailures.remove(tableName);
}
/*******************************************************************************
** How many enabled flags are cached (for tests).
*******************************************************************************/
int getCachedEnabledFlagCount()
{
return (enabledCache.size());
}
/*******************************************************************************
** Forget cached enabled flags (after an operator toggles a table).
*******************************************************************************/
@@ -493,7 +662,7 @@ public class QuickSearchRuntime
/*******************************************************************************
** Read the enabled flag from the quickSearchIndex row.
** Read the enabled flag from the quick_search_index row.
*******************************************************************************/
private boolean readEnabledFlag(String tableName)
{
@@ -535,6 +704,8 @@ public class QuickSearchRuntime
*******************************************************************************/
public void start() throws QException
{
primeConfig();
try
{
ensureIndexReady();
@@ -567,6 +738,55 @@ public class QuickSearchRuntime
/*******************************************************************************
** Prime the config records from the code defaults (never throws; a failure
** is retried on later use). Runs in a system session when no QContext is
** set, as when a launcher starts the runtime service.
*******************************************************************************/
void primeConfig()
{
try
{
if(QContext.getQInstance() != null)
{
configStore.tryPrime();
return;
}
QInstance qInstance = startInstance;
if(qInstance == null)
{
return;
}
QContext.init(qInstance, new QSystemUserSession(qInstance));
try
{
configStore.tryPrime();
}
finally
{
QContext.clear();
}
}
catch(Exception e)
{
LOG.warn("Quick Search could not prime its config records at startup; it retries on first use", e);
}
}
/*******************************************************************************
** The instance to prime against when the runtime is started without a
** QContext (QuickSearchRuntimeService.start sets it).
*******************************************************************************/
public void setStartInstance(QInstance startInstance)
{
this.startInstance = startInstance;
}
/*******************************************************************************
** Close the publisher and client. Safe to call more than once.
*******************************************************************************/
@@ -577,6 +797,8 @@ public class QuickSearchRuntime
return;
}
closed = true;
RUNTIMES.remove(this);
statusCache.close();
if(publisher != null)
{
@@ -61,6 +61,7 @@ public class QuickSearchRuntimeService implements QRuntimeServiceInterface
LOG.warn("Quick Search runtime service started, but no produced Quick Search QBit was found in the QInstance");
return;
}
runtime.setStartInstance(qInstance);
runtime.start();
}
@@ -18,6 +18,8 @@ package com.kingsrook.qbits.quicksearch;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
/*******************************************************************************
@@ -26,6 +28,12 @@ import java.util.Map;
** Describes which fields to index, optional per-field boost weights,
** optional per-field label-inclusion flags, basepull scheduling, and
** whether the table is enabled by default.
**
** Code (annotations, searchableTables) builds these as defaults; at runtime
** the effective configs are built from the quick_search_index and
** quick_search_index_field records (QuickSearchConfigStore), which add each
** field's role, analyzer and snippet flag, the table's enabled flag and its
** index row id.
*******************************************************************************/
public class QuickSearchableTableConfig
{
@@ -40,6 +48,11 @@ public class QuickSearchableTableConfig
private String recordLabelFormat;
private List<String> recordLabelFields;
private Integer maxFieldLength;
private Map<String, String> fieldRoles;
private Map<String, String> fieldAnalyzers;
private Map<String, Boolean> fieldIncludeInSnippet;
private Boolean enabled;
private Integer quickSearchIndexId;
@@ -382,4 +395,222 @@ public class QuickSearchableTableConfig
return (this);
}
/*******************************************************************************
** Getter for fieldRoles
*******************************************************************************/
public Map<String, String> getFieldRoles()
{
return (this.fieldRoles);
}
/*******************************************************************************
** Setter for fieldRoles
*******************************************************************************/
public void setFieldRoles(Map<String, String> fieldRoles)
{
this.fieldRoles = fieldRoles;
}
/*******************************************************************************
** Fluent setter for fieldRoles
*******************************************************************************/
public QuickSearchableTableConfig withFieldRoles(Map<String, String> fieldRoles)
{
this.fieldRoles = fieldRoles;
return (this);
}
/*******************************************************************************
** Getter for fieldAnalyzers
*******************************************************************************/
public Map<String, String> getFieldAnalyzers()
{
return (this.fieldAnalyzers);
}
/*******************************************************************************
** Setter for fieldAnalyzers
*******************************************************************************/
public void setFieldAnalyzers(Map<String, String> fieldAnalyzers)
{
this.fieldAnalyzers = fieldAnalyzers;
}
/*******************************************************************************
** Fluent setter for fieldAnalyzers
*******************************************************************************/
public QuickSearchableTableConfig withFieldAnalyzers(Map<String, String> fieldAnalyzers)
{
this.fieldAnalyzers = fieldAnalyzers;
return (this);
}
/*******************************************************************************
** Getter for fieldIncludeInSnippet
*******************************************************************************/
public Map<String, Boolean> getFieldIncludeInSnippet()
{
return (this.fieldIncludeInSnippet);
}
/*******************************************************************************
** Setter for fieldIncludeInSnippet
*******************************************************************************/
public void setFieldIncludeInSnippet(Map<String, Boolean> fieldIncludeInSnippet)
{
this.fieldIncludeInSnippet = fieldIncludeInSnippet;
}
/*******************************************************************************
** Fluent setter for fieldIncludeInSnippet
*******************************************************************************/
public QuickSearchableTableConfig withFieldIncludeInSnippet(Map<String, Boolean> fieldIncludeInSnippet)
{
this.fieldIncludeInSnippet = fieldIncludeInSnippet;
return (this);
}
/*******************************************************************************
** Getter for enabled
*******************************************************************************/
public Boolean getEnabled()
{
return (this.enabled);
}
/*******************************************************************************
** Setter for enabled
*******************************************************************************/
public void setEnabled(Boolean enabled)
{
this.enabled = enabled;
}
/*******************************************************************************
** Fluent setter for enabled
*******************************************************************************/
public QuickSearchableTableConfig withEnabled(Boolean enabled)
{
this.enabled = enabled;
return (this);
}
/*******************************************************************************
** Getter for quickSearchIndexId
*******************************************************************************/
public Integer getQuickSearchIndexId()
{
return (this.quickSearchIndexId);
}
/*******************************************************************************
** Setter for quickSearchIndexId
*******************************************************************************/
public void setQuickSearchIndexId(Integer quickSearchIndexId)
{
this.quickSearchIndexId = quickSearchIndexId;
}
/*******************************************************************************
** Fluent setter for quickSearchIndexId
*******************************************************************************/
public QuickSearchableTableConfig withQuickSearchIndexId(Integer quickSearchIndexId)
{
this.quickSearchIndexId = quickSearchIndexId;
return (this);
}
/*******************************************************************************
** A field's role; BODY when none is set.
*******************************************************************************/
public QuickSearchFieldRole getRole(String fieldName)
{
return (QuickSearchFieldRole.fromName(fieldRoles == null ? null : fieldRoles.get(fieldName), QuickSearchFieldRole.BODY));
}
/*******************************************************************************
** A field's analyzer: the one set, else EXACT for an EXACT field, WORD for a
** KEYWORD field and PREFIX for any other.
*******************************************************************************/
public QuickSearchFieldAnalyzer getAnalyzer(String fieldName)
{
QuickSearchFieldRole role = getRole(fieldName);
QuickSearchFieldAnalyzer fallback = role == QuickSearchFieldRole.EXACT ? QuickSearchFieldAnalyzer.EXACT
: (role == QuickSearchFieldRole.KEYWORD ? QuickSearchFieldAnalyzer.WORD : QuickSearchFieldAnalyzer.PREFIX);
if(role == QuickSearchFieldRole.EXACT)
{
return (QuickSearchFieldAnalyzer.EXACT);
}
return (QuickSearchFieldAnalyzer.fromName(fieldAnalyzers == null ? null : fieldAnalyzers.get(fieldName), fallback));
}
/*******************************************************************************
** Whether a field's text goes into the highlighted snippet (searchableText).
** Defaults to true, except for KEYWORD and EXACT fields and EXACT-analyzed
** fields, which are matched on their own instead.
*******************************************************************************/
public boolean isIncludedInSnippet(String fieldName)
{
if(getAnalyzer(fieldName) == QuickSearchFieldAnalyzer.EXACT)
{
return (false);
}
Boolean flag = fieldIncludeInSnippet == null ? null : fieldIncludeInSnippet.get(fieldName);
if(flag != null)
{
return (flag);
}
QuickSearchFieldRole role = getRole(fieldName);
return (role != QuickSearchFieldRole.KEYWORD && role != QuickSearchFieldRole.EXACT);
}
/*******************************************************************************
** Whether every field is plain: in the snippet with the PREFIX analyzer, so
** documents and queries look exactly as they did before roles existed.
*******************************************************************************/
public boolean isAllPlainFields()
{
for(String fieldName : searchableFields == null ? List.<String>of() : searchableFields)
{
if(!isIncludedInSnippet(fieldName) || getAnalyzer(fieldName) != QuickSearchFieldAnalyzer.PREFIX)
{
return (false);
}
}
return (true);
}
}
@@ -46,6 +46,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
@@ -165,7 +166,7 @@ public class QuickSearchAction
if(input.getLimitPerTable() != null && input.getLimitPerTable() > 0)
{
return (searchPerTable(client, normalizedTerm, allowedTables, offset, Math.min(input.getLimitPerTable(), maxLimit), tableConfigs, applyLocks));
return (searchPerTable(client, normalizedTerm, allowedTables, offset, Math.min(input.getLimitPerTable(), maxLimit), tableConfigs, applyLocks, Boolean.TRUE.equals(input.getSinglePass())));
}
Page page = searchPage(client, normalizedTerm, allowedTables, offset, limit, tableConfigs, applyLocks, new ScanBudget(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW, MAX_LOCKED_ROUND_TRIPS));
@@ -177,8 +178,10 @@ public class QuickSearchAction
/*******************************************************************************
** Search each allowed table separately, skipping offset results and
** returning up to limitPerTable hits in each table.
** singlePass bounds locked searches to one batch of twice the requested
** page size; it never scans again to fill gaps left by inaccessible rows.
*******************************************************************************/
private QuickSearchOutput searchPerTable(QuickSearchOpenSearchClient client, String term, List<String> allowedTables, int offset, int limitPerTable, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks) throws QException
private QuickSearchOutput searchPerTable(QuickSearchOpenSearchClient client, String term, List<String> allowedTables, int offset, int limitPerTable, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks, boolean singlePass) throws QException
{
List<QuickSearchResult> results = new ArrayList<>();
long totalHits = 0;
@@ -188,6 +191,11 @@ public class QuickSearchAction
int window = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
int tableHits = Math.max(window / allowedTables.size(), Math.min(window, 2 * limitPerTable));
int tableTrips = Math.max(2, MAX_LOCKED_ROUND_TRIPS / allowedTables.size());
if(singlePass)
{
tableHits = Math.min(window, 2 * limitPerTable);
tableTrips = 1;
}
for(String tableName : allowedTables)
{
@@ -221,7 +229,7 @@ public class QuickSearchAction
if(!applyLocks)
{
SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, pageSize, offset, tableConfigs, null);
List<QuickSearchResult> results = toResults(response);
List<QuickSearchResult> results = toResults(response, tableConfigs);
long totalHits = response.hits().total() != null ? response.hits().total().value() : results.size();
boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
@@ -255,7 +263,7 @@ public class QuickSearchAction
rawFloor = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
List<QuickSearchResult> candidates = new ArrayList<>();
for(QuickSearchResult result : toResults(response))
for(QuickSearchResult result : toResults(response, tableConfigs))
{
if(seen.add(result.getTableName() + ":" + result.getRecordId()))
{
@@ -315,7 +323,7 @@ public class QuickSearchAction
List<String> allowed = new ArrayList<>();
for(String tableName : new LinkedHashSet<>(requested))
{
if(!configured.contains(tableName))
if(!configured.contains(tableName) || QuickSearchFieldRules.hasProtectedPrimaryKey(tableName))
{
continue;
}
@@ -407,7 +415,7 @@ public class QuickSearchAction
/*******************************************************************************
**
*******************************************************************************/
private static List<QuickSearchResult> toResults(SearchResponse<OpenSearchDocument> response)
private static List<QuickSearchResult> toResults(SearchResponse<OpenSearchDocument> response, List<QuickSearchableTableConfig> tableConfigs)
{
QInstance qInstance = QContext.getQInstance();
List<QuickSearchResult> results = new ArrayList<>();
@@ -420,9 +428,16 @@ public class QuickSearchAction
continue;
}
String highlightSnippet = null;
String sourceTable = source.getSourceTable();
boolean filterFields = QuickSearchFieldRules.requiresFieldFiltering(sourceTable);
source = IndexingUtils.filterResult(source, tableConfigs.stream().filter(c -> sourceTable.equals(c.getTableName())).findFirst().orElse(null));
if(source == null)
{
continue;
}
String highlightSnippet = filterFields ? source.getSearchableText() : null;
Map<String, List<String>> highlight = hit.highlight();
if(highlight != null)
if(highlight != null && !filterFields)
{
List<String> fragments = highlight.get("searchableText");
if(fragments != null && !fragments.isEmpty())
@@ -441,7 +456,9 @@ public class QuickSearchAction
.withRecordId(source.getRecordId())
.withRecordLabel(source.getRecordLabel())
.withScore(rawScore != null ? rawScore.floatValue() : null)
.withHighlightSnippet(highlightSnippet));
.withHighlightSnippet(highlightSnippet)
.withSubtitle(source.getSubtitle())
.withDisplayValues(source.getDisplayValues()));
}
return (results);
@@ -35,6 +35,28 @@ public class QuickSearchInput extends AbstractActionInput
private Integer offset;
private List<String> tableNames;
private Integer limitPerTable;
private Boolean singlePass = false;
/*******************************************************************************
** Whether per-table searches stop after one bounded candidate batch.
*******************************************************************************/
public Boolean getSinglePass()
{
return (singlePass);
}
/*******************************************************************************
** Opt in to bounded typeahead latency, accepting partially filled pages.
*******************************************************************************/
public QuickSearchInput withSinglePass(Boolean singlePass)
{
this.singlePass = singlePass;
return (this);
}
/*******************************************************************************
@@ -18,6 +18,7 @@ package com.kingsrook.qbits.quicksearch.actions;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -25,6 +26,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlight;
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchHighlightMatch;
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
@@ -32,6 +35,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchHit;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.RecordSearchTableHits;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
@@ -50,10 +57,14 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
**
** A table is claimed when it is indexed by this QBit and enabled, and core
** record search is not backing off after a recent failure. search() sends
** one _msearch request with a top-N search per table and returns record ids
** only, best first; core re-reads them by primary key through QueryAction,
** so table permissions, record security locks and record labels are core's
** job, and ids of deleted records drop out. Each table gets
** one _msearch request with a top-N search per table (after skipping the
** input's offset) and returns record ids, best first, each with its score
** and highlights (plain text plus match ranges, from the indexed field
** values), and each table's total hits; core re-reads the ids by primary key
** through QueryAction, so table permissions, record security locks and
** record labels are core's job, and ids of deleted records drop out (core
** also keeps only highlights of fields the session can see). The index has
** no record modify dates, so a RECENT sort gets relevance order. Each table gets
** IDS_PER_LIMIT_MULTIPLIER x limitPerTable ids (at most
** RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE) so dropouts do not leave
** the table short.
@@ -69,6 +80,8 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
public static final int IDS_PER_LIMIT_MULTIPLIER = 4;
private static final String FIELD_VALUES_PREFIX = "fieldValues.";
/*******************************************************************************
@@ -85,7 +98,7 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
return (false);
}
if(runtime.getTableConfig(table.getName()) == null)
if(runtime.getTableConfig(table.getName()) == null || QuickSearchFieldRules.hasProtectedPrimaryKey(table.getName()))
{
return (false);
}
@@ -120,7 +133,8 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
List<String> tableNames = new ArrayList<>();
for(String tableName : new LinkedHashSet<>(CollectionUtils.nonNullList(input.getTableNames())))
{
if(runtime.getTableConfig(tableName) != null)
if(runtime.getTableConfig(tableName) != null
&& !QuickSearchFieldRules.hasProtectedPrimaryKey(tableName))
{
tableNames.add(tableName);
}
@@ -134,11 +148,13 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
int idsPerTable = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * IDS_PER_LIMIT_MULTIPLIER);
int timeoutMillis = config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis();
Map<String, List<String>> idsByTable;
long start = System.currentTimeMillis();
int offset = input.getOffset() == null ? 0 : Math.max(0, input.getOffset());
Map<String, RecordSearchTableHits> hitsByTable;
long start = System.currentTimeMillis();
try
{
idsByTable = runtime.getClient().searchRecordIdsPerTable(term, tableNames, idsPerTable, runtime.getDiscoveredTables(), timeoutMillis);
hitsByTable = runtime.getClient().searchRecordsPerTable(term, tableNames, idsPerTable, offset, true, runtime.getDiscoveredTables(), timeoutMillis);
}
catch(QException | RuntimeException e)
{
@@ -155,15 +171,115 @@ public class QuickSearchRecordSearchProvider implements RecordSearchProviderInte
throw (e instanceof QException qe ? qe : new QException("Quick Search record search failed: " + e.getMessage(), e));
}
Map<String, Integer> totalCountByTable = new HashMap<>();
for(String tableName : tableNames)
{
for(String recordId : CollectionUtils.nonNullList(idsByTable.get(tableName)))
RecordSearchTableHits tableHits = hitsByTable.get(tableName);
if(tableHits == null)
{
results.add(new RecordSearchResult().withTableName(tableName).withRecordId(recordId));
continue;
}
if(tableHits.totalHits() != null)
{
totalCountByTable.put(tableName, (int) Math.min(Integer.MAX_VALUE, tableHits.totalHits()));
}
for(RecordSearchHit hit : CollectionUtils.nonNullList(tableHits.hits()))
{
List<RecordSearchHighlight> highlights = highlights(hit.highlights());
results.add(new RecordSearchResult()
.withTableName(tableName)
.withRecordId(hit.recordId())
.withScore(hit.score())
.withHighlights(highlights.isEmpty() ? null : highlights));
}
}
return (new RecordSearchOutput().withResults(results));
return (new RecordSearchOutput().withResults(results).withTotalCountByTable(totalCountByTable));
}
/*******************************************************************************
** A hit's highlight fragments (by index field) as core record search
** highlights: one per fieldValues.{fieldName} field, as plain text with
** match ranges. Other index fields, and fragments without a match, are
** skipped.
*******************************************************************************/
public static List<RecordSearchHighlight> highlights(Map<String, List<String>> fragmentsByIndexField)
{
List<RecordSearchHighlight> highlights = new ArrayList<>();
for(Map.Entry<String, List<String>> entry : CollectionUtils.nonNullMap(fragmentsByIndexField).entrySet())
{
String indexField = entry.getKey();
if(indexField == null || !indexField.startsWith(FIELD_VALUES_PREFIX) || indexField.length() == FIELD_VALUES_PREFIX.length() || CollectionUtils.nullSafeIsEmpty(entry.getValue()))
{
continue;
}
RecordSearchHighlight highlight = highlight(indexField.substring(FIELD_VALUES_PREFIX.length()), entry.getValue().get(0));
if(highlight != null)
{
highlights.add(highlight);
}
}
return (highlights);
}
/*******************************************************************************
** A highlight from a fragment whose matches are wrapped in the client's
** pre/post tags: the fragment's text without the tags, and the ranges they
** wrapped (adjacent ranges merged). Null if the fragment is blank or has
** no match.
*******************************************************************************/
public static RecordSearchHighlight highlight(String fieldName, String fragment)
{
if(fragment == null)
{
return (null);
}
StringBuilder text = new StringBuilder();
List<RecordSearchHighlightMatch> matches = new ArrayList<>();
int open = -1;
for(int i = 0; i < fragment.length(); i++)
{
char c = fragment.charAt(i);
if(c == QuickSearchOpenSearchClient.HIGHLIGHT_PRE_TAG.charAt(0))
{
open = text.length();
}
else if(c == QuickSearchOpenSearchClient.HIGHLIGHT_POST_TAG.charAt(0))
{
if(open >= 0 && text.length() > open)
{
RecordSearchHighlightMatch last = matches.isEmpty() ? null : matches.get(matches.size() - 1);
if(last != null && last.getEnd() == open)
{
last.setEnd(text.length());
}
else
{
matches.add(new RecordSearchHighlightMatch().withStart(open).withEnd(text.length()));
}
}
open = -1;
}
else
{
text.append(c);
}
}
if(matches.isEmpty() || text.toString().isBlank())
{
return (null);
}
return (new RecordSearchHighlight().withFieldName(fieldName).withText(text.toString()).withMatches(matches));
}
}
@@ -17,6 +17,9 @@
package com.kingsrook.qbits.quicksearch.actions;
import java.util.Map;
/*******************************************************************************
** A single search hit returned by a QuickSearch query, representing one
** matching record with its relevance score and optional highlight snippet.
@@ -30,6 +33,8 @@ public class QuickSearchResult
private String recordLabel;
private Float score;
private String highlightSnippet;
private String subtitle;
private Map<String, String> displayValues;
/*******************************************************************************
@@ -202,4 +207,66 @@ public class QuickSearchResult
return this;
}
/*******************************************************************************
** Getter for subtitle
*******************************************************************************/
public String getSubtitle()
{
return (this.subtitle);
}
/*******************************************************************************
** Setter for subtitle
*******************************************************************************/
public void setSubtitle(String subtitle)
{
this.subtitle = subtitle;
}
/*******************************************************************************
** Fluent setter for subtitle
*******************************************************************************/
public QuickSearchResult withSubtitle(String subtitle)
{
this.subtitle = subtitle;
return (this);
}
/*******************************************************************************
** Getter for displayValues
*******************************************************************************/
public Map<String, String> getDisplayValues()
{
return (this.displayValues);
}
/*******************************************************************************
** Setter for displayValues
*******************************************************************************/
public void setDisplayValues(Map<String, String> displayValues)
{
this.displayValues = displayValues;
}
/*******************************************************************************
** Fluent setter for displayValues
*******************************************************************************/
public QuickSearchResult withDisplayValues(Map<String, String> displayValues)
{
this.displayValues = displayValues;
return (this);
}
}
@@ -0,0 +1,54 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension;
/*******************************************************************************
** Supported host navigation and UI API extension identity.
*******************************************************************************/
public final class QuickSearchApi
{
public static final String APP_NAME = QuickSearchAdminAppMetaDataProducer.APP_NAME;
public static final String HOME_HREF = "/app/" + APP_NAME;
/*******************************************************************************
** Utility class.
*******************************************************************************/
private QuickSearchApi()
{
}
/*******************************************************************************
** The spec registration shared by production and contract generation.
*******************************************************************************/
public static MiddlewareSpecExtension extension()
{
return (new MiddlewareSpecExtension().withName("quick-search").withSemanticVersion("1.1.0")
.withDescription("Quick Search administration and suggestions").withSpec(QuickSearchStatusSpec.class).withSpec(QuickSearchSuggestSpec.class));
}
}
@@ -0,0 +1,116 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import java.time.Instant;
import java.util.List;
import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema;
import com.kingsrook.qqq.openapi.model.Schema;
/*******************************************************************************
** Immutable administrative snapshot. Null measurements are unknown, never zero.
** Counts come from persisted diagnostics; countsCheckedAt identifies their age.
**
** unassignedReplicaShards is the unassigned shard count when every primary is
** active (so all unassigned shards are replicas), else null; targetReplicas and
** replicaSettingsFailed report the index replica sizing (null when unknown).
*******************************************************************************/
public record QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
Integer unassignedReplicaShards, Integer targetReplicas, Boolean replicaSettingsFailed,
Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref) implements ToSchema
{
/*******************************************************************************
** The 1.1.0-RC.14 shape, without the replica measurements (all unknown).
*******************************************************************************/
public QuickSearchStatus(State state, Boolean clusterReachable, String clusterHealth, Integer nodes, Integer activeShards, Integer unassignedShards,
Boolean indexingPaused, List<IndexRow> indexes, Integer pending, Integer exhausted, Boolean backlogTruncated,
Boolean tablesTruncated, Instant lastSuccessfulSync, String lastError, Instant checkedAt, String homeHref)
{
this(state, clusterReachable, clusterHealth, nodes, activeShards, unassignedShards, null, null, null, indexingPaused, indexes, pending, exhausted,
backlogTruncated, tablesTruncated, lastSuccessfulSync, lastError, checkedAt, homeHref);
}
/*******************************************************************************
** Yellow only because replicas cannot be placed on a single-node cluster:
** one node, every primary active and every unassigned shard a replica.
** Such a cluster cannot do better, so it is healthy rather than degraded.
*******************************************************************************/
public static boolean isSingleNodeReplicaYellow(String clusterHealth, Integer nodes, Integer unassignedReplicaShards)
{
return ("yellow".equals(clusterHealth) && Integer.valueOf(1).equals(nodes) && unassignedReplicaShards != null && unassignedReplicaShards > 0);
}
/*******************************************************************************
** Published wire schema for the same immutable value used by Java hosts.
*******************************************************************************/
@Override
public Schema toSchema()
{
return (new QuickSearchStatusSpec().defineComponentSchemas().get("QuickSearchStatusV1"));
}
/*******************************************************************************
** Protect the snapshot from mutations by callers.
*******************************************************************************/
public QuickSearchStatus
{
indexes = List.copyOf(indexes);
}
/*******************************************************************************
** Overall availability; DEGRADED also covers incomplete measurements.
*******************************************************************************/
public enum State
{
OK, DEGRADED, DOWN, DISABLED
}
/*******************************************************************************
** A persisted index row with counts from the latest saved validation, when
** available. Otherwise documentCount is the last indexing run's count and
** sourceCount/countsCheckedAt are unknown. Sync time is a successful watermark.
*******************************************************************************/
public record IndexRow(String tableName, Boolean enabled, String status, Long documentCount, Long sourceCount, Instant countsCheckedAt,
Instant lastSuccessfulSync, String lastError, Integer realTimeErrorCount) implements ToSchema
{
/*******************************************************************************
** Published per-table schema.
*******************************************************************************/
@Override
public Schema toSchema()
{
return (new QuickSearchStatusSpec().defineComponentSchemas().get("QuickSearchIndexStatusV1"));
}
}
}
@@ -0,0 +1,70 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.diagnostics.StatusDiagnostics;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
/*******************************************************************************
** Supported entry point for hosts. Uses the current QContext and checks the
** Quick Search admin app permission on EVERY call, including cache hits.
*******************************************************************************/
public final class QuickSearchStatusService
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private QuickSearchStatusService()
{
}
/*******************************************************************************
** Return a briefly cached snapshot, waiting at most two seconds for a refresh.
** Call with an authenticated QSession (or an explicitly trusted system session).
*******************************************************************************/
public static QuickSearchStatus get() throws QException
{
if(QContext.getQSession() == null)
{
throw (new QPermissionDeniedException("A session is required for Quick Search status."));
}
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime == null)
{
return (StatusDiagnostics.unavailable(QuickSearchStatus.State.DISABLED, null, QuickSearchApi.HOME_HREF));
}
String appName = runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
if(QContext.getQInstance().getApp(appName) == null || !PermissionsHelper.hasAppPermission(new QueryInput(), appName))
{
throw (new QPermissionDeniedException("Quick Search admin permission is required."));
}
return (runtime.getStatusCache().get(runtime));
}
}
@@ -0,0 +1,187 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import com.kingsrook.qqq.openapi.model.Method;
import com.kingsrook.qqq.openapi.model.Parameter;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
import io.javalin.http.HttpStatus;
/*******************************************************************************
** Administrative status, available through the spec-managed UI API only.
*******************************************************************************/
public class QuickSearchStatusSpec extends AbstractJsonEndpointSpec
{
/*******************************************************************************
** Describe the supported operation.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/quick-search/status")
.withShortSummary("Get Quick Search status")
.withLongDescription("Requires Quick Search admin permission. Five-second instance cache; two-second refresh wait. Counts are saved diagnostics, not live source scans. Unknown values are null. Backlog counts are lower bounds when backlogTruncated is true."));
}
/*******************************************************************************
** Stable generated-client operation name.
*******************************************************************************/
@Override
protected void customizeMethod(Method method)
{
method.setOperationId("getQuickSearchStatus");
}
/*******************************************************************************
** No request parameters.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
return (List.of());
}
/*******************************************************************************
** Versioned response component.
*******************************************************************************/
@Override
protected String getResponseSchemaName()
{
return ("QuickSearchStatusV1");
}
/*******************************************************************************
** Use exactly the same authorized service as in-process hosts.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (QuickSearchStatusService.get());
}
/*******************************************************************************
** Standard framework errors, including the diagnostics rate limit.
*******************************************************************************/
@Override
public List<BasicResponse> defineAdditionalBasicResponses()
{
return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Diagnostic rate limit exceeded", BasicErrorResponseV1.class.getSimpleName()),
new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", BasicErrorResponseV1.class.getSimpleName()),
new BasicResponse(HttpStatus.FORBIDDEN, "Quick Search admin permission required", BasicErrorResponseV1.class.getSimpleName())));
}
/*******************************************************************************
** Explicit schemas make nullability, bounds and measurement age discoverable.
*******************************************************************************/
@Override
public Map<String, Schema> defineComponentSchemas()
{
Map<String, Schema> status = new TreeMap<>();
status.put("state", field(Type.STRING, "Overall availability; incomplete diagnostics are DEGRADED.", false)
.withEnumValues(List.of("OK", "DEGRADED", "DOWN", "DISABLED")));
status.put("clusterReachable", field(Type.BOOLEAN, "Whether OpenSearch answered; null when not probed.", true));
status.put("clusterHealth", field(Type.STRING, "OpenSearch health color; null when unknown.", true));
for(String name : List.of("nodes", "activeShards", "unassignedShards"))
{
status.put(name, field(Type.INTEGER, "Cluster health counter; null when unavailable.", true));
}
status.put("unassignedReplicaShards", field(Type.INTEGER, "Unassigned shards when every primary is active (all are replicas); null when unknown or a primary is not active.", true));
status.put("targetReplicas", field(Type.INTEGER, "number_of_replicas Quick Search applies to its index; null when unknown or not managed (Serverless).", true));
status.put("replicaSettingsFailed", field(Type.BOOLEAN, "Sizing or applying the index replicas failed at startup; null when not attempted.", true));
status.put("indexingPaused", field(Type.BOOLEAN, "Indexing is paused; search can still serve.", true));
status.put("indexes", new Schema().withType(Type.ARRAY).withItems(new Schema().withRef("#/components/schemas/QuickSearchIndexStatusV1"))
.withDescription("At most 100 persisted index rows, ordered by id."));
status.put("pending", field(Type.INTEGER, "Pending and awaiting-reindex failed events, from at most 1000 backlog rows.", true));
status.put("exhausted", field(Type.INTEGER, "Exhausted failed events, from the same bounded backlog read.", true));
status.put("backlogTruncated", field(Type.BOOLEAN, "Counts are lower bounds when more than 1000 backlog rows exist.", false));
status.put("tablesTruncated", field(Type.BOOLEAN, "More than 100 index rows exist.", false));
status.put("lastSuccessfulSync", timestamp("Newest successful indexing watermark; not a guarantee all tables are current."));
status.put("lastError", field(Type.STRING, "Last known indexing/search error or snapshot failure; null when absent.", true));
status.put("checkedAt", timestamp("Snapshot completion time; remains unchanged on cache hits.").withNullable(false));
status.put("homeHref", field(Type.STRING, "Quick Search home link, including the configured metadata prefix.", false));
Map<String, Schema> row = new TreeMap<>();
row.put("tableName", field(Type.STRING, "Indexed source table metadata name.", false));
row.put("enabled", field(Type.BOOLEAN, "Whether this index is enabled.", false));
row.put("status", field(Type.STRING, "Persisted indexing state, such as ACTIVE or NEEDS_REINDEX.", true));
row.put("documentCount", field(Type.INTEGER, "Saved validation document count, else last indexing run count; not live.", true).withFormat("int64"));
row.put("sourceCount", field(Type.INTEGER, "Source count from saved IndexValidator report; null without validation.", true).withFormat("int64"));
row.put("countsCheckedAt", timestamp("Saved validation completion; null when validation counts are unavailable."));
row.put("lastSuccessfulSync", timestamp("Newest successful basepull/full-reindex/reconcile watermark."));
row.put("lastError", field(Type.STRING, "Persisted indexing error; null when absent.", true));
row.put("realTimeErrorCount", field(Type.INTEGER, "Persisted real-time indexing error count.", true));
return (Map.of("QuickSearchStatusV1", object(status), "QuickSearchIndexStatusV1", object(row),
"QuickSearchErrorV1", object(Map.of("error", field(Type.STRING, "Framework error message.", false)))
.withDescription("Deprecated since 1.1.0: errors use BasicErrorResponseV1.")));
}
/*******************************************************************************
** A response object always includes its properties, including nulls.
*******************************************************************************/
private static Schema object(Map<String, Schema> properties)
{
return (new Schema().withType(Type.OBJECT).withProperties(new TreeMap<>(properties)));
}
/*******************************************************************************
** A scalar with explicit unknown semantics.
*******************************************************************************/
private static Schema field(Type type, String description, Boolean nullable)
{
return (new Schema().withType(type).withDescription(description).withNullable(nullable));
}
/*******************************************************************************
** ISO-8601 instant.
*******************************************************************************/
private static Schema timestamp(String description)
{
return (field(Type.STRING, description, true).withFormat("date-time"));
}
}
@@ -0,0 +1,64 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import java.util.List;
/*******************************************************************************
** Minimal immutable suggestion response. Search internals never cross this API.
*******************************************************************************/
public record QuickSearchSuggest(String query, List<Group> groups, Boolean truncated)
{
/*******************************************************************************
** Freeze the group order for callers.
*******************************************************************************/
public QuickSearchSuggest
{
groups = List.copyOf(groups);
}
/*******************************************************************************
** One readable table, ordered by its highest scoring accessible hit.
*******************************************************************************/
public record Group(String tableName, String tableLabel, Boolean hasMore, List<Item> items)
{
/*******************************************************************************
** Freeze the per-table relevance order.
*******************************************************************************/
public Group
{
items = List.copyOf(items);
}
}
/*******************************************************************************
** Only already-filtered labels and subtitles are exposed.
*******************************************************************************/
public record Item(String recordId, String label, String subtitle)
{
}
}
@@ -0,0 +1,161 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
import com.kingsrook.qbits.quicksearch.actions.QuickSearchOutput;
import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
/*******************************************************************************
** Session-only typeahead over the existing permission-aware search action.
*******************************************************************************/
public final class QuickSearchSuggestService
{
private static final QLogger LOG = QLogger.getLogger(QuickSearchSuggestService.class);
/*******************************************************************************
** Utility class.
*******************************************************************************/
private QuickSearchSuggestService()
{
}
/*******************************************************************************
** One candidate batch per table, with one extra visible hit for hasMore.
** A short term or unavailable search returns an empty response. hasMore only
** confirms an extra accessible hit in this bounded batch, never hidden rows.
*******************************************************************************/
public static QuickSearchSuggest suggest(String query, Integer limitPerTable, Integer maxTables, String tables) throws QException
{
if(QContext.getQSession() == null)
{
throw (new QPermissionDeniedException("A session is required for Quick Search suggestions."));
}
if(query == null)
{
throw (new QBadRequestException("q is required."));
}
String normalized = IndexingUtils.normalizeSearchText(query);
if(normalized.length() > QuickSearchAction.MAX_TERM_LENGTH)
{
throw (new QBadRequestException("Search term is too long (maximum 100 characters)."));
}
Integer itemLimit = bounded(limitPerTable, 5, 10, "limitPerTable");
Integer groupLimit = bounded(maxTables, 8, 20, "maxTables");
QuickSearchSuggest empty = new QuickSearchSuggest(normalized, List.of(), false);
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(normalized.length() < QuickSearchAction.MIN_TERM_LENGTH || runtime == null)
{
return (empty);
}
List<String> tableNames = tables == null ? null : Arrays.stream(tables.split(",")).map(String::trim).filter(name -> !name.isEmpty()).distinct().toList();
if(tableNames != null && tableNames.isEmpty())
{
return (empty);
}
QuickSearchOutput output;
try
{
if(runtime.getClient() == null)
{
return (empty);
}
output = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(normalized)
.withTableNames(tableNames).withLimitPerTable(itemLimit + 1).withSinglePass(true));
}
catch(QPermissionDeniedException e)
{
throw (e);
}
catch(QException | RuntimeException e)
{
LOG.debug("Quick Search suggestions unavailable", e);
return (empty);
}
Map<String, List<QuickSearchResult>> byTable = new LinkedHashMap<>();
for(QuickSearchResult result : output.getResults())
{
byTable.computeIfAbsent(result.getTableName(), key -> new ArrayList<>()).add(result);
}
List<List<QuickSearchResult>> ranked = new ArrayList<>(byTable.values());
ranked.sort(Comparator.<List<QuickSearchResult>, Float>comparing(
hits -> hits.stream().map(QuickSearchResult::getScore).filter(score -> score != null).max(Float::compare).orElse(Float.NEGATIVE_INFINITY))
.reversed().thenComparing(hits -> hits.get(0).getTableName()));
List<QuickSearchSuggest.Group> groups = new ArrayList<>();
for(List<QuickSearchResult> hits : ranked.subList(0, Math.min(groupLimit, ranked.size())))
{
QuickSearchResult first = hits.get(0);
List<QuickSearchSuggest.Item> items = hits.stream().limit(itemLimit).map(hit ->
{
String label = hit.getRecordLabel();
if(label == null || label.isBlank())
{
label = hit.getTableLabel() + " #" + hit.getRecordId();
}
return (new QuickSearchSuggest.Item(hit.getRecordId(), label, hit.getSubtitle()));
}).toList();
groups.add(new QuickSearchSuggest.Group(first.getTableName(), first.getTableLabel(), hits.size() > itemLimit, items));
}
return (new QuickSearchSuggest(normalized, groups, ranked.size() > groupLimit));
}
/*******************************************************************************
** Reject invalid ranges instead of silently changing a caller's intent.
*******************************************************************************/
private static Integer bounded(Integer value, Integer defaultValue, Integer maximum, String name) throws QBadRequestException
{
if(value == null)
{
return (defaultValue);
}
if(value < 1 || value > maximum)
{
throw (new QBadRequestException(name + " must be between 1 and " + maximum + "."));
}
return (value);
}
}
@@ -0,0 +1,177 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.api;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import com.kingsrook.qqq.openapi.model.In;
import com.kingsrook.qqq.openapi.model.Method;
import com.kingsrook.qqq.openapi.model.Parameter;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
import io.javalin.http.HttpStatus;
/*******************************************************************************
** Permission-filtered suggestions through the spec-managed UI API.
*******************************************************************************/
public class QuickSearchSuggestSpec extends AbstractJsonEndpointSpec
{
/*******************************************************************************
** Describe the session-only operation and its bounded search semantics.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/quick-search/suggest")
.withShortSummary("Get Quick Search suggestions")
.withLongDescription("Requires a signed-in session, not Quick Search admin permission. Results inherit table READ permissions, configured record security locks and protected-field exclusions. One bounded candidate batch per table; no extra scans. Unavailable search and normalized terms shorter than two characters return empty groups."));
}
/*******************************************************************************
** Stable generated-client operation name.
*******************************************************************************/
@Override
protected void customizeMethod(Method method)
{
method.setOperationId("getQuickSearchSuggestions");
}
/*******************************************************************************
** Bounds apply after whitespace normalization; short terms are forgiving.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
return (List.of(
new Parameter().withName("q").withIn(In.QUERY).withRequired(true)
.withDescription("Search text: trim and collapse whitespace before applying the 2..100 character range. Shorter terms return empty groups; longer terms return 400.")
.withSchema(field(Type.STRING, "Normalized search term.", false)),
new Parameter().withName("limitPerTable").withIn(In.QUERY).withRequired(false)
.withDescription("Items per table: integer 1..10, default 5.").withSchema(field(Type.INTEGER, "Maximum items per table; default 5, maximum 10.", false)),
new Parameter().withName("maxTables").withIn(In.QUERY).withRequired(false)
.withDescription("Groups to return: integer 1..20, default 8.").withSchema(field(Type.INTEGER, "Maximum groups; default 8, maximum 20.", false)),
new Parameter().withName("tables").withIn(In.QUERY).withRequired(false)
.withDescription("Optional comma-separated table metadata names. Unknown, disabled and unreadable tables are omitted.")
.withSchema(field(Type.STRING, "Table-name filter; omitted means all eligible tables.", false))));
}
/*******************************************************************************
** Versioned response component.
*******************************************************************************/
@Override
protected String getResponseSchemaName()
{
return ("QuickSearchSuggestV1");
}
/*******************************************************************************
** The HTTP and in-process entry points share validation and authorization.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (QuickSearchSuggestService.suggest(input.getQueryParam("q"), input.getQueryParamInteger("limitPerTable"),
input.getQueryParamInteger("maxTables"), input.getQueryParam("tables")));
}
/*******************************************************************************
** Reuse the extension's standard framework error shape.
*******************************************************************************/
@Override
public List<BasicResponse> defineAdditionalBasicResponses()
{
return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Missing q, normalized term longer than 100 characters, or invalid limits", BasicErrorResponseV1.class.getSimpleName()),
new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", BasicErrorResponseV1.class.getSimpleName()),
new BasicResponse(HttpStatus.FORBIDDEN, "Session permission denied", BasicErrorResponseV1.class.getSimpleName())));
}
/*******************************************************************************
** Explicit wire schemas exclude scores, highlights and display values.
*******************************************************************************/
@Override
public Map<String, Schema> defineComponentSchemas()
{
Schema response = object(Map.of(
"query", field(Type.STRING, "Search term after trimming and collapsing whitespace.", false),
"groups", array("QuickSearchSuggestGroupV1", "Readable tables ordered by best accessible score; at most maxTables."),
"truncated", field(Type.BOOLEAN, "More tables had accessible hits than maxTables.", false)));
Schema group = object(Map.of(
"tableName", field(Type.STRING, "Source table metadata name.", false),
"tableLabel", field(Type.STRING, "Source table label, falling back to its metadata name.", false),
"hasMore", field(Type.BOOLEAN, "An extra accessible item was found in the bounded batch. False does not prove the index has no further matches.", false),
"items", array("QuickSearchSuggestItemV1", "At most limitPerTable accessible items in per-table relevance order.")));
Schema item = object(Map.of(
"recordId", field(Type.STRING, "Source record primary key.", false),
"label", field(Type.STRING, "Filtered record label, falling back to tableLabel #recordId when blank.", false),
"subtitle", field(Type.STRING, "Already-filtered result subtitle; null when absent.", true)));
return (Map.of("QuickSearchSuggestV1", response, "QuickSearchSuggestGroupV1", group, "QuickSearchSuggestItemV1", item));
}
/*******************************************************************************
** All response objects have explicit properties.
*******************************************************************************/
private static Schema object(Map<String, Schema> properties)
{
return (new Schema().withType(Type.OBJECT).withNullable(false).withProperties(new TreeMap<>(properties)));
}
/*******************************************************************************
** Scalar fields declare nullability and meaning.
*******************************************************************************/
private static Schema field(Type type, String description, Boolean nullable)
{
return (new Schema().withType(type).withDescription(description).withNullable(nullable));
}
/*******************************************************************************
** Arrays always exist, including when empty.
*******************************************************************************/
private static Schema array(String itemSchema, String description)
{
return (field(Type.ARRAY, description, false).withItems(new Schema().withRef("#/components/schemas/" + itemSchema)));
}
}
@@ -0,0 +1,513 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Self-priming: seeds the config records from the code defaults
** (@QuickSearchable / @QuickSearchField entities and searchableTables), on
** startup and until it has succeeded once per runtime.
**
** Precedence (the records win; code only fills in what is missing):
** - A table code declares that has no quick_search_index row, and no
** tombstone, gets a row (source CODE, enabled per enabledByDefault) and
** field rows, and is queued for its first build (status PENDING_BUILD,
** which the next basepull runs). A tombstoned table is never re-added.
** - On a row an admin has not edited (ownerEdited false), the basepull
** interval and timestamp field follow code. enabled is set only when the
** row is created, never afterwards.
** - A field code declares that has no row, and no tombstone, is added; a
** field row an admin has not edited follows code's weight and label flag.
** Either change queues a rebuild of the table. Field rows an admin added
** or edited are never changed, and nothing is ever deleted by priming:
** removing an annotation leaves its rows for the admin to remove.
** - Rows from before 1.1.0-RC.3 (no field rows yet) get their field rows
** without a rebuild when the index was already built with those fields.
**
** All writes are system writes (QuickSearchConfigWrites), so they are not
** taken for an admin's edits. The outcome is written to quick_search_settings
** (lastPrimingSummary), which the Quick Search home shows.
*******************************************************************************/
public class QuickSearchConfigPrimer
{
private static final QLogger LOG = QLogger.getLogger(QuickSearchConfigPrimer.class);
private final QuickSearchRuntime runtime;
private final QuickSearchQBitConfig config;
/*******************************************************************************
** What priming did.
**
** @param newTables labels of the tables added (and queued to build)
** @param newFields field rows added
** @param updatedFields field rows updated to code's defaults
** @param updatedTables index rows updated to code's defaults
** @param rebuilds labels of existing tables queued to rebuild
** @param skipped code tables or fields skipped for a tombstone
*******************************************************************************/
public record PrimingReport(List<String> newTables, int newFields, int updatedFields, int updatedTables, List<String> rebuilds, int skipped)
{
/*******************************************************************************
** Whether priming changed anything.
*******************************************************************************/
public boolean hasChanges()
{
return (!newTables.isEmpty() || newFields > 0 || updatedFields > 0 || updatedTables > 0 || !rebuilds.isEmpty());
}
/*******************************************************************************
** The one-line summary for the Quick Search home.
*******************************************************************************/
public String summary()
{
List<String> parts = new ArrayList<>();
if(!newTables.isEmpty())
{
parts.add("New: " + newTables.size() + (newTables.size() == 1 ? " table" : " tables") + " added and building (" + String.join(", ", newTables) + ")");
}
if(newFields > 0 || updatedFields > 0)
{
parts.add(newFields + " field" + (newFields == 1 ? "" : "s") + " added, " + updatedFields + " updated from code defaults");
}
if(!rebuilds.isEmpty())
{
parts.add("rebuilding " + String.join(", ", rebuilds));
}
if(updatedTables > 0)
{
parts.add(updatedTables + " index setting" + (updatedTables == 1 ? "" : "s") + " updated from code defaults");
}
return (parts.isEmpty() ? "No changes: the records match the code defaults." : String.join("; ", parts) + ".");
}
}
/*******************************************************************************
** Constructor
*******************************************************************************/
public QuickSearchConfigPrimer(QuickSearchRuntime runtime)
{
this.runtime = runtime;
this.config = runtime.getConfig();
}
/*******************************************************************************
** Seed the records from the code defaults. Throws when the config tables
** cannot be read or written (the caller retries later).
*******************************************************************************/
public PrimingReport prime() throws QException
{
PrimingReport report = QuickSearchConfigWrites.asSystem(this::doPrime);
if(report.hasChanges())
{
LOG.info("Quick Search primed its config records: " + report.summary());
}
return (report);
}
/*******************************************************************************
**
*******************************************************************************/
private PrimingReport doPrime() throws QException
{
QInstance qInstance = QContext.getQInstance();
String indexTable = config.getQuickSearchIndexTableName();
String fieldTable = config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME);
Set<String> tableTombstones = new HashSet<>();
Set<String> fieldTombstones = new HashSet<>();
String tombstoneTable = config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME);
if(qInstance.getTable(tombstoneTable) != null)
{
for(QRecord tombstone : queryAll(tombstoneTable))
{
String fieldName = tombstone.getValueString("fieldName");
if(fieldName == null)
{
tableTombstones.add(tombstone.getValueString("tableName"));
}
else
{
fieldTombstones.add(tombstone.getValueString("tableName") + "." + fieldName);
}
}
}
Map<String, QRecord> rowsByTable = new LinkedHashMap<>();
for(QRecord row : queryAll(indexTable))
{
rowsByTable.putIfAbsent(row.getValueString("tableName"), row);
}
Map<Integer, Map<String, QRecord>> fieldsByIndex = new HashMap<>();
for(QRecord fieldRow : queryAll(fieldTable))
{
fieldsByIndex.computeIfAbsent(fieldRow.getValueInteger("quickSearchIndexId"), k -> new LinkedHashMap<>()).putIfAbsent(fieldRow.getValueString("fieldName"), fieldRow);
}
List<String> newTables = new ArrayList<>();
List<String> rebuilds = new ArrayList<>();
int newFields = 0;
int updatedFields = 0;
int updatedTables = 0;
int skipped = 0;
for(QuickSearchableTableConfig code : runtime.getCodeTables())
{
String tableName = code.getTableName();
QTableMetaData table = qInstance.getTable(tableName);
if(table == null)
{
LOG.warn("A table declared searchable in code is not in the instance; it is not primed", logPair("tableName", tableName));
continue;
}
if(tableTombstones.contains(tableName))
{
skipped++;
continue;
}
List<QRecord> codeFields = new ArrayList<>();
List<String> fieldNames = CollectionUtils.nonNullList(code.getSearchableFields());
for(int i = 0; i < fieldNames.size(); i++)
{
String fieldName = fieldNames.get(i);
if(fieldTombstones.contains(tableName + "." + fieldName))
{
skipped++;
continue;
}
if(!QuickSearchFieldRules.isIndexable(table, fieldName))
{
QuickSearchFieldRules.warnExcludedFields(table);
continue;
}
codeFields.add(codeFieldRow(code, fieldName, i));
}
QRecord row = rowsByTable.get(tableName);
if(row == null)
{
boolean enabled = !Boolean.FALSE.equals(code.getEnabledByDefault());
QRecord inserted = insertOne(indexTable, new QRecord()
.withValue("tableName", tableName)
.withValue("enabled", enabled)
.withValue("basepullIntervalMinutes", code.getBasepullIntervalMinutes())
.withValue("basepullTimestampField", code.getBasepullTimestampField())
.withValue("source", QuickSearchIndex.SOURCE_CODE)
.withValue("ownerEdited", false)
.withValue("realTimeErrorCount", 0)
.withValue("status", enabled ? AbstractIndexingStep.STATUS_PENDING_BUILD : AbstractIndexingStep.STATUS_ACTIVE));
Integer indexId = inserted.getValueInteger("id");
codeFields.forEach(f -> f.setValue("quickSearchIndexId", indexId));
insertAll(fieldTable, codeFields);
newFields += codeFields.size();
newTables.add(QuickSearchFieldRules.label(table));
continue;
}
Integer indexId = row.getValueInteger("id");
Map<String, Serializable> updates = new LinkedHashMap<>();
if(row.getValueString("source") == null)
{
updates.put("source", QuickSearchIndex.SOURCE_CODE);
}
if(!Boolean.TRUE.equals(row.getValueBoolean("ownerEdited")))
{
if(!Objects.equals(row.getValueInteger("basepullIntervalMinutes"), code.getBasepullIntervalMinutes()) && code.getBasepullIntervalMinutes() != null)
{
updates.put("basepullIntervalMinutes", code.getBasepullIntervalMinutes());
}
if(!Objects.equals(row.getValueString("basepullTimestampField"), code.getBasepullTimestampField()))
{
updates.put("basepullTimestampField", code.getBasepullTimestampField());
}
}
if(updates.keySet().stream().anyMatch(k -> !"source".equals(k)))
{
updatedTables++;
}
Map<String, QRecord> existingFields = fieldsByIndex.getOrDefault(indexId, Map.of());
boolean migration = existingFields.isEmpty();
boolean fieldsChanged = false;
List<QRecord> toInsert = new ArrayList<>();
List<QRecord> toUpdate = new ArrayList<>();
for(QRecord codeField : codeFields)
{
QRecord existing = existingFields.get(codeField.getValueString("fieldName"));
if(existing == null)
{
codeField.setValue("quickSearchIndexId", indexId);
toInsert.add(codeField);
fieldsChanged = true;
}
else if(!Boolean.TRUE.equals(existing.getValueBoolean("ownerEdited"))
&& (!Objects.equals(existing.getValueInteger("weight"), codeField.getValueInteger("weight"))
|| !Objects.equals(Boolean.TRUE.equals(existing.getValueBoolean("includeLabel")), codeField.getValueBoolean("includeLabel"))))
{
toUpdate.add(new QRecord().withValue("id", existing.getValue("id"))
.withValue("weight", codeField.getValueInteger("weight"))
.withValue("includeLabel", codeField.getValueBoolean("includeLabel")));
fieldsChanged = true;
}
}
insertAll(fieldTable, toInsert);
updateAll(fieldTable, toUpdate);
newFields += toInsert.size();
updatedFields += toUpdate.size();
if(fieldsChanged && needsRebuild(row, code, migration))
{
updates.put("status", AbstractIndexingStep.STATUS_PENDING_BUILD);
rebuilds.add(QuickSearchFieldRules.label(table));
}
if(!updates.isEmpty())
{
QRecord update = new QRecord().withValue("id", indexId);
updates.forEach(update::setValue);
updateAll(indexTable, List.of(update));
}
}
PrimingReport report = new PrimingReport(newTables, newFields, updatedFields, updatedTables, rebuilds, skipped);
writeSettings(report);
return (report);
}
/*******************************************************************************
** Whether a primed field change needs the table rebuilt: not when it is
** disabled or already building, and not for rows from before field
** records when the index was already built with exactly code's fields
** (or never built: its first basepull reads everything anyway).
*******************************************************************************/
private static boolean needsRebuild(QRecord row, QuickSearchableTableConfig code, boolean migration)
{
String status = row.getValueString("status");
if(Boolean.FALSE.equals(row.getValueBoolean("enabled"))
|| AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status)
|| AbstractIndexingStep.STATUS_BUILDING.equals(status)
|| AbstractIndexingStep.STATUS_REBUILDING.equals(status))
{
return (false);
}
if(migration)
{
String built = row.getValueString("searchableFieldsJson");
return (built != null && !built.equals(AbstractIndexingStep.buildSearchableFieldsJson(code)));
}
return (true);
}
/*******************************************************************************
** A field row from code's defaults (role BODY; analyzer and snippet left
** to their defaults).
*******************************************************************************/
private static QRecord codeFieldRow(QuickSearchableTableConfig code, String fieldName, int sortOrder)
{
Integer weight = code.getFieldWeights() == null ? null : code.getFieldWeights().get(fieldName);
Boolean includeLabel = code.getFieldIncludeLabels() == null ? null : code.getFieldIncludeLabels().get(fieldName);
return (new QRecord()
.withValue("fieldName", fieldName)
.withValue("weight", weight == null ? 1 : weight)
.withValue("includeLabel", Boolean.TRUE.equals(includeLabel))
.withValue("role", QuickSearchFieldRole.BODY.name())
.withValue("sortOrder", (sortOrder + 1) * 10)
.withValue("source", QuickSearchIndexField.SOURCE_CODE)
.withValue("ownerEdited", false));
}
/*******************************************************************************
** Create the settings row when missing, and record the priming outcome.
*******************************************************************************/
private void writeSettings(PrimingReport report) throws QException
{
String settingsTable = config.applyTablePrefix(QuickSearchSettings.TABLE_NAME);
if(QContext.getQInstance().getTable(settingsTable) == null)
{
return;
}
QRecord values = new QRecord().withValue("lastPrimedAt", Instant.now());
if(report.hasChanges())
{
values.withValue("lastPrimingSummary", report.summary()).withValue("lastPrimingNewTables", report.newTables().size());
}
List<QRecord> rows = queryAll(settingsTable);
if(rows.isEmpty())
{
values.withValue("id", QuickSearchSettings.SETTINGS_ID).withValue("indexingPaused", false);
if(!report.hasChanges())
{
values.withValue("lastPrimingSummary", report.summary()).withValue("lastPrimingNewTables", 0);
}
insertOne(settingsTable, values);
}
else
{
///////////////////////////////////////////////////////////////////////
// Only the priming outcome (lastPrimedAt, summary) changes here, on //
// every startup: bookkeeping, not a settings change, so not audited. //
///////////////////////////////////////////////////////////////////////
values.withValue("id", rows.get(0).getValue("id"));
updateAll(settingsTable, List.of(values), true);
}
}
/*******************************************************************************
** Every row of a table.
*******************************************************************************/
public static List<QRecord> queryAll(String tableName) throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter());
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
}
/*******************************************************************************
** Insert one row, failing on any record error.
*******************************************************************************/
public static QRecord insertOne(String tableName, QRecord record) throws QException
{
List<QRecord> inserted = insertAll(tableName, List.of(record));
if(inserted.isEmpty())
{
throw (new QException("Insert into " + tableName + " returned no record"));
}
return (inserted.get(0));
}
/*******************************************************************************
** Insert rows, failing on any record error.
*******************************************************************************/
public static List<QRecord> insertAll(String tableName, List<QRecord> records) throws QException
{
if(records.isEmpty())
{
return (List.of());
}
InsertInput insertInput = new InsertInput();
insertInput.setTableName(tableName);
insertInput.setRecords(records);
InsertOutput output = new InsertAction().execute(insertInput);
List<QRecord> inserted = CollectionUtils.nonNullList(output.getRecords());
for(QRecord record : inserted)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
throw (new QException("Could not insert into " + tableName + ": " + record.getErrors().get(0).getMessage()));
}
}
return (inserted);
}
/*******************************************************************************
** Update rows, failing on any record error.
*******************************************************************************/
public static void updateAll(String tableName, List<QRecord> records) throws QException
{
updateAll(tableName, records, false);
}
/*******************************************************************************
** Update rows, failing on any record error; omitAudit skips the DML audit
** for runtime bookkeeping writes.
*******************************************************************************/
public static void updateAll(String tableName, List<QRecord> records, boolean omitAudit) throws QException
{
if(records.isEmpty())
{
return;
}
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(tableName);
updateInput.setRecords(records);
updateInput.setOmitDmlAudit(omitAudit);
for(QRecord record : CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords()))
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
throw (new QException("Could not update " + tableName + ": " + record.getErrors().get(0).getMessage()));
}
}
}
}
@@ -0,0 +1,504 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** The effective Quick Search configuration, read from the records: which
** tables are indexed (quick_search_index), with which fields, weights,
** roles and analyzers (quick_search_index_field), and the system settings
** (quick_search_settings: the global pause).
**
** Precedence: the records are the source of truth once the instance has the
** config tables and priming has run. Code (annotations and searchableTables)
** only seeds them (QuickSearchConfigPrimer). Until priming has succeeded,
** or when the config tables are not in the instance or not readable (a host
** that has not added their DDL yet), the code-declared tables are used, as
** before 1.1.0-RC.3.
**
** Reads are cached for CACHE_TTL_MILLIS and dropped at once by the config
** tables' customizers when a row changes, on every server: the runtime
** publishes a cluster cache invalidation (QuickSearchRuntime.CONFIG_CACHE_NAME).
** The TTL remains the safety net when the cluster bus is down.
*******************************************************************************/
public class QuickSearchConfigStore
{
private static final QLogger LOG = QLogger.getLogger(QuickSearchConfigStore.class);
public static final long CACHE_TTL_MILLIS = 30_000L;
public static final long PRIME_RETRY_MILLIS = 300_000L;
private static final ThreadLocal<Boolean> BUSY = ThreadLocal.withInitial(() -> false);
private final QuickSearchRuntime ownerRuntime;
private volatile Snapshot snapshot;
private volatile QRecord settings;
private volatile long settingsReadAtMillis = 0L;
private volatile boolean primed = false;
private volatile long lastPrimeAttemptMillis = 0L;
private volatile long lastFailureLogMillis = 0L;
private final Set<String> coreSearchFieldsDefaulted = ConcurrentHashMap.newKeySet();
/*******************************************************************************
** A loaded set of effective table configs.
*******************************************************************************/
private record Snapshot(List<QuickSearchableTableConfig> tables, Map<String, QuickSearchableTableConfig> byName, long loadedAtMillis)
{
}
/*******************************************************************************
** Constructor
*******************************************************************************/
public QuickSearchConfigStore(QuickSearchRuntime runtime)
{
this.ownerRuntime = runtime;
}
/*******************************************************************************
** The runtime the config hangs off (the one the store was made for, unless
** the config has since been given another, such as a wrapper of it).
*******************************************************************************/
private QuickSearchRuntime runtime()
{
QuickSearchRuntime current = ownerRuntime.getConfig() == null ? null : ownerRuntime.getConfig().getRuntime();
return (current != null ? current : ownerRuntime);
}
/*******************************************************************************
** Whether the instance in the current context has the config tables
** (quick_search_index and quick_search_index_field).
*******************************************************************************/
public boolean hasConfigTables()
{
QInstance qInstance = QContext.getQInstance();
QuickSearchQBitConfig config = runtime().getConfig();
return (qInstance != null
&& qInstance.getTable(config.getQuickSearchIndexTableName()) != null
&& qInstance.getTable(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME)) != null);
}
/*******************************************************************************
** Whether the effective config comes from the records (else from code).
*******************************************************************************/
public boolean isRecordsMode()
{
return (primed && hasConfigTables());
}
/*******************************************************************************
** Whether priming has succeeded in this runtime.
*******************************************************************************/
public boolean isPrimed()
{
return (primed);
}
/*******************************************************************************
** Record that priming succeeded (or, for tests, reset it).
*******************************************************************************/
public void setPrimed(boolean primed)
{
this.primed = primed;
this.lastPrimeAttemptMillis = 0L;
invalidate();
}
/*******************************************************************************
** The effective table configs: from the records in records mode, else the
** code-declared ones. Primes first when that has not happened yet (at most
** once per PRIME_RETRY_MILLIS while it keeps failing). Never throws.
*******************************************************************************/
public List<QuickSearchableTableConfig> getTables()
{
Snapshot current = getSnapshot();
return (current == null ? runtime().getCodeTables() : current.tables());
}
/*******************************************************************************
** One table's effective config, or null when it is not indexed.
*******************************************************************************/
public QuickSearchableTableConfig getTable(String tableName)
{
if(tableName == null)
{
return (null);
}
Snapshot current = getSnapshot();
if(current == null)
{
return (runtime().getCodeTableConfig(tableName));
}
return (current.byName().get(tableName));
}
/*******************************************************************************
** The current snapshot, loading it when stale; null means "use code".
*******************************************************************************/
private Snapshot getSnapshot()
{
if(BUSY.get() || !hasConfigTables())
{
return (null);
}
if(!primed)
{
tryPrime();
if(!primed)
{
return (null);
}
}
Snapshot current = snapshot;
if(current != null && System.currentTimeMillis() - current.loadedAtMillis() < CACHE_TTL_MILLIS)
{
return (current);
}
Snapshot loaded = load();
if(loaded != null)
{
snapshot = loaded;
return (loaded);
}
return (current);
}
/*******************************************************************************
** Prime now, unless it already succeeded or failed less than
** PRIME_RETRY_MILLIS ago. Never throws.
*******************************************************************************/
public synchronized void tryPrime()
{
if(primed || BUSY.get() || !hasConfigTables())
{
return;
}
long now = System.currentTimeMillis();
if(lastPrimeAttemptMillis != 0 && now - lastPrimeAttemptMillis < PRIME_RETRY_MILLIS)
{
return;
}
BUSY.set(true);
try
{
new QuickSearchConfigPrimer(runtime()).prime();
primed = true;
lastPrimeAttemptMillis = 0L;
}
catch(Exception | LinkageError e)
{
lastPrimeAttemptMillis = now;
LOG.warn("Quick Search could not prime its config records; using the code-declared tables until it can (retried every "
+ (PRIME_RETRY_MILLIS / 60_000) + " minutes)", e);
}
finally
{
BUSY.set(false);
invalidate();
}
}
/*******************************************************************************
** Drop the cached configs and settings (a config row changed).
*******************************************************************************/
public void invalidate()
{
snapshot = null;
settingsReadAtMillis = 0L;
}
/*******************************************************************************
** Read the records into a snapshot; null (logged, at most once a minute)
** when they cannot be read.
*******************************************************************************/
private Snapshot load()
{
BUSY.set(true);
try
{
QuickSearchQBitConfig config = runtime().getConfig();
QInstance qInstance = QContext.getQInstance();
List<QRecord> indexRows = query(config.getQuickSearchIndexTableName());
List<QRecord> fieldRows = query(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME));
Map<Integer, List<QRecord>> fieldsByIndex = new LinkedHashMap<>();
for(QRecord fieldRow : fieldRows)
{
fieldsByIndex.computeIfAbsent(fieldRow.getValueInteger("quickSearchIndexId"), k -> new ArrayList<>()).add(fieldRow);
}
List<QuickSearchableTableConfig> tables = new ArrayList<>();
Map<String, QuickSearchableTableConfig> byName = new LinkedHashMap<>();
for(QRecord indexRow : indexRows)
{
QuickSearchableTableConfig tableConfig = buildTableConfig(qInstance, config, indexRow, fieldsByIndex.getOrDefault(indexRow.getValueInteger("id"), List.of()));
if(tableConfig != null && !byName.containsKey(tableConfig.getTableName()))
{
tables.add(tableConfig);
byName.put(tableConfig.getTableName(), tableConfig);
}
}
////////////////////////////////////////////////////////////////////////
// tables added at runtime get core searchFields (once per table), so //
// the global search covers them too //
////////////////////////////////////////////////////////////////////////
if(Boolean.TRUE.equals(config.getServeCoreRecordSearch()) && qInstance != null)
{
List<QuickSearchableTableConfig> unseen = tables.stream().filter(t -> coreSearchFieldsDefaulted.add(t.getTableName())).toList();
if(!unseen.isEmpty())
{
QuickSearchQBitProducer.defaultCoreSearchFields(qInstance, unseen);
}
}
return (new Snapshot(Collections.unmodifiableList(tables), Collections.unmodifiableMap(byName), System.currentTimeMillis()));
}
catch(Exception | LinkageError e)
{
long now = System.currentTimeMillis();
if(now - lastFailureLogMillis > 60_000L)
{
lastFailureLogMillis = now;
LOG.warn("Could not read the Quick Search config records; keeping the last config read", e);
}
return (null);
}
finally
{
BUSY.set(false);
}
}
/*******************************************************************************
** Build one table's effective config from its index row and field rows.
** A table no longer in the instance is skipped; a field that is gone or
** can no longer be indexed (hidden, sensitive) is dropped with a warning.
*******************************************************************************/
static QuickSearchableTableConfig buildTableConfig(QInstance qInstance, QuickSearchQBitConfig config, QRecord indexRow, List<QRecord> fieldRows)
{
String tableName = indexRow.getValueString("tableName");
QTableMetaData table = qInstance == null || tableName == null ? null : qInstance.getTable(tableName);
if(table == null)
{
LOG.debug("Quick Search index row names a table that is not in the instance; skipped", logPair("tableName", tableName));
return (null);
}
List<QRecord> sorted = new ArrayList<>(fieldRows);
sorted.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("sortOrder") == null ? Integer.MAX_VALUE : r.getValueInteger("sortOrder"))
.thenComparing(r -> r.getValueInteger("id") == null ? Integer.MAX_VALUE : r.getValueInteger("id")));
List<String> fields = new ArrayList<>();
Map<String, Integer> weights = new LinkedHashMap<>();
Map<String, Boolean> includeLabels = new LinkedHashMap<>();
Map<String, String> roles = new LinkedHashMap<>();
Map<String, String> analyzers = new LinkedHashMap<>();
Map<String, Boolean> snippetFlags = new LinkedHashMap<>();
for(QRecord fieldRow : sorted)
{
String fieldName = fieldRow.getValueString("fieldName");
String problem = QuickSearchFieldRules.whyNotIndexable(table, fieldName);
if(problem != null)
{
QuickSearchFieldRules.warnExcludedFields(table);
continue;
}
if(fields.contains(fieldName))
{
continue;
}
fields.add(fieldName);
weights.put(fieldName, fieldRow.getValueInteger("weight") == null ? 1 : fieldRow.getValueInteger("weight"));
includeLabels.put(fieldName, Boolean.TRUE.equals(fieldRow.getValueBoolean("includeLabel")));
///////////////////////////////////////////////////////////////////////
// only non-defaults go in the maps, so a table of plain fields has //
// the same searchableFieldsJson as its code config (no false drift) //
///////////////////////////////////////////////////////////////////////
QuickSearchFieldRole role = QuickSearchFieldRole.fromName(fieldRow.getValueString("role"), QuickSearchFieldRole.BODY);
if(role != QuickSearchFieldRole.BODY)
{
roles.put(fieldName, role.name());
}
if(StringUtils.hasContent(fieldRow.getValueString("analyzer")))
{
analyzers.put(fieldName, fieldRow.getValueString("analyzer"));
}
if(fieldRow.getValueBoolean("includeInSnippet") != null)
{
snippetFlags.put(fieldName, fieldRow.getValueBoolean("includeInSnippet"));
}
}
String timestampField = indexRow.getValueString("basepullTimestampField");
if(timestampField != null && !QuickSearchFieldRules.isTimestampField(table, timestampField))
{
LOG.warn("Quick Search basepull timestamp field is not a date-time field of the table; incremental basepull is off for it",
logPair("tableName", tableName), logPair("timestampField", timestampField));
timestampField = null;
}
Integer interval = indexRow.getValueInteger("basepullIntervalMinutes");
Boolean enabled = indexRow.getValueBoolean("enabled");
QuickSearchableTableConfig codeConfig = config.getRuntime() == null ? null : config.getRuntime().getCodeTableConfig(tableName);
return (new QuickSearchableTableConfig()
.withTableName(tableName)
.withPrimaryKeyField(table.getPrimaryKeyField())
.withSearchableFields(fields)
.withFieldWeights(weights)
.withFieldIncludeLabels(includeLabels)
.withFieldRoles(roles)
.withFieldAnalyzers(analyzers)
.withFieldIncludeInSnippet(snippetFlags)
.withBasepullIntervalMinutes(interval == null ? config.getDefaultBasepullIntervalMinutes() : interval)
.withBasepullTimestampField(timestampField)
.withEnabledByDefault(codeConfig == null ? Boolean.TRUE : codeConfig.getEnabledByDefault())
.withEnabled(enabled == null || enabled)
.withQuickSearchIndexId(indexRow.getValueInteger("id"))
.withRecordLabelFormat(codeConfig == null ? null : codeConfig.getRecordLabelFormat())
.withRecordLabelFields(codeConfig == null ? null : codeConfig.getRecordLabelFields())
.withMaxFieldLength(config.getMaxFieldLength()));
}
/*******************************************************************************
** The settings row (id 1), cached like the configs; null when the table is
** not in the instance, the row does not exist, or it cannot be read.
*******************************************************************************/
public QRecord getSettings()
{
long now = System.currentTimeMillis();
if(settingsReadAtMillis != 0 && now - settingsReadAtMillis < CACHE_TTL_MILLIS)
{
return (settings);
}
QRecord read = null;
String name = runtime().getConfig().applyTablePrefix(QuickSearchSettings.TABLE_NAME);
if(QContext.getQInstance() != null && QContext.getQInstance().getTable(name) != null)
{
try
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(name);
queryInput.setFilter(new QQueryFilter().withLimit(1));
List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
read = rows.isEmpty() ? null : rows.get(0);
}
catch(Exception | LinkageError e)
{
LOG.debug("Could not read the Quick Search settings; treating indexing as not paused", e);
}
}
settings = read;
settingsReadAtMillis = now;
return (read);
}
/*******************************************************************************
** Whether an admin has paused all indexing (quick_search_settings).
*******************************************************************************/
public boolean isIndexingPaused()
{
QRecord row = getSettings();
return (row != null && Boolean.TRUE.equals(row.getValueBoolean("indexingPaused")));
}
/*******************************************************************************
** All rows of a config table.
*******************************************************************************/
private static List<QRecord> query(String tableName) throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter());
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
}
}
@@ -0,0 +1,110 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Marks the QBit's own writes to its config tables (priming, status updates
** made on the owner's behalf), so the table customizers can tell them from
** an admin's edits: only an admin's edit marks a row ownerEdited, writes a
** tombstone, or flags the index for a rebuild.
*******************************************************************************/
public final class QuickSearchConfigWrites
{
private static final ThreadLocal<Boolean> SYSTEM_WRITE = ThreadLocal.withInitial(() -> false);
/*******************************************************************************
** Work that may throw.
*******************************************************************************/
@FunctionalInterface
public interface Work<T>
{
/*******************************************************************************
** Do it.
*******************************************************************************/
T run() throws QException;
}
/*******************************************************************************
** Utility class - no instances.
*******************************************************************************/
private QuickSearchConfigWrites()
{
}
/*******************************************************************************
** Run work as a system write (nested calls stay system writes).
*******************************************************************************/
public static <T> T asSystem(Work<T> work) throws QException
{
boolean previous = SYSTEM_WRITE.get();
SYSTEM_WRITE.set(true);
try
{
return (work.run());
}
finally
{
SYSTEM_WRITE.set(previous);
}
}
/*******************************************************************************
** Whether the current thread is making a system write.
*******************************************************************************/
public static boolean isSystemWrite()
{
return (SYSTEM_WRITE.get());
}
/*******************************************************************************
** The current user's name for audit-style columns (pausedBy, removedBy):
** full name, else id, else "system".
*******************************************************************************/
public static String currentUserName()
{
QSession session = QContext.getQSession();
QUser user = session == null ? null : session.getUser();
if(user != null && StringUtils.hasContent(user.getFullName()))
{
return (user.getFullName());
}
if(user != null && StringUtils.hasContent(user.getIdReference()))
{
return (user.getIdReference());
}
return ("system");
}
}
@@ -0,0 +1,443 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.actions.ai.AiDataClassifications;
import com.kingsrook.qqq.backend.core.actions.ai.DataClassification;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
/*******************************************************************************
** Which tables and fields may be indexed, and the defaults offered for a new
** index. Shared by the Add Search Index process, the eligible-table and
** field possible-value sources, the field table's validation and priming,
** so all of them agree.
**
** A table is eligible when it is in the instance, queryable, not hidden, not
** on backend variants, not produced by a QBit (qbit-internal), not a QQQ
** system table (audits, saved views and reports, scheduled jobs, scripts),
** not named like a secret store, and has at least one indexable field. A
** field is indexable when it is not PII/PHI-classified, locked, hidden,
** sensitive, masked, a blob or a boolean.
*******************************************************************************/
public final class QuickSearchFieldRules
{
private static final QLogger LOG = QLogger.getLogger(QuickSearchFieldRules.class);
private static final WeakHashMap<QTableMetaData, Boolean> WARNED_TABLES = ClusterCaches.nodeLocal(new WeakHashMap<>(),
"warn-once log de-duplication for this node's table metadata objects");
public static final int MAX_DEFAULT_FIELDS = 20;
public static final int TITLE_WEIGHT = 3;
private static final Set<String> SYSTEM_TABLES = Set.of(
"audit", "auditdetail", "audittable", "audituser", "savedview", "savedreport", "sharedsavedview", "sharedsavedreport",
"scheduledjob", "scheduledjobparameter", "script", "scriptrevision", "scriptrevisionfile", "scriptlog", "scriptlogline",
"scripttype", "scripttypefilemode", "tabletrigger", "qqqtable", "qqqtablecache", "qqqprocess", "processtrace", "cachedrecord");
private static final Pattern SECRET_NAME = Pattern.compile("(secret|credential|password|passwd|token|api_?key|session|oauth|private_?key)", Pattern.CASE_INSENSITIVE);
/*******************************************************************************
** A field picked by default for a new index.
*******************************************************************************/
public record DefaultField(String fieldName, Integer weight, QuickSearchFieldRole role)
{
}
/*******************************************************************************
** Utility class - no instances.
*******************************************************************************/
private QuickSearchFieldRules()
{
}
/*******************************************************************************
** Why a field cannot be indexed, or null when it can.
** When the qbit moves to QQQ 4.1.0-RC.7, also honour core
** ClassifiedValues.isClassified(table, field) (the host's AuditValueProtector
** classification, qqq#1182) and FieldAccessHelper.getLockTier (qqq#1172).
*******************************************************************************/
public static String whyNotIndexable(QTableMetaData table, String fieldName)
{
if(table == null)
{
return ("the table is not in this application");
}
QFieldMetaData field = fieldName == null ? null : table.getFields().get(fieldName);
if(field == null)
{
return ("field [" + fieldName + "] does not exist on " + label(table));
}
if(Boolean.TRUE.equals(field.getIsHidden()))
{
return ("field [" + fieldName + "] is hidden");
}
DataClassification classification = AiDataClassifications.ofField(table, fieldName);
if(classification == DataClassification.PII || classification == DataClassification.PHI)
{
return ("field [" + fieldName + "] is classified " + classification + " and cannot be indexed");
}
if(field.getFieldSecurityLock() != null)
{
return ("field [" + fieldName + "] has a field security lock and cannot be indexed");
}
if(field.getIsSensitive() || (field.getType() != null && field.getType().needsMasked()))
{
return ("field [" + fieldName + "] is sensitive (its values must not reach the search index)");
}
if(field.getType() == null || field.getType() == QFieldType.BLOB || field.getType() == QFieldType.BOOLEAN)
{
return ("field [" + fieldName + "] is a " + (field.getType() == null ? "typeless" : field.getType().name().toLowerCase(Locale.ROOT)) + " field, which cannot be searched");
}
return (null);
}
/*******************************************************************************
** Whether a field can be indexed.
*******************************************************************************/
public static boolean isIndexable(QTableMetaData table, String fieldName)
{
return (whyNotIndexable(table, fieldName) == null);
}
/*******************************************************************************
** Whether a field contains protected values, independently of searchability.
** Classification falls back to the table. When the qbit moves to QQQ
** 4.1.0-RC.7, also honour ClassifiedValues.isClassified(table, field)
** (qqq#1182) and FieldAccessHelper.getLockTier (qqq#1172).
*******************************************************************************/
public static boolean isProtected(QTableMetaData table, String fieldName)
{
QFieldMetaData field = table == null || fieldName == null ? null : table.getFields().get(fieldName);
if(field == null)
{
return (false);
}
DataClassification classification = AiDataClassifications.ofField(table, fieldName);
return (classification == DataClassification.PII || classification == DataClassification.PHI
|| field.getFieldSecurityLock() != null || field.getIsSensitive()
|| (field.getType() != null && field.getType().needsMasked()));
}
/*******************************************************************************
** Warn once per table metadata lifetime; never include record values.
*******************************************************************************/
public static synchronized void warnExcludedFields(QTableMetaData table)
{
if(table != null && WARNED_TABLES.put(table, Boolean.TRUE) == null)
{
LOG.warn("Quick Search dropped fields that cannot be indexed (classified PII/PHI, secured, hidden, sensitive or unsupported fields) from table [" + table.getName() + "]");
}
}
/*******************************************************************************
** Filter explicit field lists as well as cached configurations at use time.
*******************************************************************************/
public static List<String> indexableFields(QTableMetaData table, List<String> fields)
{
List<String> result = new ArrayList<>();
for(String field : CollectionUtils.nonNullList(fields))
{
if(isIndexable(table, field))
{
result.add(field);
}
else
{
warnExcludedFields(table);
}
}
return (result);
}
/*******************************************************************************
** Resolve a table's metadata for backend query and result filtering.
*******************************************************************************/
public static QTableMetaData lookupTable(String tableName)
{
QInstance instance = QContext.getQInstance();
return (instance == null || tableName == null ? null : instance.getTable(tableName));
}
/*******************************************************************************
** Combined text and labels from an older index cannot be attributed to
** individual fields. Use only allowed field values for these tables.
*******************************************************************************/
public static boolean requiresFieldFiltering(String tableName)
{
QTableMetaData table = lookupTable(tableName);
return (table != null && table.getFields().keySet().stream().anyMatch(f -> isProtected(table, f)));
}
/*******************************************************************************
** A protected primary key cannot be sent as an index id, even for deletes.
*******************************************************************************/
public static boolean hasProtectedPrimaryKey(String tableName)
{
QTableMetaData table = lookupTable(tableName);
return (table != null && isProtected(table, table.getPrimaryKeyField()));
}
/*******************************************************************************
** The indexable fields of a table, in the table's order.
*******************************************************************************/
public static List<QFieldMetaData> indexableFields(QTableMetaData table)
{
List<QFieldMetaData> fields = new ArrayList<>();
if(table != null)
{
for(QFieldMetaData field : table.getFields().values())
{
if(isIndexable(table, field.getName()))
{
fields.add(field);
}
}
}
return (fields);
}
/*******************************************************************************
** The fields a new index starts with: the record-label fields as TITLE
** (weight 3), then the other text fields (string, text, HTML) as BODY
** (weight 1), up to MAX_DEFAULT_FIELDS. The admin adjusts them afterwards on
** the index's field rows.
*******************************************************************************/
public static List<DefaultField> defaultFields(QTableMetaData table)
{
List<DefaultField> fields = new ArrayList<>();
Set<String> picked = new LinkedHashSet<>();
if(table == null)
{
return (fields);
}
for(String labelField : CollectionUtils.nonNullList(table.getRecordLabelFields()))
{
if(isIndexable(table, labelField) && picked.add(labelField))
{
fields.add(new DefaultField(labelField, TITLE_WEIGHT, QuickSearchFieldRole.TITLE));
}
}
for(QFieldMetaData field : table.getFields().values())
{
if(fields.size() >= MAX_DEFAULT_FIELDS)
{
break;
}
if(field.getType() != null && field.getType().isStringLike() && isIndexable(table, field.getName()) && picked.add(field.getName()))
{
fields.add(new DefaultField(field.getName(), 1, QuickSearchFieldRole.BODY));
}
}
return (fields);
}
/*******************************************************************************
** Whether a field can drive incremental basepull (a date-time).
*******************************************************************************/
public static boolean isTimestampField(QTableMetaData table, String fieldName)
{
QFieldMetaData field = table == null || fieldName == null ? null : table.getFields().get(fieldName);
return (field != null && field.getType() == QFieldType.DATE_TIME);
}
/*******************************************************************************
** The timestamp field a new index starts with: modifyDate when the table
** has it, else the first date-time field named like a modify or update
** time, else none (incremental basepull off; reconcile covers the table).
*******************************************************************************/
public static String defaultTimestampField(QTableMetaData table)
{
if(table == null)
{
return (null);
}
if(isTimestampField(table, QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD))
{
return (QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD);
}
for(QFieldMetaData field : table.getFields().values())
{
String name = field.getName().toLowerCase(Locale.ROOT);
if(field.getType() == QFieldType.DATE_TIME && (name.contains("modif") || name.contains("updat")))
{
return (field.getName());
}
}
return (null);
}
/*******************************************************************************
** Why a table cannot be indexed at all (for anyone), or null when it can.
*******************************************************************************/
public static String whyNotEligible(QInstance qInstance, QuickSearchQBitConfig config, String tableName)
{
QTableMetaData table = qInstance == null || tableName == null ? null : qInstance.getTable(tableName);
if(table == null)
{
return ("table [" + tableName + "] is not in this application");
}
if(isQuickSearchTable(config, tableName) || table.getSourceQBitName() != null)
{
return (label(table) + " is internal to a QBit");
}
String normalized = tableName.replace("_", "").toLowerCase(Locale.ROOT);
if(SYSTEM_TABLES.contains(normalized))
{
return (label(table) + " is a system table");
}
if(SECRET_NAME.matcher(tableName).find())
{
return (label(table) + " looks like a store of secrets");
}
if(table.getIsHidden())
{
return (label(table) + " is hidden");
}
QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
if(backend != null && Boolean.TRUE.equals(backend.getUsesVariants()))
{
return (label(table) + " uses backend variants, which Quick Search does not support");
}
if(!table.isCapabilityEnabled(backend, Capability.TABLE_QUERY))
{
return (label(table) + " cannot be queried");
}
if(indexableFields(table).isEmpty())
{
return (label(table) + " has no field that can be searched");
}
return (null);
}
/*******************************************************************************
** Whether the current user may read a table.
*******************************************************************************/
public static boolean canRead(String tableName)
{
try
{
return (PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
}
catch(Exception e)
{
return (false);
}
}
/*******************************************************************************
** The tables the current user could add: eligible, readable and not
** already indexed, sorted by label.
*******************************************************************************/
public static List<QTableMetaData> eligibleTables(QInstance qInstance, QuickSearchQBitConfig config, Collection<String> alreadyIndexed)
{
List<QTableMetaData> tables = new ArrayList<>();
if(qInstance == null)
{
return (tables);
}
for(QTableMetaData table : qInstance.getTables().values())
{
if(alreadyIndexed.contains(table.getName()) || whyNotEligible(qInstance, config, table.getName()) != null || !canRead(table.getName()))
{
continue;
}
tables.add(table);
}
tables.sort((a, b) -> label(a).compareToIgnoreCase(label(b)));
return (tables);
}
/*******************************************************************************
** Whether a table is one of this QBit's own tables.
*******************************************************************************/
public static boolean isQuickSearchTable(QuickSearchQBitConfig config, String tableName)
{
return (tableName != null && config != null && tableName.startsWith(config.applyTablePrefix("quick_search_")));
}
/*******************************************************************************
** A table's label, or its name.
*******************************************************************************/
public static String label(QTableMetaData table)
{
return (table.getLabel() == null ? table.getName() : table.getLabel());
}
}
@@ -0,0 +1,425 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.io.Serializable;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
/*******************************************************************************
** Customizer of quick_search_index_field, the searchable fields.
**
** An admin's insert or edit is validated: the index row exists, the field
** exists on its table and can be indexed (not hidden, not isSensitive, not a
** password, blob or boolean), it is not already a field of that index, the
** weight is 1 to 1000, and the role and analyzer are known. The row is marked
** ownerEdited (an insert also source OWNER), so priming never overwrites it.
**
** Changes take effect for searches and real-time indexing at once (weights
** are applied at query time). After an admin's change other than the sort
** order, the index row is marked NEEDS_REINDEX, so the documents indexed
** before can be brought in line: Apply Search Index Changes previews the
** change and rebuilds the table. Removing a field leaves a tombstone, and
** adding it back clears it. Every change drops the cached config.
*******************************************************************************/
public class QuickSearchIndexFieldTableCustomizer implements TableCustomizerInterface
{
public static final int MAX_WEIGHT = 1000;
public static final List<String> OWNER_FIELDS = List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel", "sortOrder");
public static final List<String> DOCUMENT_FIELDS = List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel");
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
{
if(QuickSearchConfigWrites.isSystemWrite())
{
return (records);
}
Set<String> inBatch = new HashSet<>();
for(QRecord record : records)
{
Integer indexId = record.getValueInteger("quickSearchIndexId");
QRecord index = indexRow(indexId);
if(index == null)
{
record.addError(new BadInputStatusMessage("Choose the search index this field belongs to"));
continue;
}
String fieldName = record.getValueString("fieldName");
if(!validateField(record, index, fieldName, null) || !inBatch.add(indexId + "." + fieldName))
{
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
{
record.addError(new BadInputStatusMessage("Field [" + fieldName + "] is listed twice"));
}
continue;
}
if(record.getValue("weight") == null)
{
record.setValue("weight", 1);
}
if(!StringUtils.hasContent(record.getValueString("role")))
{
record.setValue("role", QuickSearchFieldRole.BODY.name());
}
if(record.getValue("includeLabel") == null)
{
record.setValue("includeLabel", false);
}
if(record.getValue("sortOrder") == null)
{
record.setValue("sortOrder", nextSortOrder(indexId));
}
validateOptions(record);
record.setValue("source", QuickSearchIndexField.SOURCE_OWNER);
record.setValue("ownerEdited", true);
}
return (records);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
{
if(QuickSearchConfigWrites.isSystemWrite())
{
return (records);
}
Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
for(QRecord record : records)
{
QRecord old = oldById.get(record.getValue("id"));
if(old == null)
{
continue;
}
if(record.getValues().containsKey("quickSearchIndexId") && !Objects.equals(record.getValueInteger("quickSearchIndexId"), old.getValueInteger("quickSearchIndexId")))
{
record.addError(new BadInputStatusMessage("A field cannot be moved to another search index; remove it and add it there"));
continue;
}
if(!changes(record, old, OWNER_FIELDS))
{
continue;
}
if(record.getValues().containsKey("fieldName") && !Objects.equals(record.getValueString("fieldName"), old.getValueString("fieldName")))
{
validateField(record, indexRow(old.getValueInteger("quickSearchIndexId")), record.getValueString("fieldName"), old.getValue("id"));
}
validateOptions(record);
record.setValue("ownerEdited", true);
}
return (records);
}
/*******************************************************************************
** Validate a field name against its index's table and the index's other
** fields (excludeId: the row being edited). Adds an error and returns false
** when it is not acceptable.
*******************************************************************************/
private static boolean validateField(QRecord record, QRecord index, String fieldName, Serializable excludeId) throws QException
{
if(index == null)
{
record.addError(new BadInputStatusMessage("The search index of this field no longer exists"));
return (false);
}
String tableName = index.getValueString("tableName");
QTableMetaData table = QContext.getQInstance().getTable(tableName);
String problem = QuickSearchFieldRules.whyNotIndexable(table, fieldName);
if(problem != null)
{
record.addError(new BadInputStatusMessage("Cannot index " + fieldName + ": " + problem));
return (false);
}
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, index.getValue("id")))
.withCriteria(new QFilterCriteria("fieldName", QCriteriaOperator.EQUALS, fieldName));
if(excludeId != null)
{
filter.withCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, excludeId));
}
QueryInput queryInput = new QueryInput();
queryInput.setTableName(fieldTableName());
queryInput.setFilter(filter);
if(CollectionUtils.nullSafeHasContents(new QueryAction().execute(queryInput).getRecords()))
{
record.addError(new BadInputStatusMessage(fieldName + " is already a searchable field of " + tableName));
return (false);
}
return (true);
}
/*******************************************************************************
** Weight 1 to MAX_WEIGHT; a known role and analyzer.
*******************************************************************************/
private static void validateOptions(QRecord record)
{
Integer weight = record.getValueInteger("weight");
if(record.getValues().containsKey("weight") && (weight == null || weight < 1 || weight > MAX_WEIGHT))
{
record.addError(new BadInputStatusMessage("Weight must be from 1 to " + MAX_WEIGHT));
}
String role = record.getValueString("role");
if(StringUtils.hasContent(role) && QuickSearchFieldRole.fromName(role, null) == null)
{
record.addError(new BadInputStatusMessage("Unknown role [" + role + "]"));
}
String analyzer = record.getValueString("analyzer");
if(StringUtils.hasContent(analyzer) && QuickSearchFieldAnalyzer.fromName(analyzer, null) == null)
{
record.addError(new BadInputStatusMessage("Unknown analyzer [" + analyzer + "]"));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
{
if(!QuickSearchConfigWrites.isSystemWrite())
{
for(QRecord record : succeeded(records))
{
QRecord index = indexRow(record.getValueInteger("quickSearchIndexId"));
if(index != null)
{
QuickSearchTombstones.remove(index.getValueString("tableName"), record.getValueString("fieldName"));
markNeedsReindex(index);
}
}
}
QuickSearchIndexTableCustomizer.invalidate();
return (records);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
{
if(!QuickSearchConfigWrites.isSystemWrite())
{
Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
Map<Integer, QRecord> marked = new HashMap<>();
for(QRecord record : succeeded(records))
{
QRecord old = oldById.get(record.getValue("id"));
if(old != null && changes(record, old, DOCUMENT_FIELDS))
{
Integer indexId = old.getValueInteger("quickSearchIndexId");
if(!marked.containsKey(indexId))
{
QRecord index = indexRow(indexId);
marked.put(indexId, index);
markNeedsReindex(index);
}
}
}
}
QuickSearchIndexTableCustomizer.invalidate();
return (records);
}
/*******************************************************************************
** An admin's removal leaves a tombstone (unless the whole index is going).
*******************************************************************************/
@Override
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
{
if(!QuickSearchConfigWrites.isSystemWrite())
{
for(QRecord record : succeeded(records))
{
QRecord index = indexRow(record.getValueInteger("quickSearchIndexId"));
if(index != null)
{
QuickSearchTombstones.add(index.getValueString("tableName"), record.getValueString("fieldName"), "removed by an admin");
markNeedsReindex(index);
}
}
}
QuickSearchIndexTableCustomizer.invalidate();
return (records);
}
/*******************************************************************************
** Mark the index NEEDS_REINDEX (a system write), unless a build is already
** queued or running.
*******************************************************************************/
static void markNeedsReindex(QRecord index) throws QException
{
if(index == null)
{
return;
}
String status = index.getValueString("status");
if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status)
|| AbstractIndexingStep.STATUS_REBUILDING.equals(status) || AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
{
return;
}
QuickSearchConfigWrites.asSystem(() ->
{
QuickSearchConfigPrimer.updateAll(QuickSearchTablePossibleValueProviders.config().getQuickSearchIndexTableName(),
List.of(new QRecord().withValue("id", index.getValue("id")).withValue("status", AbstractIndexingStep.STATUS_NEEDS_REINDEX)));
return (null);
});
}
/*******************************************************************************
** Whether the update changes any of the given fields.
*******************************************************************************/
private static boolean changes(QRecord record, QRecord old, List<String> fields)
{
for(String field : fields)
{
if(record.getValues().containsKey(field) && !Objects.equals(normalize(record.getValue(field)), normalize(old.getValue(field))))
{
return (true);
}
}
return (false);
}
/*******************************************************************************
** Compare values as strings, blank as null.
*******************************************************************************/
private static String normalize(Serializable value)
{
String s = value == null ? null : String.valueOf(value);
return (StringUtils.hasContent(s) ? s : null);
}
/*******************************************************************************
** The records without errors.
*******************************************************************************/
private static List<QRecord> succeeded(List<QRecord> records)
{
return (CollectionUtils.nonNullList(records).stream().filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())).toList());
}
/*******************************************************************************
** The index row, or null.
*******************************************************************************/
static QRecord indexRow(Integer indexId) throws QException
{
if(indexId == null)
{
return (null);
}
GetInput getInput = new GetInput(QuickSearchTablePossibleValueProviders.config().getQuickSearchIndexTableName());
getInput.setPrimaryKey(indexId);
return (new GetAction().execute(getInput).getRecord());
}
/*******************************************************************************
** One past the index's highest sortOrder, in steps of 10.
*******************************************************************************/
private static Integer nextSortOrder(Integer indexId) throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(fieldTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexId)));
int max = 0;
for(QRecord row : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
{
max = Math.max(max, row.getValueInteger("sortOrder") == null ? 0 : row.getValueInteger("sortOrder"));
}
return (max + 10);
}
/*******************************************************************************
** The field table's name.
*******************************************************************************/
private static String fieldTableName()
{
QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
return (config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME));
}
}
@@ -0,0 +1,239 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
/*******************************************************************************
** Customizer of quick_search_index, the searchable-table registry.
**
** Rows are added by Add Search Index, which validates the table (eligible,
** readable, not yet indexed), previews and queues the first build, and by
** priming; the generic insert is turned off. Any other insert is marked
** source OWNER and ownerEdited, and its schedule validated. An admin's edit of the
** enabled flag, interval or timestamp field marks the row ownerEdited, so
** priming never overwrites it; turning enabled back on queues a build. Any
** change drops the cached config, so it takes effect at once on this server.
** The QBit's own writes (QuickSearchConfigWrites) pass straight through.
*******************************************************************************/
public class QuickSearchIndexTableCustomizer implements TableCustomizerInterface
{
public static final List<String> CONFIG_FIELDS = List.of("enabled", "basepullIntervalMinutes", "basepullTimestampField");
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
{
if(QuickSearchConfigWrites.isSystemWrite())
{
return (records);
}
////////////////////////////////////////////////////////////////////////////
// rows are added by Add Search Index (which validates and previews) and //
// by priming; the generic insert is turned off. Anything else inserting //
// a row is the owner's doing: mark it so priming never overwrites it //
////////////////////////////////////////////////////////////////////////////
for(QRecord record : records)
{
if(record.getValue("source") == null)
{
record.setValue("source", QuickSearchIndex.SOURCE_OWNER);
}
if(record.getValue("ownerEdited") == null)
{
record.setValue("ownerEdited", true);
}
String tableName = record.getValueString("tableName");
if(QContext.getQInstance().getTable(tableName) != null)
{
validateSchedule(record, tableName);
}
}
return (records);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
{
if(QuickSearchConfigWrites.isSystemWrite())
{
return (records);
}
Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
for(QRecord record : records)
{
QRecord old = oldById.get(record.getValue("id"));
if(old == null || !changesConfig(record, old))
{
continue;
}
validateSchedule(record, old.getValueString("tableName"));
record.setValue("ownerEdited", true);
boolean wasEnabled = !Boolean.FALSE.equals(old.getValueBoolean("enabled"));
boolean nowEnabled = !Boolean.FALSE.equals(record.getValues().containsKey("enabled") ? record.getValueBoolean("enabled") : old.getValueBoolean("enabled"));
String status = old.getValueString("status");
if(!wasEnabled && nowEnabled && !AbstractIndexingStep.STATUS_REBUILDING.equals(status) && !AbstractIndexingStep.STATUS_BUILDING.equals(status))
{
record.setValue("status", AbstractIndexingStep.STATUS_PENDING_BUILD);
}
}
return (records);
}
/*******************************************************************************
** Whether the update changes the enabled flag, interval or timestamp field.
*******************************************************************************/
static boolean changesConfig(QRecord record, QRecord old)
{
for(String field : CONFIG_FIELDS)
{
if(record.getValues().containsKey(field) && !Objects.equals(normalize(record.getValue(field)), normalize(old.getValue(field))))
{
return (true);
}
}
return (false);
}
/*******************************************************************************
** Compare values as strings, blank as null (forms send "" and "true").
*******************************************************************************/
private static String normalize(Serializable value)
{
String s = value == null ? null : String.valueOf(value);
return (StringUtils.hasContent(s) ? s : null);
}
/*******************************************************************************
** A timestamp field must be a date-time field of the table; the interval
** cannot be negative.
*******************************************************************************/
private static void validateSchedule(QRecord record, String tableName)
{
String timestampField = record.getValueString("basepullTimestampField");
if(StringUtils.hasContent(timestampField) && !QuickSearchFieldRules.isTimestampField(QContext.getQInstance().getTable(tableName), timestampField))
{
record.addError(new BadInputStatusMessage("Timestamp field [" + timestampField + "] is not a date-time field of " + tableName));
}
Integer interval = record.getValueInteger("basepullIntervalMinutes");
if(record.getValues().containsKey("basepullIntervalMinutes") && interval != null && interval < 0)
{
record.addError(new BadInputStatusMessage("Update interval cannot be negative (0 means every update)"));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
{
invalidate();
return (records);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
{
invalidate();
return (records);
}
/*******************************************************************************
** An admin's delete leaves a tombstone, so code defaults do not add the
** table back.
*******************************************************************************/
@Override
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
{
if(!QuickSearchConfigWrites.isSystemWrite())
{
for(QRecord record : CollectionUtils.nonNullList(records))
{
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()) && record.getValueString("tableName") != null)
{
QuickSearchTombstones.add(record.getValueString("tableName"), null, "removed by an admin");
}
}
}
invalidate();
return (records);
}
/*******************************************************************************
** Drop the runtime's cached config.
*******************************************************************************/
static void invalidate()
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime != null)
{
runtime.invalidateConfig();
}
}
}
@@ -0,0 +1,149 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Customizer of quick_search_settings. Pausing stamps pausedAt and pausedBy
** (the Quick Search home shows "Paused since ... by ..."); resuming clears
** them and hands the changes deferred while paused (DEFERRED failed events)
** to the next basepull (PENDING). Every change drops the cached settings, so
** a pause takes effect at once on this server and within
** QuickSearchConfigStore.CACHE_TTL_MILLIS on the others.
*******************************************************************************/
public class QuickSearchSettingsTableCustomizer implements TableCustomizerInterface
{
private static final QLogger LOG = QLogger.getLogger(QuickSearchSettingsTableCustomizer.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
{
Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
for(QRecord record : records)
{
QRecord old = oldById.get(record.getValue("id"));
if(old == null || !record.getValues().containsKey("indexingPaused"))
{
continue;
}
boolean wasPaused = Boolean.TRUE.equals(old.getValueBoolean("indexingPaused"));
boolean nowPaused = Boolean.TRUE.equals(record.getValueBoolean("indexingPaused"));
if(!wasPaused && nowPaused)
{
record.setValue("pausedAt", Instant.now());
record.setValue("pausedBy", QuickSearchConfigWrites.currentUserName());
}
else if(wasPaused && !nowPaused)
{
record.setValue("pausedAt", null);
record.setValue("pausedBy", null);
record.setValue("pauseReason", null);
}
}
return (records);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
{
Map<Serializable, QRecord> oldById = oldRecordListToMap("id", oldRecordList).orElse(Map.of());
for(QRecord record : records)
{
QRecord old = oldById.get(record.getValue("id"));
if(old != null && CollectionUtils.nullSafeIsEmpty(record.getErrors()) && Boolean.TRUE.equals(old.getValueBoolean("indexingPaused"))
&& record.getValues().containsKey("indexingPaused") && !Boolean.TRUE.equals(record.getValueBoolean("indexingPaused")))
{
releaseDeferredEvents();
}
}
QuickSearchIndexTableCustomizer.invalidate();
return (records);
}
/*******************************************************************************
** Make the events deferred while paused PENDING, so the next basepull
** replays them. Never throws (they stay DEFERRED; a reconcile covers them).
*******************************************************************************/
static void releaseDeferredEvents()
{
try
{
QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
String table = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
if(QContext.getQInstance().getTable(table) == null)
{
return;
}
while(true)
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(table);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_DEFERRED)).withLimit(1000));
List<QRecord> deferred = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
if(deferred.isEmpty())
{
return;
}
List<QRecord> updates = new ArrayList<>();
deferred.forEach(r -> updates.add(new QRecord().withValue("id", r.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING)));
QuickSearchConfigPrimer.updateAll(table, updates);
LOG.info("Released changes deferred while Quick Search indexing was paused", logPair("count", updates.size()));
}
}
catch(Exception e)
{
LOG.warn("Could not release the changes deferred while indexing was paused; the next reconcile covers them", e);
}
}
}
@@ -0,0 +1,208 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
/*******************************************************************************
** The indexing status for the Quick Search home: the global pause ("Paused
** since ... by ..."), the last self-priming report ("New: 2 tables added and
** building..."), the tables queued or building, and those whose field
** changes wait for Apply Search Index Changes. The QBit's diagnostics widget
** shows it; hosts with their own home can use read() and bannerHtml().
**
** @param paused whether all indexing is paused
** @param pausedAt since when (null when not paused)
** @param pausedBy by whom
** @param pauseReason the reason given, if any
** @param lastPrimedAt when self-priming last ran
** @param lastPrimingSummary what it last changed
** @param building labels of tables queued for or in a build
** @param needsReindex labels of tables with field changes to apply
** @param recordsMode whether the config comes from the records (else
** code: the config tables are missing or not primed)
*******************************************************************************/
public record QuickSearchStatus(boolean paused, Instant pausedAt, String pausedBy, String pauseReason, Instant lastPrimedAt, String lastPrimingSummary,
List<String> building, List<String> needsReindex, boolean recordsMode)
{
private static final QLogger LOG = QLogger.getLogger(QuickSearchStatus.class);
private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
/*******************************************************************************
** Read the status. Never throws; parts that cannot be read are empty.
*******************************************************************************/
public static QuickSearchStatus read(QuickSearchRuntime runtime)
{
List<String> building = new ArrayList<>();
List<String> needsReindex = new ArrayList<>();
if(runtime == null)
{
return (new QuickSearchStatus(false, null, null, null, null, null, building, needsReindex, false));
}
try
{
String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
if(QContext.getQInstance() != null && QContext.getQInstance().getTable(indexTable) != null)
{
for(QRecord row : QuickSearchConfigPrimer.queryAll(indexTable))
{
String status = row.getValueString("status");
String label = labelOf(row.getValueString("tableName"));
if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
{
continue;
}
if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status)
|| AbstractIndexingStep.STATUS_REBUILDING.equals(status))
{
building.add(label);
}
else if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
{
needsReindex.add(label);
}
}
}
}
catch(Exception e)
{
LOG.debug("Could not read the index rows for the Quick Search status", e);
}
QRecord settings = runtime.getConfigStore().getSettings();
boolean paused = settings != null && Boolean.TRUE.equals(settings.getValueBoolean("indexingPaused"));
return (new QuickSearchStatus(paused,
settings == null ? null : settings.getValueInstant("pausedAt"),
settings == null ? null : settings.getValueString("pausedBy"),
settings == null ? null : settings.getValueString("pauseReason"),
settings == null ? null : settings.getValueInstant("lastPrimedAt"),
settings == null ? null : settings.getValueString("lastPrimingSummary"),
building, needsReindex, runtime.getConfigStore().isRecordsMode()));
}
/*******************************************************************************
** "Paused since 2026-10-12T14:03:00Z by Jane Admin (reason)".
*******************************************************************************/
public String pausedText()
{
return ("Paused since " + (pausedAt == null ? "an unknown time" : WHEN.format(pausedAt)) + " by " + (pausedBy == null ? "an unknown user" : pausedBy)
+ (StringUtils.hasContent(pauseReason) ? " (" + pauseReason + ")" : "")
+ ". Nothing is indexed; search keeps serving the current index, and changes are indexed when indexing resumes.");
}
/*******************************************************************************
** The building line: "Building: Blog Post, Page", or null.
*******************************************************************************/
public String buildingText()
{
if(building.isEmpty())
{
return (null);
}
return ((StringUtils.hasContent(lastPrimingSummary) && lastPrimingSummary.startsWith("New:") ? lastPrimingSummary + " " : "")
+ "Building now or with the next update: " + String.join(", ", building) + ".");
}
/*******************************************************************************
** The pending-changes line, or null.
*******************************************************************************/
public String needsReindexText()
{
return (needsReindex.isEmpty() ? null : "Field changes waiting to be applied (Apply Search Index Changes): " + String.join(", ", needsReindex) + ".");
}
/*******************************************************************************
** The status as standalone HTML, for a host's own home widget: one line
** per item, empty when there is nothing to report.
*******************************************************************************/
public String bannerHtml()
{
StringBuilder html = new StringBuilder();
if(paused)
{
html.append(banner("#b26a00", "#fff4e5", pausedText()));
}
if(buildingText() != null)
{
html.append(banner("#0b5394", "#e8f0fe", buildingText()));
}
if(needsReindexText() != null)
{
html.append(banner("#5f6368", "#f1f3f4", needsReindexText()));
}
return (html.toString());
}
/*******************************************************************************
** One banner line.
*******************************************************************************/
private static String banner(String color, String background, String text)
{
return ("<div style=\"padding:8px 12px;margin:0 0 8px 0;border-radius:6px;border-left:4px solid " + color + ";background:" + background + ";color:#202124\">"
+ DiagnosticsHtml.escape(text) + "</div>");
}
/*******************************************************************************
** A table's label, or its name.
*******************************************************************************/
private static String labelOf(String tableName)
{
var table = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
return (table == null ? String.valueOf(tableName) : QuickSearchFieldRules.label(table));
}
/*******************************************************************************
** The settings table name for links ("/app/" + name + "/1").
*******************************************************************************/
public static String settingsPath(QuickSearchRuntime runtime)
{
return ("/app/" + runtime.getConfig().applyTablePrefix(QuickSearchSettings.TABLE_NAME) + "/" + QuickSearchSettings.SETTINGS_ID);
}
}
@@ -0,0 +1,321 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
/*******************************************************************************
** Custom possible-value providers over the instance's tables and fields, for
** the config screens:
** - SearchableTables: the indexed tables (quick_search_searchable_table),
** shown by label wherever a table name is stored;
** - EligibleTables: tables the current user could add (readable, eligible,
** not yet indexed), for Add Search Index;
** - IndexableFields and TimestampFields: the indexable or date-time fields
** of the table the form is about, read from the form's other values
** (tableName, or the field row's quickSearchIndexId).
*******************************************************************************/
public final class QuickSearchTablePossibleValueProviders
{
/*******************************************************************************
** Utility holder - no instances.
*******************************************************************************/
private QuickSearchTablePossibleValueProviders()
{
}
/*******************************************************************************
** The QBit's config: the runtime's, else the static facade's.
*******************************************************************************/
static QuickSearchQBitConfig config()
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
return (runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig());
}
/*******************************************************************************
** Table possible values whose id is the table name and label its label.
*******************************************************************************/
abstract static class AbstractTableProvider implements QCustomPossibleValueProvider<String>
{
/*******************************************************************************
** The table names to offer.
*******************************************************************************/
abstract List<String> tableNames() throws QException;
/*******************************************************************************
** Any table in the instance, by label.
*******************************************************************************/
@Override
public QPossibleValue<String> getPossibleValue(Serializable idValue)
{
String tableName = ValueUtils.getValueAsString(idValue);
QTableMetaData table = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
return (tableName == null ? null : new QPossibleValue<>(tableName, table == null ? tableName : QuickSearchFieldRules.label(table)));
}
/*******************************************************************************
** The offered tables matching the search term (or ids).
*******************************************************************************/
@Override
public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
{
List<QPossibleValue<String>> values = new ArrayList<>();
for(String tableName : tableNames())
{
QPossibleValue<String> value = getPossibleValue(tableName);
if(matches(input, value))
{
values.add(value);
}
}
return (values);
}
}
/*******************************************************************************
** The indexed tables.
*******************************************************************************/
public static class SearchableTables extends AbstractTableProvider
{
/*******************************************************************************
**
*******************************************************************************/
@Override
List<String> tableNames()
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
List<String> names = new ArrayList<>();
List<QuickSearchableTableConfig> tables = runtime != null ? runtime.getDiscoveredTables() : QuickSearchQBitContext.getDiscoveredTables();
for(QuickSearchableTableConfig tableConfig : CollectionUtils.nonNullList(tables))
{
names.add(tableConfig.getTableName());
}
return (names);
}
}
/*******************************************************************************
** The tables the current user could add to Quick Search.
*******************************************************************************/
public static class EligibleTables extends AbstractTableProvider
{
/*******************************************************************************
**
*******************************************************************************/
@Override
List<String> tableNames() throws QException
{
QuickSearchQBitConfig config = config();
Set<String> indexed = new LinkedHashSet<>();
for(QRecord row : QuickSearchConfigPrimer.queryAll(config.getQuickSearchIndexTableName()))
{
indexed.add(row.getValueString("tableName"));
}
List<String> names = new ArrayList<>();
QuickSearchFieldRules.eligibleTables(QContext.getQInstance(), config, indexed).forEach(t -> names.add(t.getName()));
return (names);
}
}
/*******************************************************************************
** Field possible values of the form's table; id is the field name.
*******************************************************************************/
abstract static class AbstractFieldProvider implements QCustomPossibleValueProvider<String>
{
/*******************************************************************************
** Whether to offer a field.
*******************************************************************************/
abstract boolean offers(QTableMetaData table, QFieldMetaData field);
/*******************************************************************************
** A field name is shown as itself: without the form's values the table is
** not known here.
*******************************************************************************/
@Override
public QPossibleValue<String> getPossibleValue(Serializable idValue)
{
String fieldName = ValueUtils.getValueAsString(idValue);
return (fieldName == null ? null : new QPossibleValue<>(fieldName, fieldName));
}
/*******************************************************************************
** The offered fields of the form's table, labeled "Label (name)".
*******************************************************************************/
@Override
public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
{
List<QPossibleValue<String>> values = new ArrayList<>();
QTableMetaData table = resolveTable(input.getOtherValues());
if(table == null)
{
return (values);
}
for(QFieldMetaData field : table.getFields().values())
{
if(!offers(table, field))
{
continue;
}
QPossibleValue<String> value = new QPossibleValue<>(field.getName(), (field.getLabel() == null ? field.getName() : field.getLabel()) + " (" + field.getName() + ")");
if(matches(input, value))
{
values.add(value);
}
}
return (values);
}
}
/*******************************************************************************
** Fields that can be indexed.
*******************************************************************************/
public static class IndexableFields extends AbstractFieldProvider
{
/*******************************************************************************
**
*******************************************************************************/
@Override
boolean offers(QTableMetaData table, QFieldMetaData field)
{
return (QuickSearchFieldRules.isIndexable(table, field.getName()));
}
}
/*******************************************************************************
** Date-time fields, for the basepull timestamp.
*******************************************************************************/
public static class TimestampFields extends AbstractFieldProvider
{
/*******************************************************************************
**
*******************************************************************************/
@Override
boolean offers(QTableMetaData table, QFieldMetaData field)
{
return (QuickSearchFieldRules.isTimestampField(table, field.getName()));
}
}
/*******************************************************************************
** The table a form is about: its searchTableName or tableName value, else
** the table of its quickSearchIndexId row. Null when none resolves.
*******************************************************************************/
static QTableMetaData resolveTable(Map<String, Serializable> otherValues) throws QException
{
QInstance qInstance = QContext.getQInstance();
if(otherValues == null || qInstance == null)
{
return (null);
}
////////////////////////////////////////////////////////////////////////
// Add Search Index's searchTableName first: a process form also gets //
// tableName, the table the process runs from //
////////////////////////////////////////////////////////////////////////
String tableName = ValueUtils.getValueAsString(otherValues.get("searchTableName"));
if(!StringUtils.hasContent(tableName))
{
tableName = ValueUtils.getValueAsString(otherValues.get("tableName"));
}
if(!StringUtils.hasContent(tableName) && otherValues.get("quickSearchIndexId") != null)
{
GetInput getInput = new GetInput(config().getQuickSearchIndexTableName());
getInput.setPrimaryKey(ValueUtils.getValueAsInteger(otherValues.get("quickSearchIndexId")));
QRecord row = new GetAction().execute(getInput).getRecord();
tableName = row == null ? null : row.getValueString("tableName");
}
return (StringUtils.hasContent(tableName) ? qInstance.getTable(tableName) : null);
}
/*******************************************************************************
** Whether a value is wanted: in idList when one is given, else its label or
** id contains the search term (any case).
*******************************************************************************/
static boolean matches(SearchPossibleValueSourceInput input, QPossibleValue<String> value)
{
if(value == null)
{
return (false);
}
if(CollectionUtils.nullSafeHasContents(input.getIdList()))
{
return (input.getIdList().stream().anyMatch(id -> value.getId().equals(ValueUtils.getValueAsString(id))));
}
String term = input.getSearchTerm();
if(!StringUtils.hasContent(term))
{
return (true);
}
String lower = term.toLowerCase(Locale.ROOT);
return (value.getLabel().toLowerCase(Locale.ROOT).contains(lower) || value.getId().toLowerCase(Locale.ROOT).contains(lower));
}
}
@@ -0,0 +1,140 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.config;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
/*******************************************************************************
** Reads and writes quick_search_config_tombstone: what an admin removed, so
** priming does not add it back. Every write is a system write. With no
** tombstone table in the instance these do nothing (and priming then cannot
** tell a removed code table from a new one; add the table's DDL).
*******************************************************************************/
public final class QuickSearchTombstones
{
/*******************************************************************************
** Utility class - no instances.
*******************************************************************************/
private QuickSearchTombstones()
{
}
/*******************************************************************************
** The tombstone table's name, or null when it is not in the instance.
*******************************************************************************/
static String tableName()
{
QuickSearchQBitConfig config = QuickSearchTablePossibleValueProviders.config();
String name = config == null ? null : config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME);
return (name != null && QContext.getQInstance() != null && QContext.getQInstance().getTable(name) != null ? name : null);
}
/*******************************************************************************
** Remember that a table (fieldName null) or one of its fields was removed.
*******************************************************************************/
public static void add(String tableName, String fieldName, String reason) throws QException
{
String name = tableName();
if(name == null || !find(name, tableName, fieldName).isEmpty())
{
return;
}
QuickSearchConfigWrites.asSystem(() -> QuickSearchConfigPrimer.insertOne(name, new QRecord()
.withValue("tableName", tableName)
.withValue("fieldName", fieldName)
.withValue("removedBy", QuickSearchConfigWrites.currentUserName())
.withValue("reason", reason)));
}
/*******************************************************************************
** Forget a removal (the admin added the table or field back).
*******************************************************************************/
public static void remove(String tableName, String fieldName) throws QException
{
String name = tableName();
if(name == null)
{
return;
}
List<Serializable> ids = new ArrayList<>();
find(name, tableName, fieldName).forEach(r -> ids.add(r.getValue("id")));
if(ids.isEmpty())
{
return;
}
QuickSearchConfigWrites.asSystem(() ->
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(name);
deleteInput.setPrimaryKeys(ids);
return (new DeleteAction().execute(deleteInput));
});
}
/*******************************************************************************
** Whether a table (fieldName null) or field has a tombstone.
*******************************************************************************/
public static boolean exists(String tableName, String fieldName) throws QException
{
String name = tableName();
return (name != null && !find(name, tableName, fieldName).isEmpty());
}
/*******************************************************************************
** The tombstones for one table (fieldName null) or field.
*******************************************************************************/
static List<QRecord> find(String name, String tableName, String fieldName) throws QException
{
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName));
filter.addCriteria(fieldName == null
? new QFilterCriteria("fieldName", QCriteriaOperator.IS_BLANK)
: new QFilterCriteria("fieldName", QCriteriaOperator.EQUALS, fieldName));
QueryInput queryInput = new QueryInput();
queryInput.setTableName(name);
queryInput.setFilter(filter);
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
}
}
@@ -36,7 +36,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Writes quickSearchDiagnosticEvent rows: failed searches, diagnostic tool
** Writes quick_search_diagnostic_event rows: failed searches, diagnostic tool
** runs (the audit trail) and refused runs.
** Best effort and off the caller's thread: events are queued (at most
** QUEUE_CAPACITY) and inserted by one daemon thread as the system user, so
@@ -87,7 +87,7 @@ public class DiagnosticEventRecorder implements AutoCloseable
public void record(QuickSearchDiagnosticEvent event)
{
QInstance qInstance = QContext.getQInstance();
String tableName = config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
String tableName = config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
if(event == null || qInstance == null || !Boolean.TRUE.equals(config.getEnableDiagnosticTools()) || qInstance.getTable(tableName) == null)
{
return;
@@ -124,7 +124,7 @@ public class DiagnosticEventRecorder implements AutoCloseable
*******************************************************************************/
private void insert(QInstance qInstance, String tableName, QuickSearchDiagnosticEvent event)
{
QSession session = new QSystemUserSession();
QSession session = new QSystemUserSession(qInstance);
try
{
QContext.init(qInstance, session);
@@ -18,20 +18,27 @@ package com.kingsrook.qbits.quicksearch.diagnostics;
import java.util.Collection;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Shared pieces of the diagnostic tools: tool names, the current user, the
** per-user rate limit, and recording tool runs and search failures as
** quickSearchDiagnosticEvent rows (the audit trail).
** quick_search_diagnostic_event rows (the audit trail).
*******************************************************************************/
public final class DiagnosticsSupport
{
@@ -45,6 +52,26 @@ public final class DiagnosticsSupport
/*******************************************************************************
** Latest saved validation, shared by the home widget and host status API.
** This does not run validation or scan source tables.
*******************************************************************************/
public static QRecord latestValidation(QuickSearchQBitConfig config) throws QException
{
String tableName = config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
if(QContext.getQInstance().getTable(tableName) == null)
{
return (null);
}
var input = new QueryInput(tableName)
.withFilter(new QQueryFilter()
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
var rows = new QueryAction().execute(input).getRecords();
return (rows == null || rows.isEmpty() ? null : rows.get(0));
}
/*******************************************************************************
** Utility class - no instances.
*******************************************************************************/
@@ -28,6 +28,18 @@ import java.util.Set;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.RawResponse;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -46,18 +58,6 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.RawResponse;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -72,11 +72,11 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** - per table: database rows vs indexed documents, a sample of records
** (the lowest and highest primary keys) checked present and current in
** the index, a random sample of documents checked to still have a record,
** the quickSearchIndex row's status and last run, failed events waiting.
** the quick_search_index row's status and last run, failed events waiting.
** Database reads run as the system user, so counts compare like with like
** (the index holds every row whatever the record locks). The report holds
** counts, ids and field names, never record values. Read-only, except that
** persist() saves the report as a quickSearchValidationRun row.
** persist() saves the report as a quick_search_validation_run row.
*******************************************************************************/
public class IndexValidator
{
@@ -293,6 +293,16 @@ public class IndexValidator
/*******************************************************************************
** Shared cluster health interpretation for validation and the host status API.
*******************************************************************************/
public static Level healthLevel(String status)
{
return (healthFinding("Health", status).level());
}
/*******************************************************************************
** The live mapping against the mapping this version creates.
*******************************************************************************/
@@ -622,7 +632,7 @@ public class IndexValidator
{
try
{
String failedTable = runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
String failedTable = runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
if(QContext.getQInstance().getTable(failedTable) == null)
{
return;
@@ -646,13 +656,13 @@ public class IndexValidator
/*******************************************************************************
** Save a report as a quickSearchValidationRun row (as the system user);
** Save a report as a quick_search_validation_run row (as the system user);
** returns its id, or null when the table is not configured or the insert
** fails (logged).
*******************************************************************************/
public Integer persist(QuickSearchRuntime runtime, Report report, String triggerType, String userName)
{
String tableName = runtime.getConfig().applyPrefix(QuickSearchValidationRun.TABLE_NAME);
String tableName = runtime.getConfig().applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
{
return (null);
@@ -227,7 +227,7 @@ public class QuickSearchDeveloperPanelProvider implements DeveloperPanelProvider
/////////////////////////////////
// failed index events, if any //
/////////////////////////////////
String failedTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
String failedTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
if(recordId != null && mayRead(failedTable))
{
DeveloperPanelBlock failedBlock = failedEventsBlock(failedTable, tableName, recordId);
@@ -19,6 +19,12 @@ package com.kingsrook.qbits.quicksearch.diagnostics;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.config.QuickSearchStatus;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@@ -34,11 +40,6 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
/*******************************************************************************
@@ -79,6 +80,8 @@ public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
QuickSearchQBitConfig config = runtime.getConfig();
DiagnosticsHtml html = new DiagnosticsHtml().open();
renderIndexingStatus(html, runtime);
SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(QContext.getQInstance(), runtime);
html.subheading("Global search on this server");
if(!Boolean.TRUE.equals(state.quickSearchProvider()) || !Boolean.TRUE.equals(state.serveCoreRecordSearch()))
@@ -104,7 +107,9 @@ public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
renderLastValidation(html, config);
html.subheading("Tools");
html.paragraph("On Quick Search Index (Actions): Search Playground, Validate Search Index, Test Search Analyzer. "
html.paragraph("On Quick Search Index (Actions): Add Search Index, Apply Search Index Changes, Disable and Remove Search Index, "
+ "Search Playground, Validate Search Index, Test Search Analyzer. Searchable fields are edited on each index record. "
+ "On Quick Search Settings: Pause and Resume Indexing; on Quick Search Index Run: Cancel Index Run. "
+ "On any indexed record and on Quick Search Failed Event rows: Inspect Search Document and Reindex in Search. "
+ "Every run is recorded in Quick Search Diagnostic Event.");
return (html.close().toString());
@@ -112,11 +117,46 @@ public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
/*******************************************************************************
** Indexing: paused (since when, by whom), what self-priming added and is
** building, and field changes waiting to be applied.
*******************************************************************************/
static void renderIndexingStatus(DiagnosticsHtml html, QuickSearchRuntime runtime)
{
QuickSearchStatus status = QuickSearchStatus.read(runtime);
html.subheading("Indexing");
String banner = status.bannerHtml();
if(!banner.isEmpty())
{
html.raw(banner);
}
if(status.paused())
{
html.raw("<p><a href=\"" + DiagnosticsHtml.escape(QuickSearchStatus.settingsPath(runtime)) + "\">Quick Search Settings</a>: Resume Indexing (Actions).</p>");
}
else
{
html.status(Level.OK, "Indexing is running (real-time and scheduled). Pause it from "
+ "Quick Search Settings (Actions: Pause Indexing); search keeps serving while paused.");
}
if(!status.recordsMode())
{
html.status(Level.INFO, "The searchable tables come from code: the Quick Search config tables are not in this application yet, or not primed. "
+ "See docs/MIGRATION-1.1.md (data-driven config).");
}
else if(status.lastPrimedAt() != null)
{
html.paragraph("Self-priming ran " + status.lastPrimedAt() + ": " + (status.lastPrimingSummary() == null ? "no changes." : status.lastPrimingSummary()));
}
}
/*******************************************************************************
*******************************************************************************/
private static void renderRecentFailures(DiagnosticsHtml html, QuickSearchQBitConfig config)
{
String tableName = config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
String tableName = config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
html.subheading("Recent search failures (all servers)");
if(QContext.getQInstance().getTable(tableName) == null)
{
@@ -161,7 +201,7 @@ public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
*******************************************************************************/
private static void renderLastValidation(DiagnosticsHtml html, QuickSearchQBitConfig config)
{
String tableName = config.applyPrefix(QuickSearchValidationRun.TABLE_NAME);
String tableName = config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME);
html.subheading("Latest index validation");
if(QContext.getQInstance().getTable(tableName) == null)
{
@@ -171,18 +211,14 @@ public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
try
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
if(rows.isEmpty())
QRecord row = DiagnosticsSupport.latestValidation(config);
if(row == null)
{
html.paragraph("None yet. Run Validate Search Index from Quick Search Index (Actions)"
+ (config.getValidationCronExpression() == null ? "." : "; it is also scheduled (" + config.getValidationCronExpression() + ")."));
return;
}
QRecord row = rows.get(0);
String status = row.getValueString("status");
Level level = QuickSearchValidationRun.STATUS_PASS.equals(status) ? Level.OK : (QuickSearchValidationRun.STATUS_WARN.equals(status) ? Level.WARN : Level.FAIL);
html.raw("<p>" + DiagnosticsHtml.badge(level) + " <a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
@@ -77,6 +77,10 @@ public class RecordReindexer
{
throw (new QException("Table [" + tableName + "] is disabled in Quick Search Index; enable it before reindexing its records"));
}
if(runtime.isIndexingPaused())
{
throw (new QException("Quick Search indexing is paused; resume it in Quick Search Settings before reindexing records"));
}
Set<String> ids = new LinkedHashSet<>(recordIds);
Map<String, QRecord> readable = DiagnosticRecordReads.loadForIndexing(tableName, ids);
@@ -0,0 +1,170 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.diagnostics;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.ReentrantLock;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Instance-owned five-second cache with one in-flight refresh. A stuck backend
** cannot grow a queue or create more workers. Runtime shutdown stops the worker.
*******************************************************************************/
public class StatusCache implements AutoCloseable
{
private final ReentrantLock lock = new ReentrantLock();
private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable ->
{
Thread thread = new Thread(runnable, "quick-search-status");
thread.setDaemon(true);
return (thread);
});
private Future<QuickSearchStatus> refresh;
private QuickSearchStatus cached;
private long expiresAt;
private long refreshStartedAt;
/*******************************************************************************
** Authorization belongs to the public service, before reaching this cache.
** Only refreshes consume the existing per-user diagnostic rate limit.
*******************************************************************************/
public QuickSearchStatus get(QuickSearchRuntime runtime) throws QException
{
return (get(runtime, Instant.now().plusSeconds(2)));
}
/*******************************************************************************
** Bound both lock contention and the shared refresh wait by the caller's deadline.
*******************************************************************************/
public QuickSearchStatus get(QuickSearchRuntime runtime, Instant deadline) throws QException
{
try
{
long remaining = Duration.between(Instant.now(), deadline).toNanos();
if(remaining <= 0 || !lock.tryLock(remaining, TimeUnit.NANOSECONDS))
{
throw (new QException("Status deadline exceeded"));
}
try
{
return (read(runtime, deadline));
}
finally
{
lock.unlock();
}
}
catch(InterruptedException e)
{
Thread.currentThread().interrupt();
throw (new QException("Status refresh interrupted", e));
}
}
/*******************************************************************************
** Read or refresh while holding the deadline-aware cache lock.
*******************************************************************************/
private QuickSearchStatus read(QuickSearchRuntime runtime, Instant deadline) throws QException
{
if(cached != null && System.nanoTime() < expiresAt)
{
return (cached);
}
String homeHref = "/app/" + runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
// A refresh that finished while nobody was polling must not resurrect an old snapshot.
if(refresh != null && refresh.isDone() && System.nanoTime() - refreshStartedAt >= TimeUnit.SECONDS.toNanos(5))
{
refresh = null;
}
if(refresh == null)
{
DiagnosticsSupport.checkRateLimit(runtime, "STATUS");
QInstance instance = QContext.getQInstance();
refreshStartedAt = System.nanoTime();
refresh = executor.submit(() ->
{
QContext.init(instance, new QSystemUserSession(instance));
try
{
return (StatusDiagnostics.read(runtime));
}
finally
{
QContext.clear();
}
});
}
try
{
long remaining = Math.max(0, Duration.between(Instant.now(), deadline).toNanos());
cached = refresh.get(Math.min(TimeUnit.SECONDS.toNanos(2), remaining), TimeUnit.NANOSECONDS);
refresh = null;
}
catch(TimeoutException e)
{
// Leave the one worker in flight; never pile up abandoned requests.
cached = StatusDiagnostics.unavailable(QuickSearchStatus.State.DEGRADED, "Status refresh timed out; measurements are unknown.", homeHref);
}
catch(InterruptedException e)
{
Thread.currentThread().interrupt();
throw (new QException("Status refresh interrupted", e));
}
catch(ExecutionException e)
{
refresh = null;
cached = StatusDiagnostics.unavailable(QuickSearchStatus.State.DEGRADED, "Status diagnostics could not be read.", homeHref);
}
expiresAt = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
return (cached);
}
/*******************************************************************************
** Stop the bounded worker when its runtime closes.
*******************************************************************************/
@Override
public void close()
{
executor.shutdownNow();
}
}
@@ -0,0 +1,249 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.diagnostics;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.api.QuickSearchApi;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.State;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
** Cheap projection of existing diagnostics: one health request, bounded metadata
** reads, saved IndexValidator counts. No source scans or per-table remote calls.
*******************************************************************************/
public final class StatusDiagnostics
{
public static final int MAX_TABLES = 100;
public static final int MAX_BACKLOG = 1000;
private static final ObjectMapper MAPPER = new ObjectMapper();
/*******************************************************************************
** Utility class.
*******************************************************************************/
private StatusDiagnostics()
{
}
/*******************************************************************************
** Read as the trusted diagnostics worker after the caller's admin check.
*******************************************************************************/
public static QuickSearchStatus read(QuickSearchRuntime runtime) throws Exception
{
String homeHref = "/app/" + runtime.getConfig().applyPrefix(QuickSearchApi.APP_NAME);
List<QRecord> rows = query(runtime.getConfig().getQuickSearchIndexTableName(), new QQueryFilter()
.withOrderBy(new QFilterOrderBy("id")).withLimit(MAX_TABLES + 1));
boolean truncated = rows.size() > MAX_TABLES;
QRecord validation = DiagnosticsSupport.latestValidation(runtime.getConfig());
String json = validation == null ? null : validation.getValueString("reportJson");
JsonNode report = json == null || json.length() > 1_000_000 ? MAPPER.createObjectNode() : MAPPER.readTree(json);
Instant countsAt = validation == null ? null : validation.getValueInstant("endTime");
List<IndexRow> indexes = new ArrayList<>();
Instant lastSync = null;
String lastError = runtime.getLastRecordSearchFailureMessage();
Instant lastErrorAt = runtime.getLastRecordSearchFailureTime();
boolean degraded = truncated || runtime.isRecordSearchBackingOff() || (rows.isEmpty() && !runtime.getCodeTables().isEmpty());
for(QRecord row : rows.subList(0, Math.min(rows.size(), MAX_TABLES)))
{
String tableName = row.getValueString("tableName");
Long documents = row.getValueLong("documentCount");
Long source = null;
Instant measuredAt = null;
for(JsonNode stats : report.path("tableStats"))
{
if(tableName.equals(stats.path("tableName").asText()))
{
documents = stats.path("indexCount").isNumber() ? stats.path("indexCount").longValue() : null;
source = stats.path("databaseCount").isNumber() ? stats.path("databaseCount").longValue() : null;
measuredAt = countsAt;
break;
}
}
Instant sync = latest(row.getValueInstant("lastBasepullTime"), latest(row.getValueInstant("lastFullReindexTime"), row.getValueInstant("lastReconcileTime")));
lastSync = latest(lastSync, sync);
String error = row.getValueString("lastErrorMessage");
if(error != null && !error.isBlank() && (lastErrorAt == null || (row.getValueInstant("modifyDate") != null && row.getValueInstant("modifyDate").isAfter(lastErrorAt))))
{
lastError = error;
lastErrorAt = row.getValueInstant("modifyDate");
}
boolean enabled = !Boolean.FALSE.equals(row.getValueBoolean("enabled"));
Integer realTimeErrors = row.getValueInteger("realTimeErrorCount");
degraded |= enabled && (!"ACTIVE".equals(row.getValueString("status")) || "FAILED".equals(row.getValueString("lastRunStatus"))
|| (realTimeErrors != null && realTimeErrors > 0) || (source != null && documents != null && !source.equals(documents)));
indexes.add(new IndexRow(tableName, enabled, row.getValueString("status"), documents, source, measuredAt, sync, error, realTimeErrors));
}
List<QRecord> backlog = query(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME),
new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("PENDING", "EXHAUSTED", "AWAITING_REINDEX")))
.withOrderBy(new QFilterOrderBy("id")).withLimit(MAX_BACKLOG + 1));
int pending = 0;
int exhausted = 0;
for(QRecord row : backlog.subList(0, Math.min(backlog.size(), MAX_BACKLOG)))
{
if("EXHAUSTED".equals(row.getValueString("status")))
{
exhausted++;
}
else
{
pending++;
}
}
boolean paused = runtime.isIndexingPaused();
degraded |= paused || pending > 0 || exhausted > 0;
Boolean reachable = null;
String health = null;
Integer nodes = null;
Integer active = null;
Integer unassigned = null;
Integer unassignedReplicas = null;
Integer targetReplicas = null;
Boolean replicaSettingsFailed = null;
State state = degraded ? State.DEGRADED : State.OK;
if(!truncated && !indexes.isEmpty() && indexes.stream().noneMatch(row -> Boolean.TRUE.equals(row.enabled())))
{
state = State.DISABLED;
}
else
{
try
{
var response = new QuickSearchDiagnosticsOperations(runtime.getClient()).clusterHealth();
reachable = true;
QuickSearchDiagnosticsOperations.requireOk(response, "Cluster health unavailable");
JsonNode body = response.body();
health = body.path("status").asText("unknown");
nodes = number(body, "number_of_nodes");
active = number(body, "active_shards");
unassigned = number(body, "unassigned_shards");
unassignedReplicas = unassignedReplicaShards(body, unassigned);
var replicaStatus = runtime.getClient().getReplicaStatus();
if(replicaStatus != null)
{
targetReplicas = replicaStatus.targetReplicas();
replicaSettingsFailed = replicaStatus.failed();
}
boolean singleNodeReplicas = QuickSearchStatus.isSingleNodeReplicaYellow(health, nodes, unassignedReplicas);
if((IndexValidator.healthLevel(health) != DiagnosticsHtml.Level.OK && !singleNodeReplicas) || body.path("timed_out").asBoolean())
{
state = "red".equals(health) ? State.DOWN : State.DEGRADED;
}
}
catch(QException e)
{
if(reachable == null)
{
reachable = false;
}
state = State.DOWN;
lastError = "OpenSearch health is unavailable; see Quick Search diagnostics.";
}
}
return (new QuickSearchStatus(state, reachable, health, nodes, active, unassigned, unassignedReplicas, targetReplicas, replicaSettingsFailed,
paused, indexes, pending, exhausted, backlog.size() > MAX_BACKLOG,
truncated, lastSync, lastError, Instant.now(), homeHref));
}
/*******************************************************************************
** Unknown measurements for disabled or failed snapshots.
*******************************************************************************/
public static QuickSearchStatus unavailable(State state, String error, String homeHref)
{
return (new QuickSearchStatus(state, null, null, null, null, null, null, List.of(), null, null, false, false, null, error, Instant.now(), homeHref));
}
/*******************************************************************************
** Bounded metadata queries; failures propagate rather than looking healthy.
*******************************************************************************/
private static List<QRecord> query(String tableName, QQueryFilter filter) throws QException
{
return (CollectionUtils.nonNullList(new QueryAction().execute(new QueryInput(tableName).withFilter(filter)).getRecords()));
}
/*******************************************************************************
** From index-level cluster health: the unassigned shards when nothing is
** initializing and every index has all its primaries active (so every
** unassigned shard is a replica); null otherwise or when unknown.
*******************************************************************************/
static Integer unassignedReplicaShards(JsonNode body, Integer unassigned)
{
if(unassigned == null || !body.path("indices").isObject() || !Integer.valueOf(0).equals(number(body, "initializing_shards")))
{
return (null);
}
for(JsonNode index : body.path("indices"))
{
Integer primaries = number(index, "number_of_shards");
if(primaries == null || !primaries.equals(number(index, "active_primary_shards")))
{
return (null);
}
}
return (unassigned);
}
/*******************************************************************************
** Preserve unknown counters from incomplete cluster responses.
*******************************************************************************/
private static Integer number(JsonNode body, String field)
{
return (body.path(field).isNumber() ? body.path(field).intValue() : null);
}
/*******************************************************************************
** Newest non-null timestamp.
*******************************************************************************/
private static Instant latest(Instant first, Instant second)
{
return (first == null ? second : second == null || first.isAfter(second) ? first : second);
}
}
@@ -0,0 +1,290 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.health;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus;
import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.health.HealthDetails;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
/*******************************************************************************
** Central health projections of the existing, shared diagnostic snapshot.
** Free-form errors and remote strings never cross this boundary.
*******************************************************************************/
public class QuickSearchHealthChecks implements HealthCheckSupplier
{
/*******************************************************************************
** Register once, even if the producer is run again on the same instance.
*******************************************************************************/
public static void register(QInstance instance)
{
HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance);
if(registry.getChecks().stream().noneMatch(check -> "quick_search.opensearch".equals(check.getName())))
{
registry.withCheck(definition("quick_search.opensearch", input -> run(input, QuickSearchHealthChecks::opensearch)));
}
if(registry.getChecks().stream().noneMatch(check -> "quick_search.backlog".equals(check.getName())))
{
registry.withCheck(definition("quick_search.backlog", input -> run(input, QuickSearchHealthChecks::backlog)));
}
if(registry.getSuppliers().stream().noneMatch(supplier -> "quick_search.index".equals(supplier.getNamePrefix())))
{
registry.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix("quick_search.index").withComponent("quick_search")
.withSupplier(new QCodeReference(QuickSearchHealthChecks.class)));
}
}
/*******************************************************************************
** Enumerate persisted index identities, including disabled indexes. Execution
** reads a fresh cached snapshot, never a row captured at supplier resolution.
*******************************************************************************/
@Override
public List<HealthCheckMetaData> supply(QInstance instance) throws QException
{
QuickSearchRuntime runtime = runtime(instance);
if(runtime == null)
{
return (List.of());
}
List<HealthCheckMetaData> checks = new ArrayList<>();
for(QRecord row : QuickSearchConfigPrimer.queryAll(runtime.getConfig().getQuickSearchIndexTableName()))
{
String tableName = row.getValueString("tableName");
String name = tableName.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]+", "_").replaceAll("[._-]{2,}", "_").replaceAll("^[._-]|[._-]$", "");
if(name.isEmpty())
{
throw (new QException("Index name cannot form a health check identifier"));
}
checks.add(definition("quick_search.index." + name, input -> run(input, status -> index(status, tableName))));
}
return (checks);
}
/*******************************************************************************
** All Quick Search checks are dependencies; none gate Kubernetes readiness.
*******************************************************************************/
private static HealthCheckMetaData definition(String name, HealthCheck check)
{
return (new HealthCheckMetaData().withName(name).withLabel(name).withComponent("quick_search")
.withCategory(HealthCategory.SEARCH).withKind(HealthCheckKind.DEPENDENCY)
.withIntervalSeconds(60).withTimeoutMs(5000).withCheckInstance(check));
}
/*******************************************************************************
** Resolve from the supplied instance, not a static or previously closed runtime.
*******************************************************************************/
private static QuickSearchRuntime runtime(QInstance instance)
{
if(instance != null && instance.getQBits() != null)
{
for(QBitMetaData qbit : instance.getQBits().values())
{
if(qbit.getConfig() instanceof QuickSearchQBitConfig config)
{
return (config.getRuntime());
}
}
}
return (null);
}
/*******************************************************************************
** Share the diagnostic worker and respect the deadline even on a cache hit.
** Expected diagnostic failures use fixed messages, never exception payloads.
*******************************************************************************/
private static HealthCheckOutcome run(HealthCheckInput input, Function<QuickSearchStatus, HealthCheckOutcome> project)
{
if(!Instant.now().isBefore(input.getDeadline()))
{
return (HealthCheckOutcome.down("Quick Search health deadline exceeded"));
}
QuickSearchRuntime runtime = runtime(input.getQInstance());
if(runtime == null)
{
return (HealthCheckOutcome.unknown("Quick Search is not configured"));
}
try
{
QuickSearchStatus status = runtime.getStatusCache().get(runtime, input.getDeadline());
if(!Instant.now().isBefore(input.getDeadline()))
{
return (HealthCheckOutcome.down("Quick Search health deadline exceeded"));
}
return (project.apply(status));
}
catch(QException e)
{
return (HealthCheckOutcome.down("Quick Search diagnostics unavailable"));
}
}
/*******************************************************************************
** Cluster health is independent of backlog and per-index degradation.
*******************************************************************************/
private static HealthCheckOutcome opensearch(QuickSearchStatus status)
{
if(status.state() == QuickSearchStatus.State.DISABLED)
{
return (HealthCheckOutcome.unknown("Quick Search is disabled"));
}
HealthCheckOutcome outcome;
if(status.state() == QuickSearchStatus.State.DOWN || Boolean.FALSE.equals(status.clusterReachable()) || "red".equals(status.clusterHealth()))
{
outcome = HealthCheckOutcome.down("OpenSearch is unavailable or red");
}
else if(Boolean.TRUE.equals(status.clusterReachable()) && "green".equals(status.clusterHealth()))
{
outcome = HealthCheckOutcome.up("OpenSearch is green");
}
else if(Boolean.TRUE.equals(status.clusterReachable()) && QuickSearchStatus.isSingleNodeReplicaYellow(status.clusterHealth(), status.nodes(), status.unassignedReplicaShards()))
{
outcome = HealthCheckOutcome.up("OpenSearch is yellow only because replicas cannot be assigned on a single node");
outcome.getDetails().withValue("note", "replicas cannot be assigned on a single-node cluster");
}
else
{
outcome = HealthCheckOutcome.degraded("OpenSearch health is yellow or incomplete");
}
detail(outcome, "reachable", status.clusterReachable());
detail(outcome, "nodes", status.nodes());
detail(outcome, "activeShards", status.activeShards());
detail(outcome, "unassignedShards", status.unassignedShards());
detail(outcome, "unassignedReplicaShards", status.unassignedReplicaShards());
detail(outcome, "targetReplicas", status.targetReplicas());
detail(outcome, "replicaSettingsFailed", status.replicaSettingsFailed());
return (outcome);
}
/*******************************************************************************
** Counts are lower bounds when truncated; do not declare an incomplete backlog healthy.
*******************************************************************************/
private static HealthCheckOutcome backlog(QuickSearchStatus status)
{
HealthCheckOutcome outcome;
if(status.state() == QuickSearchStatus.State.DISABLED)
{
outcome = HealthCheckOutcome.unknown("Quick Search is disabled");
}
else if(status.pending() == null || status.exhausted() == null || status.indexingPaused() == null)
{
outcome = HealthCheckOutcome.degraded("Indexing backlog measurements unavailable");
}
else if(status.indexingPaused() || status.exhausted() > 0 || status.pending() > 100 || Boolean.TRUE.equals(status.backlogTruncated()))
{
outcome = HealthCheckOutcome.degraded("Indexing is paused or backlog needs attention");
}
else
{
outcome = HealthCheckOutcome.up("Indexing backlog is within limits");
}
detail(outcome, "pending", status.pending());
detail(outcome, "exhausted", status.exhausted());
detail(outcome, "indexingPaused", status.indexingPaused());
detail(outcome, "backlogTruncated", status.backlogTruncated());
return (outcome);
}
/*******************************************************************************
** Missing rows (including those outside the bounded snapshot) stay unknown.
** Persisted error text is represented only by a boolean.
*******************************************************************************/
private static HealthCheckOutcome index(QuickSearchStatus status, String tableName)
{
IndexRow row = status.indexes().stream().filter(candidate -> tableName.equals(candidate.tableName())).findFirst().orElse(null);
if(row == null)
{
return (HealthCheckOutcome.unknown("Index measurements unavailable"));
}
boolean hasError = row.lastError() != null && !row.lastError().isBlank();
boolean mismatch = row.documentCount() != null && row.sourceCount() != null && !row.documentCount().equals(row.sourceCount());
HealthCheckOutcome outcome;
if(Boolean.FALSE.equals(row.enabled()))
{
outcome = HealthCheckOutcome.unknown("Index is disabled");
}
else if(!"ACTIVE".equals(row.status()) || hasError || mismatch || (row.realTimeErrorCount() != null && row.realTimeErrorCount() > 0))
{
outcome = HealthCheckOutcome.degraded("Index needs attention");
}
else
{
outcome = HealthCheckOutcome.up("Index is active");
}
outcome.withDetails(new HealthDetails().withValue("hasError", hasError).withValue("countMismatch", mismatch));
detail(outcome, "enabled", row.enabled());
detail(outcome, "documentCount", row.documentCount());
detail(outcome, "sourceCount", row.sourceCount());
detail(outcome, "countsCheckedAt", row.countsCheckedAt());
detail(outcome, "lastSuccessfulSync", row.lastSuccessfulSync());
detail(outcome, "realTimeErrorCount", row.realTimeErrorCount());
return (outcome);
}
/*******************************************************************************
** Unknown measurements are omitted rather than converted to zero.
*******************************************************************************/
private static void detail(HealthCheckOutcome outcome, String name, Object value)
{
if(value != null)
{
outcome.getDetails().withValue(name, value);
}
}
}
@@ -19,7 +19,9 @@ package com.kingsrook.qbits.quicksearch.listeners;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
@@ -39,6 +41,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
@@ -66,13 +70,13 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** This listener never fails the host's write. When the re-read of inserted or
** updated records or publishing fails (including a LinkageError from a missing
** optional library while the client is built), each affected (table, record,
** action) is written to quickSearchFailedEvent for the scheduled basepull to
** action) is written to quick_search_failed_event for the scheduled basepull to
** replay. A failed re-read is recorded after the commit too, so a rolled-back
** write leaves no row.
**
** While a full reindex rebuilds into a new physical index, the alias still
** points at the old one, so a published delete never reaches the new index.
** Such deletes are also written to quickSearchFailedEvent with status
** Such deletes are also written to quick_search_failed_event with status
** AWAITING_REINDEX; the reindex applies them after the alias swap.
*******************************************************************************/
public class QuickSearchRecordChangeListener implements RecordChangeListenerInterface
@@ -90,6 +94,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
QuickSearchRuntime runtime = QuickSearchRuntime.get();
return (runtime != null
&& Boolean.TRUE.equals(runtime.getConfig().getEnableRealTimeIndexing())
&& !QuickSearchFieldRules.isQuickSearchTable(runtime.getConfig(), tableName)
&& runtime.getTableConfig(tableName) != null);
}
@@ -115,6 +120,24 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
}
String primaryKeyField = tableConfig.getPrimaryKeyField() != null ? tableConfig.getPrimaryKeyField() : "id";
if(QuickSearchFieldRules.lookupTable(tableName) == null || QuickSearchFieldRules.isProtected(QuickSearchFieldRules.lookupTable(tableName), primaryKeyField))
{
QuickSearchFieldRules.warnExcludedFields(QuickSearchFieldRules.lookupTable(tableName));
return;
}
///////////////////////////////////////////////////////////////////////////
// paused: index nothing, but keep the written keys (DEFERRED), which //
// resuming hands to the basepull; search keeps serving the old documents //
///////////////////////////////////////////////////////////////////////////
if(runtime.isIndexingPaused())
{
List<IndexEvent> deferred = event.getType() == RecordChangeType.DELETE
? toDeleteEvents(tableName, primaryKeyField, event.getRecords())
: toIndexEvents(tableName, primaryKeyField, event.getRecords());
runAfterCommit(event, () -> recordDeferred(runtime, deferred));
return;
}
List<IndexEvent> indexEvents = new ArrayList<>();
List<IndexEvent> deleteEvents = new ArrayList<>();
@@ -246,7 +269,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
/*******************************************************************************
** Write one quickSearchFailedEvent row per event. Never throws.
** Write one quick_search_failed_event row per event. Never throws.
*******************************************************************************/
static void recordFailures(QuickSearchRuntime runtime, List<IndexEvent> events, Throwable cause)
{
@@ -261,13 +284,14 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
try
{
if(!hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)))
if(!hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)))
{
LOG.error("quickSearchFailedEvent table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
LOG.error("quick_search_failed_event table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
return;
}
insertFailedEventRows(runtime, events, message, QuickSearchFailedEvent.STATUS_PENDING);
incrementRealTimeErrorCounts(runtime, events);
}
catch(Exception e)
{
@@ -277,6 +301,81 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
/*******************************************************************************
** QQQ UpdateInput has no arithmetic update. Serialize this read-modify-write
** across listeners in this JVM, after the source transaction has committed.
** Counter failures must not prevent replay or fail the source write.
*******************************************************************************/
private static synchronized void incrementRealTimeErrorCounts(QuickSearchRuntime runtime, List<IndexEvent> events)
{
Map<String, Integer> counts = new HashMap<>();
for(IndexEvent event : events)
{
counts.merge(event.getTableName(), 1, Integer::sum);
}
for(Map.Entry<String, Integer> entry : counts.entrySet())
{
try
{
String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(indexTable);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, entry.getKey())).withLimit(1));
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
if(CollectionUtils.nullSafeIsEmpty(rows))
{
continue;
}
QRecord row = rows.get(0);
Integer previous = row.getValueInteger("realTimeErrorCount");
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(indexTable);
updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id"))
.withValue("realTimeErrorCount", Math.addExact(previous == null ? 0 : previous, entry.getValue()))));
// A runtime counter, not a configuration change: not record-audited.
updateInput.setOmitDmlAudit(true);
List<QRecord> updated = QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)).getRecords();
if(CollectionUtils.nullSafeIsEmpty(updated) || CollectionUtils.nullSafeHasContents(updated.get(0).getErrors()))
{
throw (new QException("Counter update returned errors"));
}
}
catch(Exception | LinkageError e)
{
LOG.warn("Could not increment real-time indexing error count", e, logPair("tableName", entry.getKey()));
}
}
}
/*******************************************************************************
** Keep changes made while indexing is paused, for the basepull to replay
** when it resumes. Never throws.
*******************************************************************************/
static void recordDeferred(QuickSearchRuntime runtime, List<IndexEvent> events)
{
if(events.isEmpty())
{
return;
}
try
{
if(hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)))
{
insertFailedEventRows(runtime, events, "deferred: indexing was paused", QuickSearchFailedEvent.STATUS_DEFERRED);
}
}
catch(Exception e)
{
LOG.warn("Could not keep changes made while indexing is paused; the next reconcile covers them", e, logPair("eventCount", events.size()));
}
}
/*******************************************************************************
** Whether a full reindex is rebuilding the deletes' table. The status is
** read from the database (not cached) so a reindex on another node is seen.
@@ -292,7 +391,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
String tableName = deleteEvents.get(0).getTableName();
try
{
return (hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
return (hasTable(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
}
catch(Exception e)
{
@@ -347,7 +446,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
}
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
updateInput.setTableName(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
updateInput.setRecords(updates);
new UpdateAction().execute(updateInput);
}
@@ -355,7 +454,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
/*******************************************************************************
** Whether the table's quickSearchIndex row is REBUILDING.
** Whether the table's quick_search_index row is REBUILDING.
*******************************************************************************/
private static boolean isRebuilding(QuickSearchRuntime runtime, String tableName) throws QException
{
@@ -375,7 +474,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
/*******************************************************************************
** Insert one quickSearchFailedEvent row per event; returns the inserted rows.
** Insert one quick_search_failed_event row per event; returns the inserted rows.
*******************************************************************************/
static List<QRecord> insertFailedEventRows(QuickSearchRuntime runtime, List<IndexEvent> events, String message, String status) throws QException
{
@@ -392,7 +491,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
}
InsertInput insertInput = new InsertInput();
insertInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
insertInput.setTableName(runtime.getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
insertInput.setRecords(rows);
return (CollectionUtils.nonNullList(new InsertAction().execute(insertInput).getRecords()));
}
@@ -422,7 +521,17 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
{
continue;
}
events.add(new IndexEvent().withAction(IndexEventAction.INDEX).withTableName(tableName).withRecordId(recordId).withRecord(record));
QRecord safeRecord = new QRecord();
for(var field : QuickSearchFieldRules.indexableFields(QuickSearchFieldRules.lookupTable(tableName)))
{
safeRecord.setValue(field.getName(), record.getValue(field.getName()));
safeRecord.setDisplayValue(field.getName(), record.getDisplayValue(field.getName()));
}
if(!QuickSearchFieldRules.requiresFieldFiltering(tableName))
{
safeRecord.setRecordLabel(record.getRecordLabel());
}
events.add(new IndexEvent().withAction(IndexEventAction.INDEX).withTableName(tableName).withRecordId(recordId).withRecord(safeRecord));
}
return (events);
}
@@ -25,21 +25,24 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
/*******************************************************************************
** Produces the quickSearchAdmin app from the tables and processes the other
** component producers added to the production output. Runs last.
** Produces the quickSearchAdmin app from the tables the other component
** producers added to the production output. Processes are not app children
** (never in the nav): they run from their tables' Actions. Runs last.
*******************************************************************************/
public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataProducer<QAppMetaData, QuickSearchQBitConfig>
{
@@ -73,25 +76,20 @@ public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataPr
app.withWidgets(new ArrayList<>(List.of(diagnosticsWidget.getName())));
}
for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME, QuickSearchFailedEvent.TABLE_NAME,
QuickSearchValidationRun.TABLE_NAME, QuickSearchDiagnosticEvent.TABLE_NAME })
for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexField.TABLE_NAME, QuickSearchSettings.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME,
QuickSearchFailedEvent.TABLE_NAME, QuickSearchValidationRun.TABLE_NAME, QuickSearchDiagnosticEvent.TABLE_NAME, QuickSearchConfigTombstone.TABLE_NAME })
{
QTableMetaData table = produced.get(QTableMetaData.class, config.applyPrefix(tableName));
QTableMetaData table = produced.get(QTableMetaData.class, config.applyTablePrefix(tableName));
if(table != null)
{
app.withChild(table);
}
}
for(String processName : new String[] { QuickSearchProcessMetaDataHelper.BASEPULL_PROCESS_NAME, QuickSearchProcessMetaDataHelper.FULL_REINDEX_PROCESS_NAME, QuickSearchProcessMetaDataHelper.RECONCILE_PROCESS_NAME })
{
QProcessMetaData process = produced.get(QProcessMetaData.class, config.applyPrefix(processName));
if(process != null)
{
app.withChild(process);
}
}
////////////////////////////////////////////////////////////////////////
// processes are never in the nav: Basepull, Full Reindex, Reconcile //
// and the config processes run from their tables' Actions //
////////////////////////////////////////////////////////////////////////
return (app);
}
@@ -0,0 +1,384 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchIndexFieldTableCustomizer;
import com.kingsrook.qbits.quicksearch.config.QuickSearchSettingsTableCustomizer;
import com.kingsrook.qbits.quicksearch.config.QuickSearchTablePossibleValueProviders;
import com.kingsrook.qbits.quicksearch.model.QuickSearchConfigTombstone;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDocumentsChoice;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
/*******************************************************************************
** Builds the data-driven config metadata (1.1.0-RC.3): the field, settings
** and tombstone tables, the index-to-fields join and child-record widget,
** and the possible-value sources the config screens use.
*******************************************************************************/
public class QuickSearchConfigMetaDataHelper
{
public static final String ELIGIBLE_TABLE_PVS = "quick_search_eligible_table";
public static final String INDEXABLE_FIELD_PVS = "quick_search_indexable_field";
public static final String TIMESTAMP_FIELD_PVS = "quick_search_timestamp_field";
public static final String FIELD_ROLE_PVS = "quick_search_field_role";
public static final String FIELD_ANALYZER_PVS = "quick_search_field_analyzer";
public static final String DOCUMENTS_CHOICE_PVS = "quick_search_documents_choice";
public static final String CONFIG_SOURCE_PVS = "quick_search_config_source";
public static final String INDEX_FIELDS_JOIN = "quick_search_index_join_quick_search_index_field";
public static final String INDEX_FIELDS_WIDGET = "quickSearchIndexFields";
public static final String FIELDS_ASSOCIATION = "fields";
public static final String INDEX_RUNS_JOIN = "quick_search_index_join_quick_search_index_run";
public static final String INDEX_RUNS_WIDGET = "quickSearchIndexRuns";
public static final String RUN_TYPE_PVS = "quick_search_run_type";
public static final String RUN_STATUS_PVS = "quick_search_run_status";
public static final String EVENT_STATUS_PVS = "quick_search_failed_event_status";
public static final String EVENT_ACTION_PVS = "quick_search_failed_event_action";
/*******************************************************************************
** quick_search_index_field: the searchable fields of each index row.
*******************************************************************************/
public static QTableMetaData buildIndexFieldTable(QuickSearchQBitConfig config)
{
return (new QTableMetaData()
.withName(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
.withLabel("Quick Search Index Field")
.withIcon(new QIcon("text_fields"))
.withBackendName(config.getBackendName())
.withPrimaryKeyField("id")
.withRecordLabelFormat("%s")
.withRecordLabelFields("fieldName")
.withUniqueKey(new UniqueKey("quickSearchIndexId", "fieldName"))
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(QuickSearchIndexFieldTableCustomizer.class))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withIsRequired(true)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)))
.withField(new QFieldMetaData("fieldName", QFieldType.STRING).withLabel("Field").withIsRequired(true)
.withPossibleValueSourceName(config.applyTablePrefix(INDEXABLE_FIELD_PVS)))
.withField(new QFieldMetaData("weight", QFieldType.INTEGER).withDefaultValue(1))
.withField(new QFieldMetaData("role", QFieldType.STRING).withDefaultValue(QuickSearchFieldRole.BODY.name())
.withPossibleValueSourceName(config.applyTablePrefix(FIELD_ROLE_PVS)))
.withField(new QFieldMetaData("analyzer", QFieldType.STRING).withLabel("Analyzer (blank: by role)")
.withPossibleValueSourceName(config.applyTablePrefix(FIELD_ANALYZER_PVS)))
.withField(new QFieldMetaData("includeInSnippet", QFieldType.BOOLEAN).withLabel("In Snippet (blank: by role)"))
.withField(new QFieldMetaData("includeLabel", QFieldType.BOOLEAN).withLabel("Index With Field Label").withDefaultValue(false))
.withField(new QFieldMetaData("sortOrder", QFieldType.INTEGER))
.withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false).withPossibleValueSourceName(config.applyTablePrefix(CONFIG_SOURCE_PVS)))
.withField(new QFieldMetaData("ownerEdited", QFieldType.BOOLEAN).withLabel("Edited by Owner").withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withSection(new QFieldSection("identity", "Field", new QIcon("text_fields"), Tier.T1, List.of("id", "quickSearchIndexId", "fieldName", "weight", "role")))
.withSection(new QFieldSection("matching", "Matching", new QIcon("tune"), Tier.T2, List.of("analyzer", "includeInSnippet", "includeLabel", "sortOrder")))
.withSection(new QFieldSection("origin", "Origin", new QIcon("history"), Tier.T3, List.of("source", "ownerEdited", "createDate", "modifyDate"))));
}
/*******************************************************************************
** quick_search_settings: the one row of system settings (the global pause
** and the last priming report). Rows are never inserted or deleted by
** users; the QBit creates the row when it primes.
*******************************************************************************/
public static QTableMetaData buildSettingsTable(QuickSearchQBitConfig config)
{
return (new QTableMetaData()
.withName(config.applyTablePrefix(QuickSearchSettings.TABLE_NAME))
.withLabel("Quick Search Settings")
.withIcon(new QIcon("settings"))
.withBackendName(config.getBackendName())
.withPrimaryKeyField("id")
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
.withRecordLabelFormat("Quick Search Settings")
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchSettingsTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchSettingsTableCustomizer.class))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("indexingPaused", QFieldType.BOOLEAN).withLabel("Indexing Paused"))
.withField(new QFieldMetaData("pauseReason", QFieldType.STRING).withLabel("Pause Reason"))
.withField(new QFieldMetaData("pausedAt", QFieldType.DATE_TIME).withLabel("Paused Since").withIsEditable(false))
.withField(new QFieldMetaData("pausedBy", QFieldType.STRING).withLabel("Paused By").withIsEditable(false))
.withField(new QFieldMetaData("lastPrimedAt", QFieldType.DATE_TIME).withLabel("Last Primed").withIsEditable(false))
.withField(new QFieldMetaData("lastPrimingSummary", QFieldType.TEXT).withLabel("Last Priming").withIsEditable(false))
.withField(new QFieldMetaData("lastPrimingNewTables", QFieldType.INTEGER).withLabel("Tables Added by Last Priming").withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withSection(new QFieldSection("pause", "Indexing", new QIcon("pause_circle"), Tier.T1, List.of("id", "indexingPaused", "pauseReason", "pausedAt", "pausedBy")))
.withSection(new QFieldSection("priming", "Self-priming", new QIcon("auto_mode"), Tier.T2, List.of("lastPrimedAt", "lastPrimingSummary", "lastPrimingNewTables")))
.withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
}
/*******************************************************************************
** quick_search_config_tombstone: what an admin removed, so priming does
** not add it back. Written by the QBit; deleting a row lets code defaults
** seed that table or field again.
*******************************************************************************/
public static QTableMetaData buildTombstoneTable(QuickSearchQBitConfig config)
{
return (new QTableMetaData()
.withName(config.applyTablePrefix(QuickSearchConfigTombstone.TABLE_NAME))
.withLabel("Quick Search Removed Config")
.withIcon(new QIcon("delete_sweep"))
.withBackendName(config.getBackendName())
.withPrimaryKeyField("id")
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
.withRecordLabelFormat("%s %s")
.withRecordLabelFields("tableName", "fieldName")
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
.withField(new QFieldMetaData("fieldName", QFieldType.STRING).withLabel("Field (blank: whole table)").withIsEditable(false))
.withField(new QFieldMetaData("removedBy", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("reason", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Removed").withIsEditable(false)));
}
/*******************************************************************************
** quick_search_index to its field rows.
*******************************************************************************/
public static QJoinMetaData buildIndexFieldsJoin(QuickSearchQBitConfig config)
{
return (new QJoinMetaData()
.withName(config.applyTablePrefix(INDEX_FIELDS_JOIN))
.withType(JoinType.ONE_TO_MANY)
.withLeftTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
.withRightTable(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
.withJoinOn(new JoinOn("id", "quickSearchIndexId"))
.withOrderBy(new QFilterOrderBy("sortOrder"))
.withOrderBy(new QFilterOrderBy("id")));
}
/*******************************************************************************
** The field rows on the index record view, where fields are added,
** edited and removed.
*******************************************************************************/
public static QWidgetMetaData buildIndexFieldsWidget(QuickSearchQBitConfig config)
{
QWidgetMetaData widget = ChildRecordListRenderer.widgetMetaDataBuilder(buildIndexFieldsJoin(config))
.withName(config.applyPrefix(INDEX_FIELDS_WIDGET))
.withLabel("Searchable Fields")
.withCanAddChildRecord(true)
.withDisabledFieldsForNewChildRecords(Set.of("quickSearchIndexId"))
.withDefaultValuesForNewChildRecordsFromParentFields(Map.of("quickSearchIndexId", "id"))
.withOnlyIncludeFieldNames(List.of("fieldName", "weight", "role", "analyzer", "includeInSnippet", "includeLabel", "source"))
.withMaxRows(100)
.getWidgetMetaData();
return (widget);
}
/*******************************************************************************
** quick_search_index to its run history, newest first.
*******************************************************************************/
public static QJoinMetaData buildIndexRunsJoin(QuickSearchQBitConfig config)
{
return (new QJoinMetaData()
.withName(config.applyTablePrefix(INDEX_RUNS_JOIN))
.withType(JoinType.ONE_TO_MANY)
.withLeftTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
.withRightTable(config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME))
.withJoinOn(new JoinOn("id", "quickSearchIndexId"))
.withOrderBy(new QFilterOrderBy("id", false)));
}
/*******************************************************************************
** The index record's recent runs (read only).
*******************************************************************************/
public static QWidgetMetaData buildIndexRunsWidget(QuickSearchQBitConfig config)
{
return (ChildRecordListRenderer.widgetMetaDataBuilder(buildIndexRunsJoin(config))
.withName(config.applyPrefix(INDEX_RUNS_WIDGET))
.withLabel("Recent Runs")
.withCanAddChildRecord(false)
.withOnlyIncludeFieldNames(List.of("id", "runType", "status", "startTime", "endTime", "recordsProcessed", "recordsIndexed", "errorCount"))
.withMaxRows(10)
.getWidgetMetaData());
}
/*******************************************************************************
** The custom possible-value sources over the instance's tables and fields.
*******************************************************************************/
public static List<QPossibleValueSource> buildCustomPossibleValueSources(QuickSearchQBitConfig config)
{
return (List.of(
custom(config.applyTablePrefix(ELIGIBLE_TABLE_PVS), QuickSearchTablePossibleValueProviders.EligibleTables.class),
custom(config.applyTablePrefix(INDEXABLE_FIELD_PVS), QuickSearchTablePossibleValueProviders.IndexableFields.class),
custom(config.applyTablePrefix(TIMESTAMP_FIELD_PVS), QuickSearchTablePossibleValueProviders.TimestampFields.class)));
}
/*******************************************************************************
** The enum possible-value sources (role, analyzer, documents choice,
** config source).
*******************************************************************************/
public static List<QPossibleValueSource> buildEnumPossibleValueSources(QuickSearchQBitConfig config)
{
return (List.of(
QPossibleValueSource.newForEnum(config.applyTablePrefix(FIELD_ROLE_PVS), QuickSearchFieldRole.values()),
QPossibleValueSource.newForEnum(config.applyTablePrefix(FIELD_ANALYZER_PVS), QuickSearchFieldAnalyzer.values()),
QPossibleValueSource.newForEnum(config.applyTablePrefix(DOCUMENTS_CHOICE_PVS), QuickSearchDocumentsChoice.values()),
stringEnum(config.applyTablePrefix(RUN_TYPE_PVS), "BASEPULL", "Update (basepull)", "RECONCILE", "Reconcile", "FULL_REINDEX", "Full reindex / build"),
stringEnum(config.applyTablePrefix(RUN_STATUS_PVS), "RUNNING", "Running", "COMPLETED", "Completed", "FAILED", "Failed", "CANCELLED", "Cancelled"),
stringEnum(config.applyTablePrefix(EVENT_STATUS_PVS), "PENDING", "Pending (retried by the next update)", "EXHAUSTED", "Exhausted (gave up)",
"AWAITING_REINDEX", "Awaiting full reindex", "DEFERRED", "Deferred (indexing paused)"),
stringEnum(config.applyTablePrefix(EVENT_ACTION_PVS), "INDEX", "Index", "DELETE", "Delete"),
new QPossibleValueSource()
.withName(config.applyTablePrefix(CONFIG_SOURCE_PVS))
.withType(QPossibleValueSourceType.ENUM)
.withEnumValues(List.of(
new QPossibleValue<>(QuickSearchIndex.SOURCE_CODE, "Code default"),
new QPossibleValue<>(QuickSearchIndex.SOURCE_OWNER, "Added by owner")))));
}
/*******************************************************************************
** Add what quick_search_index refers to (its field, settings and tombstone
** tables, join, field-list widget and possible-value sources) to an
** instance whose tables are built with the helpers instead of produced by
** the QBit (tests, hand-assembled instances). Anything already there is
** kept.
*******************************************************************************/
public static void addCompanionMetaData(QInstance qInstance, QuickSearchQBitConfig config)
{
for(QTableMetaData table : List.of(buildIndexFieldTable(config), buildSettingsTable(config), buildTombstoneTable(config)))
{
if(qInstance.getTable(table.getName()) == null)
{
qInstance.addTable(table);
}
}
QJoinMetaData join = buildIndexFieldsJoin(config);
if(qInstance.getJoin(join.getName()) == null)
{
qInstance.addJoin(join);
}
for(QJoinMetaData extraJoin : List.of(buildIndexRunsJoin(config)))
{
if(qInstance.getJoin(extraJoin.getName()) == null)
{
qInstance.addJoin(extraJoin);
}
}
for(QWidgetMetaData widget : List.of(buildIndexFieldsWidget(config), buildIndexRunsWidget(config)))
{
if(qInstance.getWidget(widget.getName()) == null)
{
qInstance.addWidget(widget);
}
}
List<QPossibleValueSource> sources = new ArrayList<>(buildCustomPossibleValueSources(config));
sources.addAll(buildEnumPossibleValueSources(config));
sources.add(new QPossibleValueSource()
.withName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
.withType(QPossibleValueSourceType.CUSTOM)
.withIdType(QFieldType.STRING)
.withCustomCodeReference(new QCodeReference(QuickSearchTablePossibleValueProviders.SearchableTables.class)));
for(QPossibleValueSource source : sources)
{
if(qInstance.getPossibleValueSource(source.getName()) == null)
{
qInstance.addPossibleValueSource(source);
}
}
}
/*******************************************************************************
** An enum source of string ids and labels (id, label, id, label...).
*******************************************************************************/
private static QPossibleValueSource stringEnum(String name, String... idsAndLabels)
{
List<QPossibleValue<?>> values = new ArrayList<>();
for(int i = 0; i + 1 < idsAndLabels.length; i += 2)
{
values.add(new QPossibleValue<>(idsAndLabels[i], idsAndLabels[i + 1]));
}
return (new QPossibleValueSource().withName(name).withType(QPossibleValueSourceType.ENUM).withEnumValues(values));
}
/*******************************************************************************
**
*******************************************************************************/
private static QPossibleValueSource custom(String name, Class<?> provider)
{
return (new QPossibleValueSource()
.withName(name)
.withType(QPossibleValueSourceType.CUSTOM)
.withIdType(QFieldType.STRING)
.withCustomCodeReference(new QCodeReference(provider)));
}
}
@@ -0,0 +1,69 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the data-driven config's join, field-list widget and
** possible-value sources (see QuickSearchConfigMetaDataHelper).
*******************************************************************************/
public class QuickSearchConfigMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
{
QuickSearchQBitConfig config = getQBitConfig();
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
output.add(QuickSearchConfigMetaDataHelper.buildIndexFieldsJoin(config));
output.add(QuickSearchConfigMetaDataHelper.buildIndexFieldsWidget(config));
output.add(QuickSearchConfigMetaDataHelper.buildIndexRunsJoin(config));
output.add(QuickSearchConfigMetaDataHelper.buildIndexRunsWidget(config));
for(QPossibleValueSource possibleValueSource : QuickSearchConfigMetaDataHelper.buildCustomPossibleValueSources(config))
{
output.add(possibleValueSource);
}
for(QPossibleValueSource possibleValueSource : QuickSearchConfigMetaDataHelper.buildEnumPossibleValueSources(config))
{
output.add(possibleValueSource);
}
return (output);
}
/*******************************************************************************
** after tables, before the app
*******************************************************************************/
@Override
public int getSortOrder()
{
return (150);
}
}
@@ -0,0 +1,274 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDocumentsChoice;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
import com.kingsrook.qbits.quicksearch.processes.config.AddSearchIndexSteps;
import com.kingsrook.qbits.quicksearch.processes.config.ApplyIndexChangesSteps;
import com.kingsrook.qbits.quicksearch.processes.config.CancelRunSteps;
import com.kingsrook.qbits.quicksearch.processes.config.SetIndexingPausedSteps;
import com.kingsrook.qbits.quicksearch.processes.config.StopIndexingSteps;
/*******************************************************************************
** The config processes. Each is a standard streamed-ETL process with a
** review step (the transform's summary: what will happen, counts, problems)
** and a result step, run from its table's Actions (never from the nav):
** - Add Search Index, on quick_search_index (no selection needed);
** - Disable / Remove Search Index and Apply Search Index Changes, on the
** selected quick_search_index rows;
** - Cancel Index Run, on the selected quick_search_index_run rows;
** - Pause / Resume Indexing, on the quick_search_settings row.
** All need the admin permission (quick_search_admin by default).
*******************************************************************************/
public class QuickSearchConfigProcessMetaDataHelper
{
public static final String ADD_INDEX_PROCESS_NAME = "quickSearchAddIndex";
public static final String DISABLE_INDEX_PROCESS_NAME = "quickSearchDisableIndex";
public static final String REMOVE_INDEX_PROCESS_NAME = "quickSearchRemoveIndex";
public static final String APPLY_CHANGES_PROCESS_NAME = "quickSearchApplyIndexChanges";
public static final String CANCEL_RUN_PROCESS_NAME = "quickSearchCancelRun";
public static final String PAUSE_PROCESS_NAME = "quickSearchPauseIndexing";
public static final String RESUME_PROCESS_NAME = "quickSearchResumeIndexing";
/*******************************************************************************
** Every config process.
*******************************************************************************/
public static List<QProcessMetaData> buildProcesses(QuickSearchQBitConfig config)
{
return (List.of(
buildAddIndexProcess(config),
buildStopIndexingProcess(config, DISABLE_INDEX_PROCESS_NAME, "Disable Search Index", "toggle_off", StopIndexingSteps.MODE_DISABLE, QuickSearchDocumentsChoice.KEEP),
buildStopIndexingProcess(config, REMOVE_INDEX_PROCESS_NAME, "Remove Search Index", "delete", StopIndexingSteps.MODE_REMOVE, QuickSearchDocumentsChoice.DELETE),
buildApplyChangesProcess(config),
buildCancelRunProcess(config),
buildSetPausedProcess(config, PAUSE_PROCESS_NAME, "Pause Indexing", "pause_circle", true),
buildSetPausedProcess(config, RESUME_PROCESS_NAME, "Resume Indexing", "play_circle", false)));
}
/*******************************************************************************
** Add Search Index: the valid way to add a table.
*******************************************************************************/
public static QProcessMetaData buildAddIndexProcess(QuickSearchQBitConfig config)
{
String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
QProcessMetaData process = base(config, ADD_INDEX_PROCESS_NAME, "Add Search Index", "add_circle", indexTable,
AddSearchIndexSteps.Extract.class, AddSearchIndexSteps.Transform.class, AddSearchIndexSteps.Load.class, 0, List.of(
new QFieldMetaData(AddSearchIndexSteps.RECORD_TABLE, QFieldType.STRING).withLabel("Table")
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)),
new QFieldMetaData("estimatedRecords", QFieldType.INTEGER).withLabel("Records to Index"),
new QFieldMetaData("fieldCount", QFieldType.INTEGER).withLabel("Fields"),
new QFieldMetaData(AddSearchIndexSteps.FIELD_INTERVAL, QFieldType.INTEGER).withLabel("Update Interval (Minutes)"),
new QFieldMetaData(AddSearchIndexSteps.FIELD_TIMESTAMP, QFieldType.STRING).withLabel("Timestamp Field"),
new QFieldMetaData(AddSearchIndexSteps.FIELD_ENABLED, QFieldType.BOOLEAN)));
process.withStep(0, new QFrontendStepMetaData()
.withName("input")
.withLabel("Choose a table")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text",
"Only tables you can read, that are not indexed yet and are not internal or secret are offered. The review step shows the fields the index starts with "
+ "and how many records its first build indexes; change the fields afterwards under Searchable Fields on the index."))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_TABLE_NAME, QFieldType.STRING).withLabel("Table").withIsRequired(true)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.ELIGIBLE_TABLE_PVS)))
.withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_INTERVAL, QFieldType.INTEGER).withLabel("Update Interval (Minutes)")
.withDefaultValue(config.getDefaultBasepullIntervalMinutes()))
.withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_TIMESTAMP, QFieldType.STRING).withLabel("Timestamp Field (blank: detect)")
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.TIMESTAMP_FIELD_PVS)))
.withFormField(new QFieldMetaData(AddSearchIndexSteps.FIELD_ENABLED, QFieldType.BOOLEAN).withLabel("Enabled").withDefaultValue(true)));
return (process);
}
/*******************************************************************************
** Disable or Remove Search Index, with the documents choice.
*******************************************************************************/
static QProcessMetaData buildStopIndexingProcess(QuickSearchQBitConfig config, String name, String label, String icon, String mode, QuickSearchDocumentsChoice defaultChoice)
{
String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
.withName(config.applyPrefix(name))
.withLabel(label)
.withIcon(new QIcon(icon))
.withTableName(indexTable)
.withSourceTable(indexTable)
.withDestinationTable(indexTable)
.withExtractStepClass(ExtractViaQueryStep.class)
.withTransformStepClass(StopIndexingSteps.Transform.class)
.withLoadStepClass(StopIndexingSteps.Load.class)
.withSupportsFullValidation(true)
.withDoFullValidation(true)
.withTransactionLevelAutoCommit()
.withMinInputRecords(1)
.withFields(List.of(new QFieldMetaData(StopIndexingSteps.FIELD_MODE, QFieldType.STRING).withDefaultValue(mode)))
.withReviewStepRecordFields(indexReviewFields(config))
.getProcessMetaData();
process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
process.withStep(0, new QFrontendStepMetaData()
.withName("input")
.withLabel("Search documents")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text",
StopIndexingSteps.MODE_REMOVE.equals(mode)
? "Removing stops indexing the table and deletes its index and field rows. Code defaults never add it back (see Quick Search Removed Config)."
: "Disabling stops indexing the table and hides it from search; enabling it again rebuilds it."))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormField(new QFieldMetaData(StopIndexingSteps.FIELD_DOCUMENTS, QFieldType.STRING).withLabel("Search Documents").withIsRequired(true)
.withDefaultValue(defaultChoice.name())
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.DOCUMENTS_CHOICE_PVS))));
return (process);
}
/*******************************************************************************
** Apply Search Index Changes: preview field changes, then rebuild.
*******************************************************************************/
static QProcessMetaData buildApplyChangesProcess(QuickSearchQBitConfig config)
{
String indexTable = config.applyTablePrefix(QuickSearchIndex.TABLE_NAME);
return (base(config, APPLY_CHANGES_PROCESS_NAME, "Apply Search Index Changes", "published_with_changes", indexTable,
ExtractViaQueryStep.class, ApplyIndexChangesSteps.Transform.class, ApplyIndexChangesSteps.Load.class, 1, indexReviewFields(config)));
}
/*******************************************************************************
** Cancel Index Run: the cooperative cancel flag.
*******************************************************************************/
static QProcessMetaData buildCancelRunProcess(QuickSearchQBitConfig config)
{
String runTable = config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME);
return (base(config, CANCEL_RUN_PROCESS_NAME, "Cancel Index Run", "cancel", runTable,
ExtractViaQueryStep.class, CancelRunSteps.Transform.class, CancelRunSteps.Load.class, 1, List.of(
new QFieldMetaData("id", QFieldType.INTEGER),
new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)),
new QFieldMetaData("runType", QFieldType.STRING),
new QFieldMetaData("status", QFieldType.STRING),
new QFieldMetaData("recordsProcessed", QFieldType.INTEGER))));
}
/*******************************************************************************
** Pause or Resume Indexing.
*******************************************************************************/
static QProcessMetaData buildSetPausedProcess(QuickSearchQBitConfig config, String name, String label, String icon, boolean pause)
{
String settingsTable = config.applyTablePrefix(QuickSearchSettings.TABLE_NAME);
QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
.withName(config.applyPrefix(name))
.withLabel(label)
.withIcon(new QIcon(icon))
.withTableName(settingsTable)
.withSourceTable(settingsTable)
.withDestinationTable(settingsTable)
.withExtractStepClass(ExtractViaQueryStep.class)
.withTransformStepClass(SetIndexingPausedSteps.Transform.class)
.withLoadStepClass(SetIndexingPausedSteps.Load.class)
.withSupportsFullValidation(true)
.withDoFullValidation(true)
.withTransactionLevelAutoCommit()
.withMinInputRecords(1)
.withFields(List.of(new QFieldMetaData(SetIndexingPausedSteps.FIELD_PAUSE, QFieldType.BOOLEAN).withDefaultValue(pause)))
.withReviewStepRecordFields(List.of(
new QFieldMetaData("indexingPaused", QFieldType.BOOLEAN).withLabel("Indexing Paused"),
new QFieldMetaData("pausedAt", QFieldType.DATE_TIME).withLabel("Paused Since"),
new QFieldMetaData("pausedBy", QFieldType.STRING).withLabel("Paused By")))
.getProcessMetaData();
process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
if(pause)
{
process.withStep(0, new QFrontendStepMetaData()
.withName("input")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withLabel("Pause indexing")
.withFormField(new QFieldMetaData(SetIndexingPausedSteps.FIELD_REASON, QFieldType.STRING).withLabel("Reason (optional)")));
}
return (process);
}
/*******************************************************************************
** The index fields shown on the review step.
*******************************************************************************/
private static List<QFieldMetaData> indexReviewFields(QuickSearchQBitConfig config)
{
return (List.of(
new QFieldMetaData("id", QFieldType.INTEGER),
new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table")
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)),
new QFieldMetaData("enabled", QFieldType.BOOLEAN),
new QFieldMetaData("status", QFieldType.STRING),
new QFieldMetaData("documentCount", QFieldType.INTEGER)));
}
/*******************************************************************************
** A streamed-ETL process with a validated review step, gated by the admin
** permission.
*******************************************************************************/
private static QProcessMetaData base(QuickSearchQBitConfig config, String name, String label, String icon, String tableName, Class<? extends AbstractExtractStep> extract,
Class<? extends AbstractTransformStep> transform, Class<? extends AbstractLoadStep> load, Integer minInputRecords, List<QFieldMetaData> reviewFields)
{
QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
.withName(config.applyPrefix(name))
.withLabel(label)
.withIcon(new QIcon(icon))
.withTableName(tableName)
.withSourceTable(tableName)
.withDestinationTable(tableName)
.withExtractStepClass(extract)
.withTransformStepClass(transform)
.withLoadStepClass(load)
.withSupportsFullValidation(true)
.withDoFullValidation(true)
.withTransactionLevelAutoCommit()
.withMinInputRecords(minInputRecords)
.withReviewStepRecordFields(reviewFields)
.getProcessMetaData();
process.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
return (process);
}
}
@@ -0,0 +1,59 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the config processes (see QuickSearchConfigProcessMetaDataHelper).
*******************************************************************************/
public class QuickSearchConfigProcessesMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
{
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
for(QProcessMetaData process : QuickSearchConfigProcessMetaDataHelper.buildProcesses(getQBitConfig()))
{
output.add(process);
}
return (output);
}
/*******************************************************************************
** after tables, before the app
*******************************************************************************/
@Override
public int getSortOrder()
{
return (200);
}
}
@@ -0,0 +1,53 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces quick_search_config_tombstone, what an admin removed.
*******************************************************************************/
public class QuickSearchConfigTombstoneTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public QTableMetaData produce(QInstance qInstance) throws QException
{
return (QuickSearchConfigMetaDataHelper.buildTombstoneTable(getQBitConfig()));
}
/*******************************************************************************
** tables before processes and the app
*******************************************************************************/
@Override
public int getSortOrder()
{
return (100);
}
}
@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the quickSearchDiagnosticEvent table when enableDiagnosticTools is true.
** Produces the quick_search_diagnostic_event table when enableDiagnosticTools is true.
*******************************************************************************/
public class QuickSearchDiagnosticEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
@@ -217,8 +217,8 @@ public class QuickSearchDiagnosticsProcessMetaDataHelper
processes.add(buildPlaygroundProcess(config));
processes.add(buildValidateProcess(config));
processes.add(buildAnalyzeProcess(config));
processes.add(buildInspectProcess(config, config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)));
processes.add(buildReindexRecordsProcess(config, config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)));
processes.add(buildInspectProcess(config, config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)));
processes.add(buildReindexRecordsProcess(config, config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME)));
return (processes);
}
@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the quickSearchFailedEvent table.
** Produces the quick_search_failed_event table.
*******************************************************************************/
public class QuickSearchFailedEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
@@ -0,0 +1,53 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces quick_search_index_field, the searchable fields of each index.
*******************************************************************************/
public class QuickSearchIndexFieldTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public QTableMetaData produce(QInstance qInstance) throws QException
{
return (QuickSearchConfigMetaDataHelper.buildIndexFieldTable(getQBitConfig()));
}
/*******************************************************************************
** tables before processes and the app
*******************************************************************************/
@Override
public int getSortOrder()
{
return (100);
}
}
@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the possible value source over the quickSearchIndex table.
** Produces the possible value source over the quick_search_index table.
*******************************************************************************/
public class QuickSearchIndexPossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
{
@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the quickSearchIndexRun table.
** Produces the quick_search_index_run table.
*******************************************************************************/
public class QuickSearchIndexRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the quickSearchIndex table.
** Produces the quick_search_index table.
*******************************************************************************/
public class QuickSearchIndexTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
@@ -22,6 +22,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
@@ -51,7 +53,7 @@ public class QuickSearchProcessMetaDataHelper
public static QProcessMetaData buildBasepullProcess(QuickSearchQBitConfig config)
{
QProcessMetaData process = buildProcess(config, BASEPULL_PROCESS_NAME, "Quick Search Basepull Index", "basepull", BasepullIndexStep.class, "update");
process.withStep(resultStep("Basepull complete", "tablesIndexed", "failedEventsReplayed"));
process.withStep(resultStep("Basepull complete", "tablesIndexed", "tablesBuilt", "failedEventsReplayed", "paused"));
if(StringUtils.hasContent(config.getSchedulerName()))
{
@@ -73,7 +75,7 @@ public class QuickSearchProcessMetaDataHelper
{
QProcessMetaData process = buildProcess(config, FULL_REINDEX_PROCESS_NAME, "Quick Search Full Reindex", "fullReindex", FullReindexStep.class, "refresh");
process.withStep(0, inputStep(config, "Leave the table blank to rebuild every table into a fresh index and swap the alias; pick a table to rebuild just that table in place."));
process.withStep(resultStep("Full reindex complete", "recordsIndexed", "aliasSwapped"));
process.withStep(resultStep("Full reindex complete", "recordsIndexed", "aliasSwapped", "message"));
return (process);
}
@@ -87,7 +89,7 @@ public class QuickSearchProcessMetaDataHelper
{
QProcessMetaData process = buildProcess(config, RECONCILE_PROCESS_NAME, "Quick Search Reconcile Index", "reconcile", ReconcileIndexStep.class, "sync");
process.withStep(0, inputStep(config, "Leave the table blank to reconcile every table and remove documents of unconfigured tables; pick a table to reconcile just that one."));
process.withStep(resultStep("Reconcile complete", "recordsIndexed", "documentsRemoved", "orphanDocumentsRemoved", "tablesSkipped"));
process.withStep(resultStep("Reconcile complete", "recordsIndexed", "documentsRemoved", "orphanDocumentsRemoved", "tablesSkipped", "message"));
if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getReconcileCronExpression()))
{
@@ -104,18 +106,21 @@ public class QuickSearchProcessMetaDataHelper
/*******************************************************************************
** Optional table picker shown before the backend step.
** Optional table picker shown before the backend step. Its field is
** searchTableName, not tableName, which a process run from a table gets
** from the frontend (the table it runs from).
*******************************************************************************/
private static QFrontendStepMetaData inputStep(QuickSearchQBitConfig config, String help)
{
return (new QFrontendStepMetaData()
.withName("input")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HELP_TEXT).withValue("text", help))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withLabel("Choose table")
.withFormField(new QFieldMetaData("tableName", QFieldType.STRING)
.withFormField(new QFieldMetaData("searchTableName", QFieldType.STRING)
.withLabel("Table (optional)")
.withPossibleValueSourceName(config.applyPrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
.withIsRequired(false))
.withViewField(new QFieldMetaData("help", QFieldType.STRING).withLabel(" ").withDefaultValue(help)));
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
.withIsRequired(false)));
}
@@ -140,10 +145,15 @@ public class QuickSearchProcessMetaDataHelper
*******************************************************************************/
private static QProcessMetaData buildProcess(QuickSearchQBitConfig config, String baseName, String label, String stepName, Class<?> stepClass, String icon)
{
///////////////////////////////////////////////////////////////////////
// run from Quick Search Index's Actions (processes are never in the //
// nav); the input screen's table picker is searchTableName //
///////////////////////////////////////////////////////////////////////
return (new QProcessMetaData()
.withName(config.applyPrefix(baseName))
.withLabel(label)
.withIcon(new QIcon(icon))
.withTableName(config.getQuickSearchIndexTableName())
.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone())
.withStep(new QBackendStepMetaData()
.withName(stepName)
@@ -17,70 +17,45 @@
package com.kingsrook.qbits.quicksearch.metadata;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.SearchableTableConfig;
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
import com.kingsrook.qbits.quicksearch.config.QuickSearchTablePossibleValueProviders;
/*******************************************************************************
** Enum possible value source listing the searchable tables (from annotated
** entity classes and config-driven tables), used by the process input screens
** so an operator can pick one table for a full reindex or reconcile.
** Possible value source listing the indexed tables, used by the process input
** screens (pick one table for a full reindex or reconcile) and to show stored
** table names by label.
*******************************************************************************/
public class QuickSearchSearchableTablePossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
{
public static final String NAME = "quickSearchSearchableTable";
public static final String NAME = "quick_search_searchable_table";
/** Possible-value source name before 1.1.0-RC.3 (camelCase). */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_NAME = "quickSearchSearchableTable";
/*******************************************************************************
**
** A custom source over the effective indexed tables (since 1.1.0-RC.3; it
** was an enum of the code-declared tables), so tables added at runtime
** appear at once, by label.
*******************************************************************************/
@Override
public QPossibleValueSource produce(QInstance qInstance) throws QException
{
QuickSearchQBitConfig config = getQBitConfig();
Set<String> tableNames = new LinkedHashSet<>();
for(Class<?> entityClass : config.getSearchableEntityClasses() == null ? List.<Class<?>>of() : config.getSearchableEntityClasses())
{
QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class);
if(annotation != null)
{
tableNames.add(annotation.tableName());
}
}
for(SearchableTableConfig tableConfig : config.getSearchableTables() == null ? List.<SearchableTableConfig>of() : config.getSearchableTables())
{
if(tableConfig.getTableName() != null)
{
tableNames.add(tableConfig.getTableName());
}
}
List<QPossibleValue<?>> values = new ArrayList<>();
for(String tableName : tableNames)
{
QTableMetaData table = qInstance.getTable(tableName);
String label = table != null && table.getLabel() != null ? table.getLabel() : tableName;
values.add(new QPossibleValue<>(tableName, label));
}
return (new QPossibleValueSource()
.withName(config.applyPrefix(NAME))
.withType(QPossibleValueSourceType.ENUM)
.withEnumValues(values));
.withName(getQBitConfig().applyTablePrefix(NAME))
.withType(QPossibleValueSourceType.CUSTOM)
.withIdType(QFieldType.STRING)
.withCustomCodeReference(new QCodeReference(QuickSearchTablePossibleValueProviders.SearchableTables.class)));
}
@@ -0,0 +1,53 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.metadata;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces quick_search_settings, the system settings row (global pause).
*******************************************************************************/
public class QuickSearchSettingsTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public QTableMetaData produce(QInstance qInstance) throws QException
{
return (QuickSearchConfigMetaDataHelper.buildSettingsTable(getQBitConfig()));
}
/*******************************************************************************
** tables before processes and the app
*******************************************************************************/
@Override
public int getSortOrder()
{
return (100);
}
}
@@ -18,19 +18,29 @@ package com.kingsrook.qbits.quicksearch.metadata;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchIndexTableCustomizer;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
@@ -43,25 +53,46 @@ public class QuickSearchTableMetaDataHelper
{
/*******************************************************************************
** quickSearchIndex: one row per indexed table.
** quick_search_index: one row per indexed table.
*******************************************************************************/
public static QTableMetaData buildIndexTable(QuickSearchQBitConfig config)
{
////////////////////////////////////////////////////////////////////////////
// rows are added by Add Search Index (and by self-priming) and removed //
// by Remove Search Index, which validate, preview and build or clean up //
// the index; the generic create and delete screens are turned off //
////////////////////////////////////////////////////////////////////////////
return (new QTableMetaData()
.withName(config.applyPrefix(QuickSearchIndex.TABLE_NAME))
.withName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME))
.withLabel("Quick Search Index")
.withIcon(new QIcon("manage_search"))
.withBackendName(config.getBackendName())
.withPrimaryKeyField("id")
.withRecordLabelFields("tableName")
.withUniqueKey(new UniqueKey("tableName"))
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
.withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(QuickSearchIndexTableCustomizer.class))
.withAssociation(new Association()
.withName(QuickSearchConfigMetaDataHelper.FIELDS_ASSOCIATION)
.withAssociatedTableName(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME))
.withJoinName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.INDEX_FIELDS_JOIN)))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withIsEditable(false))
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table").withIsRequired(true).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
.withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
.withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("searchableFieldsJson", QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER).withLabel("Update Interval (Minutes)"))
.withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withLabel("Timestamp Field")
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.TIMESTAMP_FIELD_PVS)))
.withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.CONFIG_SOURCE_PVS)))
.withField(new QFieldMetaData("ownerEdited", QFieldType.BOOLEAN).withLabel("Edited by Owner").withIsEditable(false))
.withField(new QFieldMetaData("searchableFieldsJson", QFieldType.TEXT).withLabel("Fields as Last Built").withIsEditable(false))
.withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("lastReconcileTime", QFieldType.DATE_TIME).withIsEditable(false))
@@ -69,71 +100,110 @@ public class QuickSearchTableMetaDataHelper
.withField(new QFieldMetaData("lastErrorMessage", QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData("recordCount", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("documentCount", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("realTimeErrorCount", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("realTimeErrorCount", QFieldType.INTEGER).withIsEditable(false)
.withHelpContent(new QHelpContent("Real-time indexing failures since the last full reindex or basepull")))
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withSection(new QFieldSection("identity", "Search Index", new QIcon("manage_search"), Tier.T1,
List.of("id", "tableName", "enabled", "status", "basepullIntervalMinutes", "basepullTimestampField")))
.withSection(new QFieldSection("fields", "Searchable Fields", new QIcon("text_fields"), Tier.T2, null)
.withWidgetName(config.applyPrefix(QuickSearchConfigMetaDataHelper.INDEX_FIELDS_WIDGET)))
.withSection(new QFieldSection("runs", "Recent Runs", new QIcon("history"), Tier.T2, null)
.withWidgetName(config.applyPrefix(QuickSearchConfigMetaDataHelper.INDEX_RUNS_WIDGET)))
.withSection(new QFieldSection("activity", "Activity", new QIcon("insights"), Tier.T2,
List.of("lastBasepullTime", "lastFullReindexTime", "lastReconcileTime", "lastRunStatus", "lastErrorMessage", "recordCount", "documentCount", "realTimeErrorCount")))
.withSection(new QFieldSection("origin", "Origin", new QIcon("info"), Tier.T3,
List.of("source", "ownerEdited", "searchableFieldsJson", "createDate", "modifyDate"))));
}
/*******************************************************************************
** quickSearchIndexRun: history of basepull, reindex and reconcile runs.
** quick_search_index_run: history of basepull, reindex and reconcile runs.
*******************************************************************************/
public static QTableMetaData buildIndexRunTable(QuickSearchQBitConfig config)
{
/////////////////////////////////////////////////////////////////////////
// run history is written by the QBit only: no create or edit screens //
// (Delete stays, for clearing); Cancel Index Run sets the cancel flag //
/////////////////////////////////////////////////////////////////////////
return (new QTableMetaData()
.withName(config.applyPrefix(QuickSearchIndexRun.TABLE_NAME))
.withName(config.applyTablePrefix(QuickSearchIndexRun.TABLE_NAME))
.withLabel("Quick Search Index Run")
.withIcon(new QIcon("history"))
.withBackendName(config.getBackendName())
.withPrimaryKeyField("id")
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
.withRecordLabelFormat("%s %s")
.withRecordLabelFields("runType", "startTime")
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withIsEditable(false)
.withPossibleValueSourceName(config.applyPrefix(QuickSearchIndex.TABLE_NAME)))
.withField(new QFieldMetaData("runType", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Search Index").withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)))
.withField(new QFieldMetaData("runType", QFieldType.STRING).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.RUN_TYPE_PVS)))
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.RUN_STATUS_PVS)))
.withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("errorCount", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData("cancelRequested", QFieldType.BOOLEAN).withIsEditable(false))
.withField(new QFieldMetaData("cancelRequestedBy", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Last Heartbeat").withIsEditable(false))
.withSection(new QFieldSection("run", "Run", new QIcon("history"), Tier.T1, List.of("id", "quickSearchIndexId", "runType", "status", "startTime", "endTime")))
.withSection(new QFieldSection("counts", "Counts", new QIcon("functions"), Tier.T2, List.of("recordsProcessed", "recordsIndexed", "errorCount", "errorMessage")))
.withSection(new QFieldSection("cancel", "Cancel", new QIcon("cancel"), Tier.T2, List.of("cancelRequested", "cancelRequestedBy")))
.withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
}
/*******************************************************************************
** quickSearchFailedEvent: real-time index or delete events that could not
** quick_search_failed_event: real-time index or delete events that could not
** reach OpenSearch, replayed by the scheduled basepull.
*******************************************************************************/
public static QTableMetaData buildFailedEventTable(QuickSearchQBitConfig config)
{
/////////////////////////////////////////////////////////////////////////
// system events, written by the QBit only: no create or edit screens //
// (Delete stays, for clearing) //
/////////////////////////////////////////////////////////////////////////
return (new QTableMetaData()
.withName(config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME))
.withName(config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME))
.withLabel("Quick Search Failed Event")
.withIcon(new QIcon("error_outline"))
.withBackendName(config.getBackendName())
.withPrimaryKeyField("id")
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE)
.withRecordLabelFormat("%s %s")
.withRecordLabelFields("tableName", "recordId")
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table").withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME)))
.withField(new QFieldMetaData("recordId", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("action", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("action", QFieldType.STRING).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.EVENT_ACTION_PVS)))
.withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData("attempts", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false)
.withPossibleValueSourceName(config.applyTablePrefix(QuickSearchConfigMetaDataHelper.EVENT_STATUS_PVS)))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withSection(new QFieldSection("event", "Event", new QIcon("error_outline"), Tier.T1, List.of("id", "tableName", "recordId", "action", "status")))
.withSection(new QFieldSection("error", "Error and retries", new QIcon("replay"), Tier.T2, List.of("errorMessage", "attempts")))
.withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_today"), Tier.T3, List.of("createDate", "modifyDate"))));
}
/*******************************************************************************
** quickSearchDiagnosticEvent: failed searches, diagnostic tool runs (the
** quick_search_diagnostic_event: failed searches, diagnostic tool runs (the
** audit trail) and refused runs. Written by the QBit only: insert, update
** and delete are disabled for users and the API (core actions, which the
** QBit uses, do not check capabilities).
@@ -141,7 +211,7 @@ public class QuickSearchTableMetaDataHelper
public static QTableMetaData buildDiagnosticEventTable(QuickSearchQBitConfig config)
{
return (new QTableMetaData()
.withName(config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME))
.withName(config.applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME))
.withLabel("Quick Search Diagnostic Event")
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
.withIcon(new QIcon("bug_report"))
@@ -171,7 +241,7 @@ public class QuickSearchTableMetaDataHelper
/*******************************************************************************
** quickSearchValidationRun: index validation reports (history). The
** quick_search_validation_run: index validation reports (history). The
** report is an HTML field, shown in full on the record view; only the
** QBit writes rows (insert, update and delete are disabled for users and
** the API), so the HTML is always the QBit's escaped output.
@@ -179,7 +249,7 @@ public class QuickSearchTableMetaDataHelper
public static QTableMetaData buildValidationRunTable(QuickSearchQBitConfig config)
{
return (new QTableMetaData()
.withName(config.applyPrefix(QuickSearchValidationRun.TABLE_NAME))
.withName(config.applyTablePrefix(QuickSearchValidationRun.TABLE_NAME))
.withLabel("Quick Search Validation Run")
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
.withIcon(new QIcon("fact_check"))
@@ -199,7 +269,8 @@ public class QuickSearchTableMetaDataHelper
.withField(new QFieldMetaData("warnCount", QFieldType.INTEGER).withLabel("Warnings").withIsEditable(false))
.withField(new QFieldMetaData("summary", QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData("reportHtml", QFieldType.HTML).withLabel("Report").withIsEditable(false))
.withField(new QFieldMetaData("reportJson", QFieldType.TEXT).withLabel("Report JSON").withIsEditable(false))
.withField(new QFieldMetaData("reportJson", QFieldType.TEXT).withLabel("Report JSON").withIsEditable(false)
.withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json"))))
.withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
@@ -212,12 +283,12 @@ public class QuickSearchTableMetaDataHelper
/*******************************************************************************
** Possible value source over quickSearchIndex, so run rows show the table
** Possible value source over quick_search_index, so run rows show the table
** name instead of an id.
*******************************************************************************/
public static QPossibleValueSource buildIndexPossibleValueSource(QuickSearchQBitConfig config)
{
return (QPossibleValueSource.newForTable(config.applyPrefix(QuickSearchIndex.TABLE_NAME)));
return (QPossibleValueSource.newForTable(config.applyTablePrefix(QuickSearchIndex.TABLE_NAME)));
}
}
@@ -25,7 +25,7 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
/*******************************************************************************
** Produces the quickSearchValidationRun table when enableDiagnosticTools is true.
** Produces the quick_search_validation_run table when enableDiagnosticTools is true.
*******************************************************************************/
public class QuickSearchValidationRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
{
@@ -0,0 +1,227 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.model;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
** Entity bean for the quick_search_config_tombstone table: a table (fieldName
** null) or one field the owner removed from Quick Search. Startup priming
** never re-adds what a tombstone names, even when code still declares it;
** deleting the tombstone lets code defaults seed it again.
*******************************************************************************/
public class QuickSearchConfigTombstone extends QRecordEntity
{
public static final String TABLE_NAME = "quick_search_config_tombstone";
@QField()
private Integer id;
@QField()
private String tableName;
@QField()
private String fieldName;
@QField()
private String removedBy;
@QField()
private String reason;
@QField()
private Instant createDate;
/*******************************************************************************
** Getter for id
*******************************************************************************/
public Integer getId()
{
return (this.id);
}
/*******************************************************************************
** Setter for id
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
*******************************************************************************/
public QuickSearchConfigTombstone withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
*******************************************************************************/
public QuickSearchConfigTombstone withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for fieldName
*******************************************************************************/
public String getFieldName()
{
return (this.fieldName);
}
/*******************************************************************************
** Setter for fieldName
*******************************************************************************/
public void setFieldName(String fieldName)
{
this.fieldName = fieldName;
}
/*******************************************************************************
** Fluent setter for fieldName
*******************************************************************************/
public QuickSearchConfigTombstone withFieldName(String fieldName)
{
this.fieldName = fieldName;
return (this);
}
/*******************************************************************************
** Getter for removedBy
*******************************************************************************/
public String getRemovedBy()
{
return (this.removedBy);
}
/*******************************************************************************
** Setter for removedBy
*******************************************************************************/
public void setRemovedBy(String removedBy)
{
this.removedBy = removedBy;
}
/*******************************************************************************
** Fluent setter for removedBy
*******************************************************************************/
public QuickSearchConfigTombstone withRemovedBy(String removedBy)
{
this.removedBy = removedBy;
return (this);
}
/*******************************************************************************
** Getter for reason
*******************************************************************************/
public String getReason()
{
return (this.reason);
}
/*******************************************************************************
** Setter for reason
*******************************************************************************/
public void setReason(String reason)
{
this.reason = reason;
}
/*******************************************************************************
** Fluent setter for reason
*******************************************************************************/
public QuickSearchConfigTombstone withReason(String reason)
{
this.reason = reason;
return (this);
}
/*******************************************************************************
** Getter for createDate
*******************************************************************************/
public Instant getCreateDate()
{
return (this.createDate);
}
/*******************************************************************************
** Setter for createDate
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
*******************************************************************************/
public QuickSearchConfigTombstone withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
}
@@ -32,7 +32,11 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
*******************************************************************************/
public class QuickSearchDiagnosticEvent extends QRecordEntity
{
public static final String TABLE_NAME = "quickSearchDiagnosticEvent";
public static final String TABLE_NAME = "quick_search_diagnostic_event";
/** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_TABLE_NAME = "quickSearchDiagnosticEvent";
public static final String TYPE_SEARCH_FAILED = "SEARCH_FAILED";
public static final String TYPE_TOOL_RUN = "TOOL_RUN";
@@ -0,0 +1,90 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.model;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
/*******************************************************************************
** What happens to a table's search documents when it is disabled or
** removed: DELETE takes them out of the index now (they stop matching at
** once); KEEP leaves them in the index. A disabled table's kept documents
** are hidden from search and rebuilt when it is enabled again; a removed
** table's are swept by the next all-tables reconcile.
*******************************************************************************/
public enum QuickSearchDocumentsChoice implements PossibleValueEnum<String>
{
DELETE("Delete the documents from the search index"),
KEEP("Keep the documents");
private final String label;
/*******************************************************************************
** Constructor
*******************************************************************************/
QuickSearchDocumentsChoice(String label)
{
this.label = label;
}
/*******************************************************************************
** The value stored in the database (the constant's name).
*******************************************************************************/
@Override
public String getPossibleValueId()
{
return (name());
}
/*******************************************************************************
** The label shown in the UI.
*******************************************************************************/
@Override
public String getPossibleValueLabel()
{
return (label);
}
/*******************************************************************************
** The constant with the given name, or the fallback when it is null or
** unknown (a value written by a newer version, or by hand).
*******************************************************************************/
public static QuickSearchDocumentsChoice fromName(String value, QuickSearchDocumentsChoice fallback)
{
if(value != null)
{
for(QuickSearchDocumentsChoice candidate : values())
{
if(candidate.name().equalsIgnoreCase(value.trim()))
{
return (candidate);
}
}
}
return (fallback);
}
}
@@ -29,11 +29,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
*******************************************************************************/
public class QuickSearchFailedEvent extends QRecordEntity
{
public static final String TABLE_NAME = "quickSearchFailedEvent";
public static final String TABLE_NAME = "quick_search_failed_event";
/** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_TABLE_NAME = "quickSearchFailedEvent";
public static final String STATUS_PENDING = "PENDING";
public static final String STATUS_EXHAUSTED = "EXHAUSTED";
/** A change made while indexing was paused; becomes PENDING (replayed by basepull) when indexing resumes. */
public static final String STATUS_DEFERRED = "DEFERRED";
///////////////////////////////////////////////////////////////////////////
// a delete made while a full reindex rebuilds into a new physical index; //
// the reindex applies it after the swap, and basepull does not replay it //
@@ -0,0 +1,90 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.model;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
/*******************************************************************************
** How a searchable field's text is analyzed. PREFIX (the default) matches
** word prefixes as you type; WORD matches whole words; ENGLISH matches English
** word stems (run, running); EXACT matches the whole value, ignoring case.
*******************************************************************************/
public enum QuickSearchFieldAnalyzer implements PossibleValueEnum<String>
{
PREFIX("Prefix (as you type)"),
WORD("Whole words"),
ENGLISH("English (stemmed)"),
EXACT("Exact value");
private final String label;
/*******************************************************************************
** Constructor
*******************************************************************************/
QuickSearchFieldAnalyzer(String label)
{
this.label = label;
}
/*******************************************************************************
** The value stored in the database (the constant's name).
*******************************************************************************/
@Override
public String getPossibleValueId()
{
return (name());
}
/*******************************************************************************
** The label shown in the UI.
*******************************************************************************/
@Override
public String getPossibleValueLabel()
{
return (label);
}
/*******************************************************************************
** The constant with the given name, or the fallback when it is null or
** unknown (a value written by a newer version, or by hand).
*******************************************************************************/
public static QuickSearchFieldAnalyzer fromName(String value, QuickSearchFieldAnalyzer fallback)
{
if(value != null)
{
for(QuickSearchFieldAnalyzer candidate : values())
{
if(candidate.name().equalsIgnoreCase(value.trim()))
{
return (candidate);
}
}
}
return (fallback);
}
}
@@ -0,0 +1,93 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.model;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
/*******************************************************************************
** What a searchable field is for. TITLE fields make up the hit's label,
** SUBTITLE fields its subtitle and DISPLAY (T1) fields the short details shown
** with it; BODY is ordinary searchable text; KEYWORD is searchable but kept
** out of the highlighted snippet; EXACT matches the whole value only.
*******************************************************************************/
public enum QuickSearchFieldRole implements PossibleValueEnum<String>
{
DISPLAY("T1 display"),
TITLE("Title"),
SUBTITLE("Subtitle"),
BODY("Body"),
KEYWORD("Keyword"),
EXACT("Exact");
private final String label;
/*******************************************************************************
** Constructor
*******************************************************************************/
QuickSearchFieldRole(String label)
{
this.label = label;
}
/*******************************************************************************
** The value stored in the database (the constant's name).
*******************************************************************************/
@Override
public String getPossibleValueId()
{
return (name());
}
/*******************************************************************************
** The label shown in the UI.
*******************************************************************************/
@Override
public String getPossibleValueLabel()
{
return (label);
}
/*******************************************************************************
** The constant with the given name, or the fallback when it is null or
** unknown (a value written by a newer version, or by hand).
*******************************************************************************/
public static QuickSearchFieldRole fromName(String value, QuickSearchFieldRole fallback)
{
if(value != null)
{
for(QuickSearchFieldRole candidate : values())
{
if(candidate.name().equalsIgnoreCase(value.trim()))
{
return (candidate);
}
}
}
return (fallback);
}
}
@@ -23,7 +23,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
** Entity bean for the quickSearchIndex table.
** Entity bean for the quick_search_index table.
**
** Tracks configuration for each table that participates in the quick search
** index, including basepull scheduling, searchable field definitions, and
@@ -31,7 +31,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
*******************************************************************************/
public class QuickSearchIndex extends QRecordEntity
{
public static final String TABLE_NAME = "quickSearchIndex";
public static final String TABLE_NAME = "quick_search_index";
/** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_TABLE_NAME = "quickSearchIndex";
///////////////////////////////////////////////////////////////////////
// source: who created the row. CODE rows were seeded from code //
// defaults; OWNER rows were added by an admin. ownerEdited marks a //
// row whose config an admin changed: priming never overwrites it. //
///////////////////////////////////////////////////////////////////////
public static final String SOURCE_CODE = "CODE";
public static final String SOURCE_OWNER = "OWNER";
@QField()
private Integer id;
@@ -69,6 +81,12 @@ public class QuickSearchIndex extends QRecordEntity
@QField()
private Instant modifyDate;
@QField()
private String source;
@QField()
private Boolean ownerEdited;
/*******************************************************************************
@@ -321,4 +339,65 @@ public class QuickSearchIndex extends QRecordEntity
return (this);
}
/*******************************************************************************
** Getter for source
*******************************************************************************/
public String getSource()
{
return (this.source);
}
/*******************************************************************************
** Setter for source
*******************************************************************************/
public void setSource(String source)
{
this.source = source;
}
/*******************************************************************************
** Fluent setter for source
*******************************************************************************/
public QuickSearchIndex withSource(String source)
{
this.source = source;
return (this);
}
/*******************************************************************************
** Getter for ownerEdited
*******************************************************************************/
public Boolean getOwnerEdited()
{
return (this.ownerEdited);
}
/*******************************************************************************
** Setter for ownerEdited
*******************************************************************************/
public void setOwnerEdited(Boolean ownerEdited)
{
this.ownerEdited = ownerEdited;
}
/*******************************************************************************
** Fluent setter for ownerEdited
*******************************************************************************/
public QuickSearchIndex withOwnerEdited(Boolean ownerEdited)
{
this.ownerEdited = ownerEdited;
return (this);
}
}
@@ -0,0 +1,454 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.model;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
** Entity bean for the quick_search_index_field table: one searchable field of
** one quick_search_index row. These rows are the source of truth for what is
** indexed and how it is boosted; code (@QuickSearchField, searchableTables)
** only seeds them (see QuickSearchConfigPrimer for the precedence rules).
*******************************************************************************/
public class QuickSearchIndexField extends QRecordEntity
{
public static final String TABLE_NAME = "quick_search_index_field";
public static final String SOURCE_CODE = "CODE";
public static final String SOURCE_OWNER = "OWNER";
@QField()
private Integer id;
@QField()
private Integer quickSearchIndexId;
@QField()
private String fieldName;
@QField()
private Integer weight;
@QField()
private String role;
@QField()
private String analyzer;
@QField()
private Boolean includeInSnippet;
@QField()
private Boolean includeLabel;
@QField()
private Integer sortOrder;
@QField()
private String source;
@QField()
private Boolean ownerEdited;
@QField()
private Instant createDate;
@QField()
private Instant modifyDate;
/*******************************************************************************
** Getter for id
*******************************************************************************/
public Integer getId()
{
return (this.id);
}
/*******************************************************************************
** Setter for id
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
*******************************************************************************/
public QuickSearchIndexField withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for quickSearchIndexId
*******************************************************************************/
public Integer getQuickSearchIndexId()
{
return (this.quickSearchIndexId);
}
/*******************************************************************************
** Setter for quickSearchIndexId
*******************************************************************************/
public void setQuickSearchIndexId(Integer quickSearchIndexId)
{
this.quickSearchIndexId = quickSearchIndexId;
}
/*******************************************************************************
** Fluent setter for quickSearchIndexId
*******************************************************************************/
public QuickSearchIndexField withQuickSearchIndexId(Integer quickSearchIndexId)
{
this.quickSearchIndexId = quickSearchIndexId;
return (this);
}
/*******************************************************************************
** Getter for fieldName
*******************************************************************************/
public String getFieldName()
{
return (this.fieldName);
}
/*******************************************************************************
** Setter for fieldName
*******************************************************************************/
public void setFieldName(String fieldName)
{
this.fieldName = fieldName;
}
/*******************************************************************************
** Fluent setter for fieldName
*******************************************************************************/
public QuickSearchIndexField withFieldName(String fieldName)
{
this.fieldName = fieldName;
return (this);
}
/*******************************************************************************
** Getter for weight
*******************************************************************************/
public Integer getWeight()
{
return (this.weight);
}
/*******************************************************************************
** Setter for weight
*******************************************************************************/
public void setWeight(Integer weight)
{
this.weight = weight;
}
/*******************************************************************************
** Fluent setter for weight
*******************************************************************************/
public QuickSearchIndexField withWeight(Integer weight)
{
this.weight = weight;
return (this);
}
/*******************************************************************************
** Getter for role
*******************************************************************************/
public String getRole()
{
return (this.role);
}
/*******************************************************************************
** Setter for role
*******************************************************************************/
public void setRole(String role)
{
this.role = role;
}
/*******************************************************************************
** Fluent setter for role
*******************************************************************************/
public QuickSearchIndexField withRole(String role)
{
this.role = role;
return (this);
}
/*******************************************************************************
** Getter for analyzer
*******************************************************************************/
public String getAnalyzer()
{
return (this.analyzer);
}
/*******************************************************************************
** Setter for analyzer
*******************************************************************************/
public void setAnalyzer(String analyzer)
{
this.analyzer = analyzer;
}
/*******************************************************************************
** Fluent setter for analyzer
*******************************************************************************/
public QuickSearchIndexField withAnalyzer(String analyzer)
{
this.analyzer = analyzer;
return (this);
}
/*******************************************************************************
** Getter for includeInSnippet
*******************************************************************************/
public Boolean getIncludeInSnippet()
{
return (this.includeInSnippet);
}
/*******************************************************************************
** Setter for includeInSnippet
*******************************************************************************/
public void setIncludeInSnippet(Boolean includeInSnippet)
{
this.includeInSnippet = includeInSnippet;
}
/*******************************************************************************
** Fluent setter for includeInSnippet
*******************************************************************************/
public QuickSearchIndexField withIncludeInSnippet(Boolean includeInSnippet)
{
this.includeInSnippet = includeInSnippet;
return (this);
}
/*******************************************************************************
** Getter for includeLabel
*******************************************************************************/
public Boolean getIncludeLabel()
{
return (this.includeLabel);
}
/*******************************************************************************
** Setter for includeLabel
*******************************************************************************/
public void setIncludeLabel(Boolean includeLabel)
{
this.includeLabel = includeLabel;
}
/*******************************************************************************
** Fluent setter for includeLabel
*******************************************************************************/
public QuickSearchIndexField withIncludeLabel(Boolean includeLabel)
{
this.includeLabel = includeLabel;
return (this);
}
/*******************************************************************************
** Getter for sortOrder
*******************************************************************************/
public Integer getSortOrder()
{
return (this.sortOrder);
}
/*******************************************************************************
** Setter for sortOrder
*******************************************************************************/
public void setSortOrder(Integer sortOrder)
{
this.sortOrder = sortOrder;
}
/*******************************************************************************
** Fluent setter for sortOrder
*******************************************************************************/
public QuickSearchIndexField withSortOrder(Integer sortOrder)
{
this.sortOrder = sortOrder;
return (this);
}
/*******************************************************************************
** Getter for source
*******************************************************************************/
public String getSource()
{
return (this.source);
}
/*******************************************************************************
** Setter for source
*******************************************************************************/
public void setSource(String source)
{
this.source = source;
}
/*******************************************************************************
** Fluent setter for source
*******************************************************************************/
public QuickSearchIndexField withSource(String source)
{
this.source = source;
return (this);
}
/*******************************************************************************
** Getter for ownerEdited
*******************************************************************************/
public Boolean getOwnerEdited()
{
return (this.ownerEdited);
}
/*******************************************************************************
** Setter for ownerEdited
*******************************************************************************/
public void setOwnerEdited(Boolean ownerEdited)
{
this.ownerEdited = ownerEdited;
}
/*******************************************************************************
** Fluent setter for ownerEdited
*******************************************************************************/
public QuickSearchIndexField withOwnerEdited(Boolean ownerEdited)
{
this.ownerEdited = ownerEdited;
return (this);
}
/*******************************************************************************
** Getter for createDate
*******************************************************************************/
public Instant getCreateDate()
{
return (this.createDate);
}
/*******************************************************************************
** Setter for createDate
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
*******************************************************************************/
public QuickSearchIndexField withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Getter for modifyDate
*******************************************************************************/
public Instant getModifyDate()
{
return (this.modifyDate);
}
/*******************************************************************************
** Setter for modifyDate
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Fluent setter for modifyDate
*******************************************************************************/
public QuickSearchIndexField withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
}
@@ -23,7 +23,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
** Entity bean for the quickSearchIndexRun table.
** Entity bean for the quick_search_index_run table.
**
** Records a single execution of the indexing process for a given
** QuickSearchIndex, capturing run type, timing, record counts, and any errors
@@ -31,7 +31,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
*******************************************************************************/
public class QuickSearchIndexRun extends QRecordEntity
{
public static final String TABLE_NAME = "quickSearchIndexRun";
public static final String TABLE_NAME = "quick_search_index_run";
/** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_TABLE_NAME = "quickSearchIndexRun";
@QField()
private Integer id;
@@ -69,6 +73,12 @@ public class QuickSearchIndexRun extends QRecordEntity
@QField()
private Instant modifyDate;
@QField()
private Boolean cancelRequested;
@QField()
private String cancelRequestedBy;
/*******************************************************************************
@@ -321,4 +331,65 @@ public class QuickSearchIndexRun extends QRecordEntity
return (this);
}
/*******************************************************************************
** Getter for cancelRequested
*******************************************************************************/
public Boolean getCancelRequested()
{
return (this.cancelRequested);
}
/*******************************************************************************
** Setter for cancelRequested
*******************************************************************************/
public void setCancelRequested(Boolean cancelRequested)
{
this.cancelRequested = cancelRequested;
}
/*******************************************************************************
** Fluent setter for cancelRequested
*******************************************************************************/
public QuickSearchIndexRun withCancelRequested(Boolean cancelRequested)
{
this.cancelRequested = cancelRequested;
return (this);
}
/*******************************************************************************
** Getter for cancelRequestedBy
*******************************************************************************/
public String getCancelRequestedBy()
{
return (this.cancelRequestedBy);
}
/*******************************************************************************
** Setter for cancelRequestedBy
*******************************************************************************/
public void setCancelRequestedBy(String cancelRequestedBy)
{
this.cancelRequestedBy = cancelRequestedBy;
}
/*******************************************************************************
** Fluent setter for cancelRequestedBy
*******************************************************************************/
public QuickSearchIndexRun withCancelRequestedBy(String cancelRequestedBy)
{
this.cancelRequestedBy = cancelRequestedBy;
return (this);
}
}
@@ -0,0 +1,356 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.model;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
** Entity bean for the quick_search_settings table: the one row (id 1) of
** system-level Quick Search settings. indexingPaused stops all indexing
** (real-time and scheduled) while search keeps serving the existing index;
** the priming fields report what startup self-priming last added.
*******************************************************************************/
public class QuickSearchSettings extends QRecordEntity
{
public static final String TABLE_NAME = "quick_search_settings";
public static final Integer SETTINGS_ID = 1;
@QField()
private Integer id;
@QField()
private Boolean indexingPaused;
@QField()
private Instant pausedAt;
@QField()
private String pausedBy;
@QField()
private String pauseReason;
@QField()
private Instant lastPrimedAt;
@QField()
private String lastPrimingSummary;
@QField()
private Integer lastPrimingNewTables;
@QField()
private Instant createDate;
@QField()
private Instant modifyDate;
/*******************************************************************************
** Getter for id
*******************************************************************************/
public Integer getId()
{
return (this.id);
}
/*******************************************************************************
** Setter for id
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
*******************************************************************************/
public QuickSearchSettings withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Getter for indexingPaused
*******************************************************************************/
public Boolean getIndexingPaused()
{
return (this.indexingPaused);
}
/*******************************************************************************
** Setter for indexingPaused
*******************************************************************************/
public void setIndexingPaused(Boolean indexingPaused)
{
this.indexingPaused = indexingPaused;
}
/*******************************************************************************
** Fluent setter for indexingPaused
*******************************************************************************/
public QuickSearchSettings withIndexingPaused(Boolean indexingPaused)
{
this.indexingPaused = indexingPaused;
return (this);
}
/*******************************************************************************
** Getter for pausedAt
*******************************************************************************/
public Instant getPausedAt()
{
return (this.pausedAt);
}
/*******************************************************************************
** Setter for pausedAt
*******************************************************************************/
public void setPausedAt(Instant pausedAt)
{
this.pausedAt = pausedAt;
}
/*******************************************************************************
** Fluent setter for pausedAt
*******************************************************************************/
public QuickSearchSettings withPausedAt(Instant pausedAt)
{
this.pausedAt = pausedAt;
return (this);
}
/*******************************************************************************
** Getter for pausedBy
*******************************************************************************/
public String getPausedBy()
{
return (this.pausedBy);
}
/*******************************************************************************
** Setter for pausedBy
*******************************************************************************/
public void setPausedBy(String pausedBy)
{
this.pausedBy = pausedBy;
}
/*******************************************************************************
** Fluent setter for pausedBy
*******************************************************************************/
public QuickSearchSettings withPausedBy(String pausedBy)
{
this.pausedBy = pausedBy;
return (this);
}
/*******************************************************************************
** Getter for pauseReason
*******************************************************************************/
public String getPauseReason()
{
return (this.pauseReason);
}
/*******************************************************************************
** Setter for pauseReason
*******************************************************************************/
public void setPauseReason(String pauseReason)
{
this.pauseReason = pauseReason;
}
/*******************************************************************************
** Fluent setter for pauseReason
*******************************************************************************/
public QuickSearchSettings withPauseReason(String pauseReason)
{
this.pauseReason = pauseReason;
return (this);
}
/*******************************************************************************
** Getter for lastPrimedAt
*******************************************************************************/
public Instant getLastPrimedAt()
{
return (this.lastPrimedAt);
}
/*******************************************************************************
** Setter for lastPrimedAt
*******************************************************************************/
public void setLastPrimedAt(Instant lastPrimedAt)
{
this.lastPrimedAt = lastPrimedAt;
}
/*******************************************************************************
** Fluent setter for lastPrimedAt
*******************************************************************************/
public QuickSearchSettings withLastPrimedAt(Instant lastPrimedAt)
{
this.lastPrimedAt = lastPrimedAt;
return (this);
}
/*******************************************************************************
** Getter for lastPrimingSummary
*******************************************************************************/
public String getLastPrimingSummary()
{
return (this.lastPrimingSummary);
}
/*******************************************************************************
** Setter for lastPrimingSummary
*******************************************************************************/
public void setLastPrimingSummary(String lastPrimingSummary)
{
this.lastPrimingSummary = lastPrimingSummary;
}
/*******************************************************************************
** Fluent setter for lastPrimingSummary
*******************************************************************************/
public QuickSearchSettings withLastPrimingSummary(String lastPrimingSummary)
{
this.lastPrimingSummary = lastPrimingSummary;
return (this);
}
/*******************************************************************************
** Getter for lastPrimingNewTables
*******************************************************************************/
public Integer getLastPrimingNewTables()
{
return (this.lastPrimingNewTables);
}
/*******************************************************************************
** Setter for lastPrimingNewTables
*******************************************************************************/
public void setLastPrimingNewTables(Integer lastPrimingNewTables)
{
this.lastPrimingNewTables = lastPrimingNewTables;
}
/*******************************************************************************
** Fluent setter for lastPrimingNewTables
*******************************************************************************/
public QuickSearchSettings withLastPrimingNewTables(Integer lastPrimingNewTables)
{
this.lastPrimingNewTables = lastPrimingNewTables;
return (this);
}
/*******************************************************************************
** Getter for createDate
*******************************************************************************/
public Instant getCreateDate()
{
return (this.createDate);
}
/*******************************************************************************
** Setter for createDate
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Fluent setter for createDate
*******************************************************************************/
public QuickSearchSettings withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Getter for modifyDate
*******************************************************************************/
public Instant getModifyDate()
{
return (this.modifyDate);
}
/*******************************************************************************
** Setter for modifyDate
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Fluent setter for modifyDate
*******************************************************************************/
public QuickSearchSettings withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
}
@@ -31,7 +31,11 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
*******************************************************************************/
public class QuickSearchValidationRun extends QRecordEntity
{
public static final String TABLE_NAME = "quickSearchValidationRun";
public static final String TABLE_NAME = "quick_search_validation_run";
/** Table name before 1.1.0-RC.3 (camelCase). Kept only for upgrade scripts; see docs/MIGRATION-1.1.md. */
@Deprecated(since = "1.1.0-RC.3")
public static final String LEGACY_TABLE_NAME = "quickSearchValidationRun";
public static final String STATUS_PASS = "PASS";
public static final String STATUS_WARN = "WARN";
@@ -20,6 +20,7 @@ package com.kingsrook.qbits.quicksearch.opensearch;
import java.time.Instant;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
/*******************************************************************************
@@ -38,6 +39,21 @@ public class OpenSearchDocument
private Instant indexedAt;
private Map<String, Object> fieldValues;
////////////////////////////////////////////////////////////////////////
// set only for tables whose fields have roles or non-default //
// analyzers, so other documents are written exactly as before //
////////////////////////////////////////////////////////////////////////
@JsonInclude(JsonInclude.Include.NON_NULL)
private String subtitle;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Map<String, String> displayValues;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Map<String, Object> fieldValuesWord;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Map<String, Object> fieldValuesEnglish;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Map<String, Object> fieldValuesExact;
private transient Long version;
@@ -212,4 +228,155 @@ public class OpenSearchDocument
return (sourceTable + ":" + recordId);
}
/*******************************************************************************
** Getter for subtitle
*******************************************************************************/
public String getSubtitle()
{
return (this.subtitle);
}
/*******************************************************************************
** Setter for subtitle
*******************************************************************************/
public void setSubtitle(String subtitle)
{
this.subtitle = subtitle;
}
/*******************************************************************************
** Fluent setter for subtitle
*******************************************************************************/
public OpenSearchDocument withSubtitle(String subtitle)
{
this.subtitle = subtitle;
return (this);
}
/*******************************************************************************
** Getter for displayValues
*******************************************************************************/
public Map<String, String> getDisplayValues()
{
return (this.displayValues);
}
/*******************************************************************************
** Setter for displayValues
*******************************************************************************/
public void setDisplayValues(Map<String, String> displayValues)
{
this.displayValues = displayValues;
}
/*******************************************************************************
** Fluent setter for displayValues
*******************************************************************************/
public OpenSearchDocument withDisplayValues(Map<String, String> displayValues)
{
this.displayValues = displayValues;
return (this);
}
/*******************************************************************************
** Getter for fieldValuesWord
*******************************************************************************/
public Map<String, Object> getFieldValuesWord()
{
return (this.fieldValuesWord);
}
/*******************************************************************************
** Setter for fieldValuesWord
*******************************************************************************/
public void setFieldValuesWord(Map<String, Object> fieldValuesWord)
{
this.fieldValuesWord = fieldValuesWord;
}
/*******************************************************************************
** Fluent setter for fieldValuesWord
*******************************************************************************/
public OpenSearchDocument withFieldValuesWord(Map<String, Object> fieldValuesWord)
{
this.fieldValuesWord = fieldValuesWord;
return (this);
}
/*******************************************************************************
** Getter for fieldValuesEnglish
*******************************************************************************/
public Map<String, Object> getFieldValuesEnglish()
{
return (this.fieldValuesEnglish);
}
/*******************************************************************************
** Setter for fieldValuesEnglish
*******************************************************************************/
public void setFieldValuesEnglish(Map<String, Object> fieldValuesEnglish)
{
this.fieldValuesEnglish = fieldValuesEnglish;
}
/*******************************************************************************
** Fluent setter for fieldValuesEnglish
*******************************************************************************/
public OpenSearchDocument withFieldValuesEnglish(Map<String, Object> fieldValuesEnglish)
{
this.fieldValuesEnglish = fieldValuesEnglish;
return (this);
}
/*******************************************************************************
** Getter for fieldValuesExact
*******************************************************************************/
public Map<String, Object> getFieldValuesExact()
{
return (this.fieldValuesExact);
}
/*******************************************************************************
** Setter for fieldValuesExact
*******************************************************************************/
public void setFieldValuesExact(Map<String, Object> fieldValuesExact)
{
this.fieldValuesExact = fieldValuesExact;
}
/*******************************************************************************
** Fluent setter for fieldValuesExact
*******************************************************************************/
public OpenSearchDocument withFieldValuesExact(Map<String, Object> fieldValuesExact)
{
this.fieldValuesExact = fieldValuesExact;
return (this);
}
}
@@ -33,6 +33,8 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
import jakarta.json.stream.JsonGenerator;
import org.opensearch.client.json.JsonpMapper;
import org.opensearch.client.json.JsonpSerializable;
@@ -42,6 +44,7 @@ import org.opensearch.client.opensearch._types.query_dsl.Query;
import org.opensearch.client.opensearch.core.ExplainResponse;
import org.opensearch.client.opensearch.core.SearchRequest;
import org.opensearch.client.opensearch.core.SearchResponse;
import org.opensearch.client.opensearch.core.search.Highlight;
import org.opensearch.client.opensearch.core.search.Hit;
import org.opensearch.client.opensearch.core.search.HighlightField;
import org.opensearch.client.opensearch.core.search.MatchedQueries;
@@ -169,13 +172,14 @@ public class QuickSearchDiagnosticsOperations
{
int boundedSize = Math.max(1, Math.min(size, MAX_PROBE_SIZE));
Query query = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, true);
Highlight highlight = QuickSearchFieldRules.requiresFieldFiltering(tableName) ? null : Highlight.of(h -> h
.requireFieldMatch(true)
.fields("searchableText", HighlightField.of(hf -> hf))
.fields("fieldValues.*", HighlightField.of(hf -> hf)));
SearchRequest request = SearchRequest.of(r -> r
.index(indexName)
.query(query)
.highlight(h -> h
.requireFieldMatch(true)
.fields("searchableText", HighlightField.of(hf -> hf))
.fields("fieldValues.*", HighlightField.of(hf -> hf)))
.highlight(highlight)
.trackTotalHits(t -> t.count(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW))
.size(boundedSize));
@@ -189,17 +193,26 @@ public class QuickSearchDiagnosticsOperations
for(Hit<OpenSearchDocument> hit : response.hits().hits())
{
OpenSearchDocument source = hit.source();
boolean filterFields = QuickSearchFieldRules.requiresFieldFiltering(tableName);
if(source != null)
{
source = IndexingUtils.filterResult(source, tableConfigs.stream().filter(c -> tableName.equals(c.getTableName())).findFirst().orElse(null));
}
if(filterFields && source == null)
{
continue;
}
hits.add(new ProbeHit(
source == null ? idFromDocumentId(hit.id()) : source.getRecordId(),
source == null ? null : source.getRecordLabel(),
hit.score(),
matchedClauses(hit.matchedQueries()),
hit.highlight() == null ? Map.of() : new LinkedHashMap<>(hit.highlight())));
filterFields || hit.highlight() == null ? Map.of() : new LinkedHashMap<>(hit.highlight())));
}
Long total = response.hits().total() == null ? (long) hits.size() : response.hits().total().value();
Boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
return (new SearchProbe(tableName, toJson(request), toJson(response), response.took(), wall, total, lowerBound, hits));
return (new SearchProbe(tableName, toJson(request), QuickSearchFieldRules.requiresFieldFiltering(tableName) ? "{}" : toJson(response), response.took(), wall, total, lowerBound, hits));
}
catch(OpenSearchException | IOException e)
{
@@ -308,7 +321,7 @@ public class QuickSearchDiagnosticsOperations
requireOk(response, "Document read failed");
JsonNode body = response.body();
return (new DocumentView(body.path("found").asBoolean(false), body.has("_version") ? body.get("_version").asLong() : null,
body.path("_index").asText(null), body.get("_source")));
body.path("_index").asText(null), filterDocumentSource(tableName, body.get("_source"))));
}
@@ -338,7 +351,7 @@ public class QuickSearchDiagnosticsOperations
{
if(doc.path("found").asBoolean(false) && doc.has("_source"))
{
sources.put(idFromDocumentId(doc.path("_id").asText()), doc.get("_source"));
sources.put(idFromDocumentId(doc.path("_id").asText()), filterDocumentSource(tableName, doc.get("_source")));
}
}
return (sources);
@@ -346,6 +359,39 @@ public class QuickSearchDiagnosticsOperations
/*******************************************************************************
** Old documents may retain protected values. Diagnostic reads expose only
** attributable, indexable fields, never the old combined presentation.
*******************************************************************************/
private static JsonNode filterDocumentSource(String tableName, JsonNode source)
{
if(source == null || !QuickSearchFieldRules.requiresFieldFiltering(tableName))
{
return (source);
}
ObjectNode filtered = JsonNodeFactory.instance.objectNode();
filtered.put("sourceTable", tableName);
var table = QuickSearchFieldRules.lookupTable(tableName);
if(!QuickSearchFieldRules.isProtected(table, table.getPrimaryKeyField()))
{
filtered.set("recordId", source.get("recordId"));
}
for(String container : List.of("fieldValues", "fieldValuesWord", "fieldValuesEnglish", "fieldValuesExact"))
{
ObjectNode values = filtered.putObject(container);
source.path(container).fields().forEachRemaining(entry ->
{
if(QuickSearchFieldRules.isIndexable(table, entry.getKey()))
{
values.set(entry.getKey(), entry.getValue());
}
});
}
return (filtered);
}
/*******************************************************************************
** Run _analyze on the index with a named analyzer (an index analyzer or a
** built-in one). Text is truncated to MAX_ANALYZE_TEXT_LENGTH.
@@ -34,8 +34,13 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.fasterxml.jackson.databind.JsonNode;
import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldAnalyzer;
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
import org.opensearch.client.json.JsonData;
import org.opensearch.client.opensearch.OpenSearchAsyncClient;
import org.opensearch.client.opensearch.OpenSearchClient;
@@ -51,6 +56,7 @@ import org.opensearch.client.opensearch._types.query_dsl.MultiMatchQuery;
import org.opensearch.client.opensearch._types.query_dsl.Operator;
import org.opensearch.client.opensearch._types.query_dsl.Query;
import org.opensearch.client.opensearch._types.query_dsl.TermQuery;
import org.opensearch.client.opensearch._types.query_dsl.TextQueryType;
import org.opensearch.client.opensearch.core.BulkRequest;
import org.opensearch.client.opensearch.core.BulkResponse;
import org.opensearch.client.opensearch.core.DeleteByQueryRequest;
@@ -104,6 +110,18 @@ public class QuickSearchOpenSearchClient implements Closeable
public static final int MAX_RESULT_WINDOW = 10_000;
public static final String MATCH_CLAUSE_NAME = "searchableText";
public static final String FIELD_CLAUSE_PREFIX = "field:";
public static final String PATH_PREFIX = "fieldValues";
public static final String PATH_WORD = "fieldValuesWord";
public static final String PATH_ENGLISH = "fieldValuesEnglish";
public static final String PATH_EXACT = "fieldValuesExact";
////////////////////////////////////////////////////////////////////////////
// record search highlight markers: private-use characters, which do not //
// occur in indexed text, so fragments convert to plain text and ranges //
////////////////////////////////////////////////////////////////////////////
public static final String HIGHLIGHT_PRE_TAG = "\uE000"; // private use area
public static final String HIGHLIGHT_POST_TAG = "\uE001"; // private use area
public static final int HIGHLIGHT_FRAGMENT_SIZE = 160;
private final OpenSearchClient client;
private final OpenSearchAsyncClient asyncClient;
@@ -111,9 +129,26 @@ public class QuickSearchOpenSearchClient implements Closeable
private final String indexName;
private final int maxBulkRequestBytes;
private final ObjectMapper sizingMapper;
private final Integer indexReplicas;
private final Integer maxAutoReplicas;
private final Boolean replicasManaged;
private volatile Boolean mappingOutdated = false;
private volatile Boolean indexCreated = false;
private volatile Instant indexCreatedAt;
private Boolean closed = false;
private volatile ReplicaStatus replicaStatus;
/*******************************************************************************
** Outcome of the latest replica sizing: the target number_of_replicas, the
** data nodes it was sized for (null when explicitly configured), how many
** physical indexes were changed, and whether applying it failed.
*******************************************************************************/
public record ReplicaStatus(Integer targetReplicas, Integer dataNodes, Integer updatedIndexes, Boolean failed)
{
}
@@ -128,6 +163,9 @@ public class QuickSearchOpenSearchClient implements Closeable
this.indexName = config.getOpensearchIndexName();
this.maxBulkRequestBytes = config.getMaxBulkRequestBytes() == null ? 5 * 1024 * 1024 : config.getMaxBulkRequestBytes();
this.sizingMapper = new ObjectMapper().registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
this.indexReplicas = config.getIndexReplicas();
this.maxAutoReplicas = config.getMaxAutoReplicas() == null ? QuickSearchQBitConfig.DEFAULT_MAX_AUTO_REPLICAS : config.getMaxAutoReplicas();
this.replicasManaged = !(config.getEffectiveAuthMode() == QuickSearchAuthMode.AWS_SIGV4 && "aoss".equals(config.getEffectiveAwsServiceName()));
}
@@ -152,7 +190,9 @@ public class QuickSearchOpenSearchClient implements Closeable
else
{
LOG.info("OpenSearch index already exists", logPair("indexName", indexName));
extendMapping();
}
reconcileReplicas();
return;
}
@@ -160,6 +200,8 @@ public class QuickSearchOpenSearchClient implements Closeable
createPhysicalIndex(physicalIndex);
client.indices().updateAliases(u -> u.actions(a -> a.add(ad -> ad.index(physicalIndex).alias(indexName))));
mappingOutdated = false;
indexCreated = true;
indexCreatedAt = Instant.now();
LOG.info("Created OpenSearch index", logPair("indexName", indexName), logPair("physicalIndex", physicalIndex));
}
catch(OpenSearchException | java.io.IOException e)
@@ -170,6 +212,101 @@ public class QuickSearchOpenSearchClient implements Closeable
/*******************************************************************************
** Whether ensureIndexExists found no index and created an empty one (a new
** install, or the index was deleted). Self-priming then queues a build of
** every enabled table. Cleared by clearIndexCreated().
*******************************************************************************/
public boolean wasIndexCreated()
{
return (Boolean.TRUE.equals(indexCreated));
}
/*******************************************************************************
** When ensureIndexExists last created the index (null when it has not).
*******************************************************************************/
public Instant getIndexCreatedAt()
{
return (indexCreatedAt);
}
/*******************************************************************************
** Forget that the index was created (its builds have been queued).
*******************************************************************************/
public void clearIndexCreated()
{
indexCreated = false;
}
/*******************************************************************************
** Whether the alias (or a legacy concrete index) exists. Throws when the
** cluster cannot be asked.
*******************************************************************************/
public boolean indexExists() throws QException
{
try
{
return (client.indices().exists(ExistsRequest.of(r -> r.index(indexName))).value());
}
catch(OpenSearchException | java.io.IOException e)
{
throw wrap("Failed to check whether index [" + indexName + "] exists", e);
}
}
/*******************************************************************************
** Number of documents in the index, all tables.
*******************************************************************************/
public Long countAllDocuments() throws QException
{
try
{
return (client.count(c -> c.index(indexName)).count());
}
catch(OpenSearchException | java.io.IOException e)
{
throw wrap("Failed to count documents in index [" + indexName + "]", e);
}
}
/*******************************************************************************
** Add the field-analyzer templates and the role properties (added in
** 1.1.0-RC.3) to an index created before them. Additive, so the mapping
** version stays 2 and no rebuild is needed; a failure is logged and only
** means those fields fall back to dynamic mapping.
*******************************************************************************/
void extendMapping()
{
try
{
client.indices().putMapping(p -> p
.index(indexName)
.dynamicTemplates(dynamicTemplates())
.properties("subtitle", pr -> pr.text(t -> t.index(false)))
.properties("displayValues", pr -> pr.object(o -> o.enabled(false)))
.properties(PATH_WORD, pr -> pr.object(o -> o))
.properties(PATH_ENGLISH, pr -> pr.object(o -> o))
.properties(PATH_EXACT, pr -> pr.object(o -> o)));
}
catch(Exception e)
{
LOG.warn("Could not add the field-analyzer mapping to the existing index; WORD, ENGLISH and EXACT fields use dynamic mapping until a full reindex",
e, logPair("indexName", indexName));
}
}
/*******************************************************************************
** Whether the live index was created with an older mapping than this
** client writes. A full reindex of all tables rebuilds it.
@@ -198,13 +335,21 @@ public class QuickSearchOpenSearchClient implements Closeable
{
try
{
Integer replicas = replicasForNewIndex();
client.indices().create(CreateIndexRequest.of(r -> r
.index(physicalIndex)
.settings(s -> s
.analysis(a -> a
.settings(s ->
{
s.analysis(a -> a
.analyzer(INDEX_ANALYZER, an -> an.custom(c -> c.tokenizer("standard").filter("lowercase", "edge_ngram_filter")))
.analyzer(SEARCH_ANALYZER, an -> an.custom(c -> c.tokenizer("standard").filter("lowercase")))
.filter("edge_ngram_filter", f -> f.definition(d -> d.edgeNgram(en -> en.minGram(2).maxGram(20))))))
.filter("edge_ngram_filter", f -> f.definition(d -> d.edgeNgram(en -> en.minGram(2).maxGram(20)))));
if(replicas != null)
{
s.numberOfReplicas(replicas);
}
return (s);
})
.mappings(buildMapping())));
}
catch(OpenSearchException | java.io.IOException e)
@@ -215,20 +360,142 @@ public class QuickSearchOpenSearchClient implements Closeable
/*******************************************************************************
** Replicas a new physical index should ask for; null leaves the server
** default (Serverless, or the cluster could not be sized; logged).
*******************************************************************************/
private Integer replicasForNewIndex()
{
if(!Boolean.TRUE.equals(replicasManaged))
{
return (null);
}
try
{
ReplicaStatus target = targetReplicas();
replicaStatus = new ReplicaStatus(target.targetReplicas(), target.dataNodes(), 0, false);
return (target.targetReplicas());
}
catch(Exception e)
{
replicaStatus = new ReplicaStatus(null, null, 0, true);
LOG.warn("Could not size Quick Search index replicas; the new index uses the server default", e, logPair("indexName", indexName));
return (null);
}
}
/*******************************************************************************
** The configured indexReplicas, else min(maxAutoReplicas, data nodes - 1)
** from cluster health, so a single-node cluster gets 0 and stays green.
*******************************************************************************/
ReplicaStatus targetReplicas() throws QException
{
if(indexReplicas != null)
{
return (new ReplicaStatus(indexReplicas, null, 0, false));
}
QuickSearchDiagnosticsOperations.RawResponse health = new QuickSearchDiagnosticsOperations(this).raw("GET", "/_cluster/health", null);
QuickSearchDiagnosticsOperations.requireOk(health, "Cluster health unavailable");
JsonNode dataNodes = health.body() == null ? null : health.body().path("number_of_data_nodes");
if(dataNodes == null || !dataNodes.isNumber())
{
throw (new QException("Cluster health did not report number_of_data_nodes"));
}
return (new ReplicaStatus(autoReplicas(dataNodes.intValue(), maxAutoReplicas), dataNodes.intValue(), 0, false));
}
/*******************************************************************************
** Automatic replica count: a replica can only be assigned to a different
** data node than its primary, so never more than data nodes - 1.
*******************************************************************************/
static int autoReplicas(int dataNodes, int maxAutoReplicas)
{
return (Math.max(0, Math.min(maxAutoReplicas, dataNodes - 1)));
}
/*******************************************************************************
** Bring number_of_replicas of the physical index(es) behind the alias to
** the target. Idempotent (only differing indexes are updated) and failure
** tolerant: a failure is logged and kept in {@link #getReplicaStatus()},
** never thrown, so it cannot block startup. Indexes with
** auto_expand_replicas set are left to OpenSearch. Null on Serverless.
*******************************************************************************/
public ReplicaStatus reconcileReplicas()
{
if(!Boolean.TRUE.equals(replicasManaged))
{
return (null);
}
Integer target = null;
Integer nodes = null;
try
{
ReplicaStatus sized = targetReplicas();
target = sized.targetReplicas();
nodes = sized.dataNodes();
QuickSearchDiagnosticsOperations operations = new QuickSearchDiagnosticsOperations(this);
QuickSearchDiagnosticsOperations.RawResponse settings = operations.raw("GET", "/" + QuickSearchDiagnosticsOperations.encode(indexName) + "/_settings", null);
QuickSearchDiagnosticsOperations.requireOk(settings, "Index settings unavailable");
int updated = 0;
for(Map.Entry<String, JsonNode> entry : settings.body().properties())
{
JsonNode index = entry.getValue().path("settings").path("index");
String current = index.path("number_of_replicas").asText(null);
String autoExpand = index.path("auto_expand_replicas").asText("false");
if(!"false".equalsIgnoreCase(autoExpand) || String.valueOf(target).equals(current))
{
continue;
}
QuickSearchDiagnosticsOperations.RawResponse put = operations.raw("PUT", "/" + QuickSearchDiagnosticsOperations.encode(entry.getKey()) + "/_settings",
"{\"index\":{\"number_of_replicas\":" + target + "}}");
QuickSearchDiagnosticsOperations.requireOk(put, "Updating number_of_replicas failed");
updated++;
LOG.info("Updated Quick Search index replicas", logPair("physicalIndex", entry.getKey()), logPair("from", current),
logPair("to", target), logPair("dataNodes", nodes));
}
replicaStatus = new ReplicaStatus(target, nodes, updated, false);
}
catch(Exception e)
{
LOG.warn("Could not reconcile Quick Search index replicas; continuing", e, logPair("indexName", indexName), logPair("targetReplicas", target));
replicaStatus = new ReplicaStatus(target, nodes, 0, true);
}
return (replicaStatus);
}
/*******************************************************************************
** The latest replica sizing outcome; null before any (or on Serverless).
*******************************************************************************/
public ReplicaStatus getReplicaStatus()
{
return (replicaStatus);
}
/*******************************************************************************
**
*******************************************************************************/
private TypeMapping buildMapping()
{
DynamicTemplate fieldValuesAsText = DynamicTemplate.of(t -> t
.pathMatch("fieldValues.*")
.mapping(p -> p.text(tx -> tx.analyzer(INDEX_ANALYZER).searchAnalyzer(SEARCH_ANALYZER))));
return (TypeMapping.of(m -> m
.meta(META_MAPPING_VERSION, JsonData.of(MAPPING_VERSION))
.dateDetection(false)
.numericDetection(false)
.dynamicTemplates(Map.of("fieldValuesAsText", fieldValuesAsText))
.dynamicTemplates(dynamicTemplates())
.properties("subtitle", p -> p.text(t -> t.index(false)))
.properties("displayValues", p -> p.object(o -> o.enabled(false)))
.properties(PATH_WORD, p -> p.object(o -> o))
.properties(PATH_ENGLISH, p -> p.object(o -> o))
.properties(PATH_EXACT, p -> p.object(o -> o))
.properties("sourceTable", p -> p.keyword(k -> k))
.properties("recordId", p -> p.keyword(k -> k))
.properties("recordLabel", p -> p.text(t -> t.analyzer(INDEX_ANALYZER).searchAnalyzer(SEARCH_ANALYZER)))
@@ -239,6 +506,27 @@ public class QuickSearchOpenSearchClient implements Closeable
/*******************************************************************************
** The dynamic templates: fieldValues.* with the prefix analyzers (as since
** mapping version 2), and the field-analyzer copies: fieldValuesWord.*
** (standard), fieldValuesEnglish.* (english) and fieldValuesExact.*
** (keyword; values are lower-cased before indexing).
*******************************************************************************/
static List<Map<String, DynamicTemplate>> dynamicTemplates()
{
return (List.of(
Map.of("fieldValuesAsText", DynamicTemplate.of(t -> t.pathMatch(PATH_PREFIX + ".*")
.mapping(p -> p.text(tx -> tx.analyzer(INDEX_ANALYZER).searchAnalyzer(SEARCH_ANALYZER))))),
Map.of("fieldValuesWordAsText", DynamicTemplate.of(t -> t.pathMatch(PATH_WORD + ".*")
.mapping(p -> p.text(tx -> tx.analyzer("standard"))))),
Map.of("fieldValuesEnglishAsText", DynamicTemplate.of(t -> t.pathMatch(PATH_ENGLISH + ".*")
.mapping(p -> p.text(tx -> tx.analyzer("english"))))),
Map.of("fieldValuesExactAsKeyword", DynamicTemplate.of(t -> t.pathMatch(PATH_EXACT + ".*")
.mapping(p -> p.keyword(k -> k.ignoreAbove(1024)))))));
}
/*******************************************************************************
** Read quickSearchMappingVersion from the live mapping's _meta; 1 when
** absent (a pre-1.0 index).
@@ -668,7 +956,8 @@ public class QuickSearchOpenSearchClient implements Closeable
try
{
Query finalQuery = buildSearchQuery(searchTerm, allowedTables, tableConfigs);
Highlight highlight = Highlight.of(h -> h.fields("searchableText", HighlightField.of(hf -> hf)));
Highlight highlight = allowedTables.stream().anyMatch(QuickSearchFieldRules::requiresFieldFiltering) ? null
: Highlight.of(h -> h.fields("searchableText", HighlightField.of(hf -> hf)));
SearchRequest request = SearchRequest.of(r -> r
.index(indexName)
@@ -690,36 +979,94 @@ public class QuickSearchOpenSearchClient implements Closeable
/*******************************************************************************
** Record ids of the best matches in each table, best first, for core record
** search. One _msearch request with one search per table (the same query and
** ranking as {@link #search}, restricted to that table), so every table gets
** its own top idsPerTable hits. Only sourceTable and recordId are fetched.
** One hit of a record search: the record id, its relevance score, and its
** highlight fragments by index field (e.g., "fieldValues.firstName"), with
** matches between {@link #HIGHLIGHT_PRE_TAG} and {@link #HIGHLIGHT_POST_TAG}.
*******************************************************************************/
public record RecordSearchHit(String recordId, Double score, Map<String, List<String>> highlights)
{
}
/*******************************************************************************
** The hits of a record search in one table, best first, and the table's
** total number of matches (counted up to {@link #MAX_RESULT_WINDOW}; null
** if unknown).
*******************************************************************************/
public record RecordSearchTableHits(List<RecordSearchHit> hits, Long totalHits)
{
}
/*******************************************************************************
** Record ids of the best matches in each table, best first - see
** {@link #searchRecordsPerTable}, which also returns scores, highlights and
** totals.
*******************************************************************************/
public Map<String, List<String>> searchRecordIdsPerTable(String searchTerm, List<String> tableNames, int idsPerTable, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
{
Map<String, List<String>> idsByTable = new LinkedHashMap<>();
for(Map.Entry<String, RecordSearchTableHits> entry : searchRecordsPerTable(searchTerm, tableNames, idsPerTable, 0, false, tableConfigs, timeoutMillis).entrySet())
{
idsByTable.put(entry.getKey(), entry.getValue().hits().stream().map(RecordSearchHit::recordId).toList());
}
return (idsByTable);
}
/*******************************************************************************
** The best matches in each table, best first, for core record search. One
** _msearch request with one search per table (the same query and ranking as
** {@link #search}, restricted to that table), so every table gets its own
** top hitsPerTable hits, after skipping offset. Only sourceTable and
** recordId are fetched, plus each hit's score, the table's total hits
** (counted up to MAX_RESULT_WINDOW) and - when highlight is set - one
** highlight fragment per matching fieldValues.* field.
**
** The whole request is bounded by timeoutMillis of wall-clock time (and the
** same server-side search timeout): when it runs out, the request is
** cancelled and a QException is thrown. A failed search for any table fails
** the call. Returns every requested table as a key, in the given order.
*******************************************************************************/
public Map<String, List<String>> searchRecordIdsPerTable(String searchTerm, List<String> tableNames, int idsPerTable, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
public Map<String, RecordSearchTableHits> searchRecordsPerTable(String searchTerm, List<String> tableNames, int hitsPerTable, int offset, boolean highlight, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
{
Map<String, List<String>> idsByTable = new LinkedHashMap<>();
Map<String, RecordSearchTableHits> hitsByTable = new LinkedHashMap<>();
if(tableNames == null || tableNames.isEmpty())
{
return (idsByTable);
return (hitsByTable);
}
int from = Math.max(0, Math.min(offset, MAX_RESULT_WINDOW - hitsPerTable));
List<RequestItem> searches = new ArrayList<>();
for(String tableName : tableNames)
{
Query query = buildSearchQuery(searchTerm, List.of(tableName), tableConfigs);
searches.add(RequestItem.of(ri -> ri
.header(h -> h.index(indexName))
.body(b -> b
.query(query)
.size(idsPerTable)
.trackTotalHits(t -> t.enabled(false))
.source(sc -> sc.filter(f -> f.includes("sourceTable", "recordId")))
.timeout(timeoutMillis + "ms"))));
.body(b ->
{
b.query(query)
.from(from)
.size(hitsPerTable)
.trackTotalHits(t -> t.count(MAX_RESULT_WINDOW))
.source(sc -> sc.filter(f -> f.includes("sourceTable", "recordId")))
.timeout(timeoutMillis + "ms");
if(highlight)
{
b.highlight(h -> h
.preTags(HIGHLIGHT_PRE_TAG)
.postTags(HIGHLIGHT_POST_TAG)
.requireFieldMatch(false)
.numberOfFragments(1)
.fragmentSize(HIGHLIGHT_FRAGMENT_SIZE)
.fields("fieldValues.*", HighlightField.of(hf -> hf)));
}
return (b);
})));
}
CompletableFuture<MsearchResponse<OpenSearchDocument>> future = null;
@@ -743,17 +1090,19 @@ public class QuickSearchOpenSearchClient implements Closeable
throw (new QException("Record search failed for table [" + tableNames.get(i) + "]: " + reason));
}
List<String> ids = new ArrayList<>();
List<RecordSearchHit> hits = new ArrayList<>();
for(Hit<OpenSearchDocument> hit : item.result().hits().hits())
{
if(hit.source() != null && hit.source().getRecordId() != null)
{
ids.add(hit.source().getRecordId());
hits.add(new RecordSearchHit(hit.source().getRecordId(), hit.score(), hit.highlight()));
}
}
idsByTable.put(tableNames.get(i), ids);
Long totalHits = item.result().hits().total() == null ? null : item.result().hits().total().value();
hitsByTable.put(tableNames.get(i), new RecordSearchTableHits(hits, totalHits));
}
return (idsByTable);
return (hitsByTable);
}
catch(TimeoutException e)
{
@@ -803,12 +1152,35 @@ public class QuickSearchOpenSearchClient implements Closeable
{
List<FieldValue> allowed = allowedTables.stream().map(FieldValue::of).toList();
Query snippetMatch = Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
{
mm.query(searchTerm).fields("searchableText").operator(Operator.And);
return (named ? mm.queryName(MATCH_CLAUSE_NAME) : mm);
})));
////////////////////////////////////////////////////////////////////////////
// fields kept out of the snippet (KEYWORD, EXACT, or includeInSnippet //
// off) match on their own; with none, the query is exactly as before //
////////////////////////////////////////////////////////////////////////////
List<String> aggregateTables = allowedTables.stream().filter(t -> !QuickSearchFieldRules.requiresFieldFiltering(t)).toList();
if(aggregateTables.isEmpty())
{
snippetMatch = Query.of(q -> q.matchNone(m -> m));
}
else if(aggregateTables.size() != allowedTables.size())
{
Query aggregateMatch = snippetMatch;
snippetMatch = Query.of(q -> q.bool(b -> b.must(aggregateMatch).filter(f -> f.terms(t -> t.field("sourceTable")
.terms(v -> v.value(aggregateTables.stream().map(FieldValue::of).toList()))))));
}
List<Query> ownMatches = buildOwnFieldMatchQueries(searchTerm, tableConfigs, allowedTables);
ownMatches.addAll(buildFilteredSnippetQueries(searchTerm, tableConfigs, allowedTables));
Query safeSnippetMatch = snippetMatch;
Query match = ownMatches.isEmpty() ? safeSnippetMatch
: Query.of(q -> q.bool(b -> b.should(safeSnippetMatch).should(ownMatches).minimumShouldMatch("1")));
BoolQuery.Builder boolBuilder = new BoolQuery.Builder()
.must(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
{
mm.query(searchTerm).fields("searchableText").operator(Operator.And);
return (named ? mm.queryName(MATCH_CLAUSE_NAME) : mm);
}))))
.must(match)
.filter(Query.of(q -> q.terms(t -> t.field("sourceTable").terms(tv -> tv.value(allowed)))))
.should(buildFieldBoostQueries(searchTerm, tableConfigs, allowedTables, named));
@@ -817,6 +1189,88 @@ public class QuickSearchOpenSearchClient implements Closeable
/*******************************************************************************
** For tables with excluded fields, match only attributable field values.
** Old combined text may still contain fields that became classified later.
*******************************************************************************/
private List<Query> buildFilteredSnippetQueries(String term, List<QuickSearchableTableConfig> configs, Collection<String> allowedTables)
{
List<Query> queries = new ArrayList<>();
for(QuickSearchableTableConfig config : configs == null ? List.<QuickSearchableTableConfig>of() : configs)
{
if(!allowedTables.contains(config.getTableName()) || !QuickSearchFieldRules.requiresFieldFiltering(config.getTableName()))
{
continue;
}
var table = QuickSearchFieldRules.lookupTable(config.getTableName());
List<String> fields = QuickSearchFieldRules.indexableFields(table, config.getSearchableFields()).stream()
.filter(config::isIncludedInSnippet).map(f -> PATH_PREFIX + "." + f).toList();
if(fields.isEmpty() || QuickSearchFieldRules.isProtected(table, table.getPrimaryKeyField()))
{
continue;
}
queries.add(Query.of(q -> q.bool(b -> b
.filter(f -> f.term(t -> t.field("sourceTable").value(FieldValue.of(config.getTableName()))))
.must(m -> m.multiMatch(mm -> mm.query(term).fields(fields).type(TextQueryType.CrossFields)
.operator(Operator.And).lenient(true))))));
}
return (queries);
}
/*******************************************************************************
** Matches for the fields that are not in searchableText, each limited to
** its own table: an EXACT field matches the whole (lower-cased) term, any
** other matches every term with its analyzer.
*******************************************************************************/
private List<Query> buildOwnFieldMatchQueries(String searchTerm, List<QuickSearchableTableConfig> tableConfigs, Collection<String> allowedTables)
{
List<Query> queries = new ArrayList<>();
for(QuickSearchableTableConfig tableConfig : tableConfigs == null ? List.<QuickSearchableTableConfig>of() : tableConfigs)
{
if(QuickSearchFieldRules.hasProtectedPrimaryKey(tableConfig.getTableName()) || !allowedTables.contains(tableConfig.getTableName()) || tableConfig.isAllPlainFields())
{
continue;
}
Query inTable = Query.of(q -> q.term(t -> t.field("sourceTable").value(FieldValue.of(tableConfig.getTableName()))));
for(String field : tableConfig.getSearchableFields() == null ? List.<String>of() : tableConfig.getSearchableFields())
{
if(tableConfig.isIncludedInSnippet(field)
|| (QuickSearchFieldRules.lookupTable(tableConfig.getTableName()) != null && !QuickSearchFieldRules.isIndexable(QuickSearchFieldRules.lookupTable(tableConfig.getTableName()), field)))
{
continue;
}
QuickSearchFieldAnalyzer analyzer = tableConfig.getAnalyzer(field);
Query fieldMatch = analyzer == QuickSearchFieldAnalyzer.EXACT
? Query.of(q -> q.term(t -> t.field(PATH_EXACT + "." + field).value(FieldValue.of(IndexingUtils.normalizeExact(searchTerm)))))
: Query.of(q -> q.multiMatch(mm -> mm.query(searchTerm).fields(fieldPath(analyzer) + "." + field).operator(Operator.And).lenient(true)));
queries.add(Query.of(q -> q.bool(b -> b.filter(inTable).must(fieldMatch))));
}
}
return (queries);
}
/*******************************************************************************
** The document object a field's analyzer writes it to.
*******************************************************************************/
static String fieldPath(QuickSearchFieldAnalyzer analyzer)
{
return (switch(analyzer == null ? QuickSearchFieldAnalyzer.PREFIX : analyzer)
{
case WORD -> PATH_WORD;
case ENGLISH -> PATH_ENGLISH;
case EXACT -> PATH_EXACT;
default -> PATH_PREFIX;
});
}
/*******************************************************************************
**
*******************************************************************************/
@@ -830,7 +1284,7 @@ public class QuickSearchOpenSearchClient implements Closeable
for(QuickSearchableTableConfig tableConfig : tableConfigs)
{
if(!allowedTables.contains(tableConfig.getTableName()))
if(QuickSearchFieldRules.hasProtectedPrimaryKey(tableConfig.getTableName()) || !allowedTables.contains(tableConfig.getTableName()))
{
continue;
}
@@ -839,11 +1293,28 @@ public class QuickSearchOpenSearchClient implements Closeable
List<String> fields = tableConfig.getSearchableFields() == null ? List.of() : tableConfig.getSearchableFields();
for(String field : fields)
{
float boost = (weights == null || weights.get(field) == null) ? 1f : weights.get(field).floatValue();
if(QuickSearchFieldRules.lookupTable(tableConfig.getTableName()) != null && !QuickSearchFieldRules.isIndexable(QuickSearchFieldRules.lookupTable(tableConfig.getTableName()), field))
{
continue;
}
float boost = (weights == null || weights.get(field) == null) ? 1f : weights.get(field).floatValue();
QuickSearchFieldAnalyzer analyzer = tableConfig.getAnalyzer(field);
if(analyzer == QuickSearchFieldAnalyzer.EXACT)
{
String exact = IndexingUtils.normalizeExact(searchTerm);
queries.add(Query.of(q -> q.term(t ->
{
t.field(PATH_EXACT + "." + field).value(FieldValue.of(exact)).boost(boost);
return (named ? t.queryName(FIELD_CLAUSE_PREFIX + field) : t);
})));
continue;
}
String path = fieldPath(analyzer) + "." + field;
queries.add(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
{
mm.query(searchTerm)
.fields("fieldValues." + field)
.fields(path)
.operator(Operator.And)
.lenient(true)
.boost(boost);
@@ -28,6 +28,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -48,7 +50,10 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
@@ -71,9 +76,14 @@ public abstract class AbstractIndexingStep implements BackendStep
public static final String STATUS_ACTIVE = "ACTIVE";
public static final String STATUS_NEEDS_REINDEX = "NEEDS_REINDEX";
public static final String STATUS_REBUILDING = "REBUILDING";
public static final String STATUS_PENDING_BUILD = "PENDING_BUILD";
public static final String STATUS_BUILDING = "BUILDING";
public static final String RUN_COMPLETED = "COMPLETED";
public static final String RUN_FAILED = "FAILED";
public static final String RUN_RUNNING = "RUNNING";
public static final String RUN_CANCELLED = "CANCELLED";
private String cancelledBy = "an admin";
@@ -122,7 +132,7 @@ public abstract class AbstractIndexingStep implements BackendStep
protected List<QuickSearchableTableConfig> getDiscoveredTables()
{
List<QuickSearchableTableConfig> tables = QuickSearchQBitContext.getDiscoveredTables();
return (tables == null ? Collections.emptyList() : tables);
return (tables == null ? Collections.emptyList() : tables.stream().filter(t -> !QuickSearchFieldRules.hasProtectedPrimaryKey(t.getTableName())).toList());
}
@@ -132,13 +142,14 @@ public abstract class AbstractIndexingStep implements BackendStep
*******************************************************************************/
protected QuickSearchableTableConfig getTableConfig(String tableName)
{
return (QuickSearchQBitContext.getTableConfig(tableName));
return (QuickSearchFieldRules.hasProtectedPrimaryKey(tableName) ? null : QuickSearchQBitContext.getTableConfig(tableName));
}
/*******************************************************************************
** Make sure a quickSearchIndex row exists for the table. The table's unique
** Make sure a quick_search_index row exists for the table (hosts without
** the config tables only; see below). The table's unique
** key on tableName makes a concurrent first insert fail, which is handled by
** re-querying.
*******************************************************************************/
@@ -149,6 +160,17 @@ public abstract class AbstractIndexingStep implements BackendStep
return;
}
///////////////////////////////////////////////////////////////////////////
// with the config tables, rows come only from priming (which honors //
// tombstones) and Add Search Index: never re-create a removed table //
// here, even while priming has not succeeded and code is in use //
///////////////////////////////////////////////////////////////////////////
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime != null && runtime.getConfigStore().hasConfigTables())
{
return;
}
try
{
QRecord record = new QRecord()
@@ -163,7 +185,7 @@ public abstract class AbstractIndexingStep implements BackendStep
InsertInput insertInput = new InsertInput();
insertInput.setTableName(getConfig().getQuickSearchIndexTableName());
insertInput.setRecords(List.of(record));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
InsertOutput insertOutput = QuickSearchConfigWrites.asSystem(() -> new InsertAction().execute(insertInput));
if(insertOutput.getRecords().isEmpty() || CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
{
@@ -183,7 +205,7 @@ public abstract class AbstractIndexingStep implements BackendStep
/*******************************************************************************
** The quickSearchIndex row for a table, or null.
** The quick_search_index row for a table, or null.
*******************************************************************************/
protected QRecord queryIndexRow(String tableName) throws QException
{
@@ -197,7 +219,7 @@ public abstract class AbstractIndexingStep implements BackendStep
/*******************************************************************************
** The id of the quickSearchIndex row for a table.
** The id of the quick_search_index row for a table.
*******************************************************************************/
protected Integer queryIndexId(String tableName) throws QException
{
@@ -285,7 +307,7 @@ public abstract class AbstractIndexingStep implements BackendStep
/*******************************************************************************
** JSON array describing a table's searchable fields, weights and labels.
*******************************************************************************/
protected static String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
public static String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
{
if(tableConfig.getSearchableFields() == null || tableConfig.getSearchableFields().isEmpty())
{
@@ -306,13 +328,44 @@ public abstract class AbstractIndexingStep implements BackendStep
Boolean includeLabel = tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName) ? tableConfig.getFieldIncludeLabels().get(fieldName) : false;
String escapedName = fieldName.replace("\\", "\\\\").replace("\"", "\\\"");
sb.append("{\"fieldName\":\"").append(escapedName).append("\",\"weight\":").append(weight).append(",\"includeLabel\":").append(includeLabel).append("}");
sb.append("{\"fieldName\":\"").append(escapedName).append("\",\"weight\":").append(weight).append(",\"includeLabel\":").append(includeLabel);
///////////////////////////////////////////////////////////////////
// roles, analyzers and snippet flags only when set, so a table //
// of plain fields keeps the JSON it had before they existed //
///////////////////////////////////////////////////////////////////
appendIfSet(sb, "role", tableConfig.getFieldRoles(), fieldName);
appendIfSet(sb, "analyzer", tableConfig.getFieldAnalyzers(), fieldName);
appendIfSet(sb, "includeInSnippet", tableConfig.getFieldIncludeInSnippet(), fieldName);
sb.append("}");
}
return (sb.append("]").toString());
}
/*******************************************************************************
** Append ,"key":value for a field's entry in the map, when it has one.
*******************************************************************************/
private static void appendIfSet(StringBuilder sb, String key, Map<String, ?> values, String fieldName)
{
Object value = values == null ? null : values.get(fieldName);
if(value != null)
{
sb.append(",\"").append(key).append("\":");
if(value instanceof Boolean)
{
sb.append(value);
}
else
{
sb.append("\"").append(String.valueOf(value).replace("\\", "\\\\").replace("\"", "\\\"")).append("\"");
}
}
}
/*******************************************************************************
** Insert a RUNNING run record and return it with its id.
*******************************************************************************/
@@ -329,7 +382,7 @@ public abstract class AbstractIndexingStep implements BackendStep
InsertOutput insertOutput = new InsertAction().execute(insertInput);
if(insertOutput.getRecords().isEmpty())
{
throw new QException("Failed to create quickSearchIndexRun record: no record returned from insert");
throw new QException("Failed to create quick_search_index_run record: no record returned from insert");
}
QRecord inserted = insertOutput.getRecords().get(0);
@@ -440,7 +493,10 @@ public abstract class AbstractIndexingStep implements BackendStep
/*******************************************************************************
** Update fields on a quickSearchIndex row.
** Update runtime state on a quick_search_index row (status, last run, last
** basepull time, document count). Indexing bookkeeping, not a configuration
** change: it is not record-audited, or every basepull tick would write an
** audit row. Admin edits to the row go through the table and stay audited.
*******************************************************************************/
protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
{
@@ -453,7 +509,8 @@ public abstract class AbstractIndexingStep implements BackendStep
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
updateInput.setRecords(List.of(record));
new UpdateAction().execute(updateInput);
updateInput.setOmitDmlAudit(true);
QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput));
}
@@ -557,6 +614,7 @@ public abstract class AbstractIndexingStep implements BackendStep
}
heartbeat(run);
stopIfCancelledOrPaused(run, new IndexCounts(processed, indexed, errors, skipped, firstError));
}
return (new IndexCounts(processed, indexed, errors, skipped, firstError));
@@ -564,6 +622,122 @@ public abstract class AbstractIndexingStep implements BackendStep
/*******************************************************************************
** The table a reindex or reconcile is limited to, or null for all:
** searchTableName (the input screen's field), else tableName (API
** callers). A Quick Search table is never meant: a process run from the
** Quick Search Index table gets that table as its tableName.
*******************************************************************************/
protected String tableNameInput(RunBackendStepInput input)
{
String tableName = input.getValueString("searchTableName");
if(tableName == null || tableName.isBlank())
{
tableName = input.getValueString("tableName");
}
QuickSearchQBitConfig config = QuickSearchQBitContext.getConfig();
if(tableName == null || tableName.isBlank() || (config != null && QuickSearchFieldRules.isQuickSearchTable(config, tableName)))
{
return (null);
}
return (tableName);
}
/*******************************************************************************
** When an admin has paused all indexing: say so in the output (the result
** screen's message) and return true, so the step does nothing.
*******************************************************************************/
protected boolean isPausedSkip(RunBackendStepOutput output)
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime == null || !runtime.isIndexingPaused())
{
return (false);
}
LOG.info("Quick Search indexing is paused; " + getClass().getSimpleName() + " skipped");
output.addValue("message", "Quick Search indexing is paused; nothing was indexed. Resume indexing from Quick Search Settings first.");
output.addValue("paused", true);
return (true);
}
/*******************************************************************************
** Between batches: stop the run, with its counts so far, when an admin
** asked to cancel it or paused all indexing.
*******************************************************************************/
protected void stopIfCancelledOrPaused(QuickSearchIndexRun run, IndexCounts counts) throws QuickSearchRunCancelledException
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime != null && runtime.isIndexingPaused())
{
throw (new QuickSearchRunCancelledException("Stopped: Quick Search indexing was paused", counts));
}
if(isCancelRequested(run))
{
throw (new QuickSearchRunCancelledException("Cancelled by " + cancelledBy, counts));
}
}
/*******************************************************************************
** Whether the run's cancel flag is set (read from its record, so a cancel
** made on another server is seen). A read failure is not a cancel.
*******************************************************************************/
protected boolean isCancelRequested(QuickSearchIndexRun run)
{
if(run == null || run.getId() == null)
{
return (false);
}
return (isAnyCancelRequested(List.of(run.getId())));
}
/*******************************************************************************
** Whether any of the runs' cancel flags is set; remembers who asked.
*******************************************************************************/
protected boolean isAnyCancelRequested(List<Integer> runIds)
{
try
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, new ArrayList<>(runIds)))
.withCriteria(new QFilterCriteria("cancelRequested", QCriteriaOperator.EQUALS, true)));
List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
if(rows.isEmpty())
{
return (false);
}
cancelledBy = rows.get(0).getValueString("cancelRequestedBy") == null ? "an admin" : rows.get(0).getValueString("cancelRequestedBy");
return (true);
}
catch(Exception e)
{
LOG.debug("Could not read the cancel flag of a run", e);
return (false);
}
}
/*******************************************************************************
** End a run that was cancelled (or stopped by a pause): CANCELLED with its
** counts so far. Never throws.
*******************************************************************************/
protected void completeCancelledRun(QuickSearchIndexRun run, QuickSearchRunCancelledException e)
{
IndexCounts counts = e.getCounts();
completeRunRecord(run, RUN_CANCELLED, counts == null ? 0 : counts.processed(), counts == null ? 0 : counts.indexed(), counts == null ? 0 : counts.errors(), e.getMessage());
}
/*******************************************************************************
** Delete run records older than runHistoryRetentionDays. Never throws.
*******************************************************************************/
@@ -577,8 +751,8 @@ public abstract class AbstractIndexingStep implements BackendStep
Instant cutoff = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
purgeOlderThan(getConfigOrNull().getQuickSearchIndexRunTableName(), "startTime", cutoff, retentionDays);
purgeOlderThan(getConfigOrNull().applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME), "createDate", cutoff, retentionDays);
purgeOlderThan(getConfigOrNull().applyPrefix(QuickSearchValidationRun.TABLE_NAME), "createDate", cutoff, retentionDays);
purgeOlderThan(getConfigOrNull().applyTablePrefix(QuickSearchDiagnosticEvent.TABLE_NAME), "createDate", cutoff, retentionDays);
purgeOlderThan(getConfigOrNull().applyTablePrefix(QuickSearchValidationRun.TABLE_NAME), "createDate", cutoff, retentionDays);
}
@@ -43,7 +43,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
@@ -61,7 +64,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** are replayed in the same run) and replays failed real-time events, and
** afterwards purges old run history.
**
** The watermark stored on the quickSearchIndex row is the run's start time,
** The watermark stored on the quick_search_index row is the run's start time,
** captured before the first query, so rows modified during the run are read
** again next time (re-indexing is idempotent). A table without a timestamp
** field is left to the reconcile process.
@@ -84,11 +87,25 @@ public class BasepullIndexStep extends AbstractIndexingStep
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
List<String> failures = new ArrayList<>();
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime != null)
{
runtime.getConfigStore().tryPrime();
if(runtime.isIndexingPaused())
{
LOG.info("Quick Search indexing is paused; basepull skipped");
output.addValue("tablesIndexed", 0);
output.addValue("paused", true);
return;
}
}
List<String> failures = new ArrayList<>();
List<QRecord> liveRuns = List.of();
try
{
new FullReindexStep().recoverDeadRun(true);
liveRuns = new FullReindexStep().recoverDeadRun(true);
}
catch(Exception e)
{
@@ -96,6 +113,16 @@ public class BasepullIndexStep extends AbstractIndexingStep
failures.add("full reindex recovery: " + e.getMessage());
}
try
{
queueBuildsIfIndexMissingOrEmpty();
}
catch(Exception e)
{
LOG.warn("Could not check whether the search index is missing or empty", e);
failures.add("index check: " + e.getMessage());
}
try
{
replayFailedEvents(output);
@@ -118,14 +145,38 @@ public class BasepullIndexStep extends AbstractIndexingStep
if(CollectionUtils.nullSafeIsEmpty(indexRecords))
{
LOG.info("No enabled quickSearchIndex rows found; nothing to basepull");
LOG.info("No enabled quick_search_index rows found; nothing to basepull");
purgeOldRuns();
return;
}
Integer tablesIndexed = 0;
Integer tablesBuilt = 0;
for(QRecord indexRecord : indexRecords)
{
////////////////////////////////////////////////////////////////////////
// self-priming: a table queued for a build (new, changed by code, or //
// the index was missing) is built in place instead of basepulled; a //
// BUILDING row with no live run was left by a stopped server //
////////////////////////////////////////////////////////////////////////
String status = indexRecord.getValueString("status");
if(STATUS_PENDING_BUILD.equals(status) || (STATUS_BUILDING.equals(status) && CollectionUtils.nullSafeIsEmpty(liveRuns)))
{
try
{
if(buildTable(indexRecord, runtime))
{
tablesBuilt++;
}
}
catch(Exception e)
{
LOG.warn("Building a queued search index failed; it is retried after its update interval", e, logPair("tableName", indexRecord.getValueString("tableName")));
failures.add(indexRecord.getValueString("tableName") + " build: " + e.getMessage());
}
continue;
}
QuickSearchIndex index = new QuickSearchIndex()
.withId(indexRecord.getValueInteger("id"))
.withTableName(indexRecord.getValueString("tableName"))
@@ -157,6 +208,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
purgeOldRuns();
output.addValue("tablesIndexed", tablesIndexed);
output.addValue("tablesBuilt", tablesBuilt);
if(!failures.isEmpty())
{
@@ -166,6 +218,148 @@ public class BasepullIndexStep extends AbstractIndexingStep
/*******************************************************************************
** Build one queued table in place (the reconcile algorithm, recorded as a
** FULL_REINDEX run): BUILDING while it runs, ACTIVE after. A failure
** queues it again, retried once its update interval has passed since the
** failure; a cancel leaves it NEEDS_REINDEX (Apply Search Index Changes
** starts it again); a pause leaves it queued for when indexing resumes.
** Returns false when it was skipped (backing off after a failure).
*******************************************************************************/
public boolean buildTable(QRecord row, QuickSearchRuntime runtime) throws QException
{
String tableName = row.getValueString("tableName");
Integer indexId = row.getValueInteger("id");
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
if(tableConfig == null)
{
throw (new QException("Table is no longer configured for Quick Search"));
}
Integer intervalMinutes = row.getValueInteger("basepullIntervalMinutes");
long backoffMillis = Math.max(1, intervalMinutes == null ? 5 : intervalMinutes) * 60_000L;
if(runtime != null && System.currentTimeMillis() - runtime.getLastBuildFailureMillis(tableName) < backoffMillis)
{
return (false);
}
updateIndexRow(indexId, Map.of("status", STATUS_BUILDING));
try
{
new ReconcileIndexStep().reconcileTable(tableConfig, FullReindexStep.RUN_TYPE);
if(runtime != null)
{
runtime.clearBuildFailure(tableName);
}
LOG.info("Built queued search index", logPair("tableName", tableName));
return (true);
}
catch(QuickSearchRunCancelledException e)
{
boolean paused = runtime != null && runtime.isIndexingPaused();
updateIndexRow(indexId, Map.of("status", paused ? STATUS_PENDING_BUILD : STATUS_NEEDS_REINDEX));
LOG.info("Queued search index build stopped: " + e.getMessage(), logPair("tableName", tableName));
return (false);
}
catch(Exception e)
{
updateIndexRow(indexId, Map.of("status", STATUS_PENDING_BUILD));
if(runtime != null)
{
runtime.recordBuildFailure(tableName);
}
throw (e instanceof QException qe ? qe : new QException("Build failed: " + e.getMessage(), e));
}
}
/*******************************************************************************
** Self-priming of the index itself: when the alias is gone (a new cluster,
** or the index was deleted) it is created again. When the index was just
** created, every enabled table that had documents in the lost index (and
** has not been fully rebuilt since) is queued for a build; when it is
** empty while a table last counted documents in it, every enabled table is.
*******************************************************************************/
void queueBuildsIfIndexMissingOrEmpty() throws QException
{
QuickSearchOpenSearchClient client = getClient();
String reason = null;
if(!client.indexExists())
{
client.ensureIndexExists();
}
List<QRecord> rows = queryEnabledRows();
if(client.wasIndexCreated())
{
///////////////////////////////////////////////////////////////////////
// only tables that had documents in the lost index and have not been //
// fully rebuilt since: a table never basepulled reads everything on //
// its first basepull anyway //
///////////////////////////////////////////////////////////////////////
Instant createdAt = client.getIndexCreatedAt() == null ? Instant.EPOCH : client.getIndexCreatedAt();
rows = rows.stream().filter(r -> r.getValue("lastBasepullTime") != null
&& !isAfter(r.getValueInstant("lastFullReindexTime"), createdAt) && !isAfter(r.getValueInstant("lastReconcileTime"), createdAt)).toList();
reason = "the search index was missing and has been created";
}
else
{
Long documents = client.countAllDocuments();
if(documents != null && documents == 0 && rows.stream().anyMatch(r -> r.getValueInteger("documentCount") != null && r.getValueInteger("documentCount") > 0))
{
reason = "the search index is empty";
}
}
if(reason == null)
{
return;
}
int queued = 0;
for(QRecord row : rows)
{
String status = row.getValueString("status");
if(!STATUS_PENDING_BUILD.equals(status) && !STATUS_BUILDING.equals(status) && !STATUS_REBUILDING.equals(status))
{
QuickSearchConfigWrites.asSystem(() ->
{
updateIndexRow(row.getValueInteger("id"), Map.of("status", STATUS_PENDING_BUILD));
return (null);
});
queued++;
}
}
client.clearIndexCreated();
LOG.info("Queued a build of every enabled table: " + reason, logPair("tablesQueued", queued));
}
/*******************************************************************************
** Whether a time is set and after another.
*******************************************************************************/
private static boolean isAfter(Instant time, Instant other)
{
return (time != null && time.isAfter(other));
}
/*******************************************************************************
** The enabled quick_search_index rows.
*******************************************************************************/
private List<QRecord> queryEnabledRows() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)));
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
}
/*******************************************************************************
** Due when never run, or when the interval has elapsed since the last run.
*******************************************************************************/
@@ -196,7 +390,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
if(tableConfig == null)
{
String message = "Table is no longer configured for Quick Search; disable or remove its quickSearchIndex row";
String message = "Table is no longer configured for Quick Search; disable or remove its quick_search_index row";
completeRunRecord(run, RUN_FAILED, 0, 0, 1, message);
throw (new QException(tableName + ": " + message));
}
@@ -230,6 +424,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
{
Map<String, Serializable> values = new HashMap<>();
values.put("lastBasepullTime", runStart);
values.put("realTimeErrorCount", 0);
Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
if(documentCount != null)
{
@@ -249,6 +444,11 @@ public class BasepullIndexStep extends AbstractIndexingStep
throw (new QException("Basepull indexed " + counts.indexed() + " of " + counts.processed() + " records for table [" + tableName + "]; first error: " + counts.firstError()));
}
}
catch(QuickSearchRunCancelledException e)
{
completeCancelledRun(run, e);
throw e;
}
catch(QException e)
{
if(!RUN_FAILED.equals(run.getStatus()))
@@ -277,7 +477,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
void replayFailedEvents(RunBackendStepOutput output) throws QException
{
QuickSearchQBitConfig config = getConfig();
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
String failedEventTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(failedEventTable);
@@ -307,6 +507,11 @@ public class BasepullIndexStep extends AbstractIndexingStep
for(Map.Entry<String, List<QRecord>> entry : deleteByTable.entrySet())
{
if(QuickSearchFieldRules.hasProtectedPrimaryKey(entry.getKey()))
{
markReplayOutcome(entry.getValue(), false, List.of("The primary key cannot be indexed"), succeeded, failed);
continue;
}
List<QRecord> toDelete = entry.getValue();
try
{
@@ -59,7 +59,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Rebuild the index from source.
**
** Without a "tableName" input, every enabled table is indexed into a fresh
** Without a "searchTableName" (or "tableName") input, every enabled table is indexed into a fresh
** physical index with the current mapping; when all of them succeed the alias
** is swapped to it and the old index is deleted, so search keeps serving the
** old documents until the new set is complete. This is also how an index
@@ -69,7 +69,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** While the new index is built, real-time writes still go through the alias
** to the old one. Updates are caught by the next basepull (lastBasepullTime
** is set to each table's start). Deletes are caught here: each table's
** quickSearchIndex row is REBUILDING for the whole rebuild, which makes the
** quick_search_index row is REBUILDING for the whole rebuild, which makes the
** record change listener (on any node) also record each delete as an
** AWAITING_REINDEX failed event. Once the rebuild's own writes are done and
** the alias is swapped, those deletes are applied through the alias.
@@ -82,7 +82,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** (see recoverDeadRun); a full reindex then refuses to start while another
** one is alive.
**
** With a "tableName" input, that table is rebuilt in place with the reconcile
** With a "searchTableName" (or "tableName") input, that table is rebuilt in place with the reconcile
** algorithm (index everything, then delete what was not re-indexed).
*******************************************************************************/
public class FullReindexStep extends AbstractIndexingStep
@@ -91,6 +91,9 @@ public class FullReindexStep extends AbstractIndexingStep
public static final String RUN_TYPE = "FULL_REINDEX";
private List<Integer> cancelGroup;
private QuickSearchIndexRun runOfCurrentTable;
/*******************************************************************************
@@ -99,7 +102,14 @@ public class FullReindexStep extends AbstractIndexingStep
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
String tableNameFilter = input.getValueString("tableName");
if(isPausedSkip(output))
{
output.addValue("recordsIndexed", 0);
output.addValue("aliasSwapped", false);
return;
}
String tableNameFilter = tableNameInput(input);
boolean allTables = tableNameFilter == null || tableNameFilter.isBlank();
//////////////////////////////////////////////////////////////////////////////
@@ -143,6 +153,22 @@ public class FullReindexStep extends AbstractIndexingStep
/*******************************************************************************
** An all-tables rebuild is one run per table; cancelling any of them
** cancels the rebuild (the old index stays in service).
*******************************************************************************/
@Override
protected boolean isCancelRequested(QuickSearchIndexRun run)
{
if(cancelGroup != null && !cancelGroup.isEmpty())
{
return (isAnyCancelRequested(cancelGroup));
}
return (super.isCancelRequested(run));
}
/*******************************************************************************
** Index every table into a new physical index, then swap the alias.
*******************************************************************************/
@@ -181,11 +207,14 @@ public class FullReindexStep extends AbstractIndexingStep
marked = true;
markRebuilding(tables, priorStatus);
cancelGroup = new ArrayList<>();
runs.values().forEach(r -> cancelGroup.add(r.getId()));
for(QuickSearchableTableConfig tableConfig : tables)
{
String tableName = tableConfig.getTableName();
startTimes.put(tableName, Instant.now());
runOfCurrentTable = runs.get(tableConfig);
IndexCounts tableCounts = indexAllRecords(client, tableConfig, List.of(), physicalIndex, runs.get(tableConfig));
counts.put(tableName, tableCounts);
indexed = indexed + tableCounts.indexed();
@@ -210,6 +239,7 @@ public class FullReindexStep extends AbstractIndexingStep
Map<String, Serializable> values = new HashMap<>();
values.put("lastFullReindexTime", Instant.now());
values.put("realTimeErrorCount", 0);
values.put("lastBasepullTime", startTimes.get(tableName));
values.put("recordCount", tableCounts.processed());
Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
@@ -273,6 +303,13 @@ public class FullReindexStep extends AbstractIndexingStep
for(Map.Entry<QuickSearchableTableConfig, QuickSearchIndexRun> entry : runs.entrySet())
{
IndexCounts tableCounts = counts.get(entry.getKey().getTableName());
if(e instanceof QuickSearchRunCancelledException cancelled)
{
IndexCounts stopped = tableCounts != null ? tableCounts : (entry.getValue() == runOfCurrentTable ? cancelled.getCounts() : null);
completeRunRecord(entry.getValue(), RUN_CANCELLED, stopped == null ? 0 : stopped.processed(), stopped == null ? 0 : stopped.indexed(),
stopped == null ? 0 : stopped.errors(), e.getMessage() + "; the previous index stays in service");
continue;
}
completeRunRecord(entry.getValue(), RUN_FAILED,
tableCounts == null ? 0 : tableCounts.processed(),
tableCounts == null ? 0 : tableCounts.indexed(),
@@ -479,7 +516,7 @@ public class FullReindexStep extends AbstractIndexingStep
private void handCapturedDeletesToBasepull(Serializable lastCapturedId) throws QException
{
QuickSearchQBitConfig config = getConfig();
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
String failedEventTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
Serializable afterId = null;
Integer handedOver = 0;
@@ -646,7 +683,7 @@ public class FullReindexStep extends AbstractIndexingStep
/*******************************************************************************
** Every quickSearchIndex row that is REBUILDING.
** Every quick_search_index row that is REBUILDING.
*******************************************************************************/
private List<QRecord> queryRebuildingIndexRows() throws QException
{
@@ -665,7 +702,7 @@ public class FullReindexStep extends AbstractIndexingStep
private Serializable queryLastCapturedDeleteId() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
queryInput.setTableName(getConfig().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME));
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX))
.withOrderBy(new QFilterOrderBy("id", false))
.withLimit(1));
@@ -757,7 +794,7 @@ public class FullReindexStep extends AbstractIndexingStep
try
{
QuickSearchQBitConfig config = getConfig();
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
String failedEventTable = config.applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
Serializable lastCapturedId = queryLastCapturedDeleteId();
Serializable afterId = null;
@@ -18,9 +18,12 @@ package com.kingsrook.qbits.quicksearch.processes;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext;
@@ -30,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFieldRole;
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -46,6 +51,8 @@ public class IndexingUtils
{
private static final QLogger LOG = QLogger.getLogger(IndexingUtils.class);
static final int MAX_ROLE_VALUE_LENGTH = 500;
/*******************************************************************************
@@ -97,6 +104,10 @@ public class IndexingUtils
for(String fieldName : fields)
{
if(table != null && !QuickSearchFieldRules.isIndexable(table, fieldName))
{
continue;
}
String value = displayValue(record, fieldName, maxFieldLength);
if(value == null)
{
@@ -151,7 +162,7 @@ public class IndexingUtils
*******************************************************************************/
public static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List<String> fields, Map<String, Integer> fieldWeights, Map<String, Boolean> includeLabels)
{
return (buildDocument(record, tableName, primaryKeyField, fields, includeLabels, lookupTable(tableName), null, null));
return (buildDocument(record, tableName, primaryKeyField, fields, includeLabels, lookupTable(tableName), null, null, null));
}
@@ -159,30 +170,149 @@ public class IndexingUtils
/*******************************************************************************
** Build a document for a configured table: display values, label fallback,
** truncation, and an external version from the basepull timestamp field.
** Returns null when the record has no primary key.
** Returns null when metadata is unavailable or the primary key is missing
** or excluded by the field policy.
*******************************************************************************/
public static OpenSearchDocument buildDocument(QRecord record, QuickSearchableTableConfig tableConfig)
{
QTableMetaData table = lookupTable(tableConfig.getTableName());
OpenSearchDocument doc = buildDocument(record, tableConfig.getTableName(), tableConfig.getPrimaryKeyField(), tableConfig.getSearchableFields(),
tableConfig.getFieldIncludeLabels(), table, tableConfig.getMaxFieldLength(), tableConfig.getBasepullTimestampField());
tableConfig.getFieldIncludeLabels(), table, tableConfig.getMaxFieldLength(), tableConfig.getBasepullTimestampField(), tableConfig.getRecordLabelFields());
if(doc != null && tableConfig.getRecordLabelFormat() != null && tableConfig.getRecordLabelFields() != null)
{
doc.withRecordLabel(formatLabel(record, tableConfig, doc.getRecordLabel()));
}
if(doc != null)
{
applyRolesAndAnalyzers(doc, record, tableConfig, table);
}
return (doc);
}
/*******************************************************************************
** Apply the fields' roles and analyzers (from quick_search_index_field):
** only snippet fields go into searchableText; WORD, ENGLISH and EXACT
** fields are also written to fieldValuesWord, fieldValuesEnglish and
** fieldValuesExact (lower-cased) for their analyzers; TITLE fields make the
** record label, SUBTITLE fields the subtitle and DISPLAY fields the
** displayValues (by field label). A table whose fields are all plain BODY
** fields is left exactly as before.
*******************************************************************************/
static void applyRolesAndAnalyzers(OpenSearchDocument doc, QRecord record, QuickSearchableTableConfig tableConfig, QTableMetaData table)
{
List<String> fields = table == null ? List.of() : QuickSearchFieldRules.indexableFields(table, tableConfig.getSearchableFields());
if(!tableConfig.isAllPlainFields())
{
List<String> snippetFields = new ArrayList<>();
for(String fieldName : fields)
{
if(tableConfig.isIncludedInSnippet(fieldName))
{
snippetFields.add(fieldName);
}
}
doc.withSearchableText(buildSearchableText(record, snippetFields, tableConfig.getFieldIncludeLabels(), table, tableConfig.getMaxFieldLength()));
Map<String, Object> word = new LinkedHashMap<>();
Map<String, Object> english = new LinkedHashMap<>();
Map<String, Object> exact = new LinkedHashMap<>();
for(String fieldName : fields)
{
String value = displayValue(record, fieldName, tableConfig.getMaxFieldLength());
if(value == null)
{
continue;
}
switch(tableConfig.getAnalyzer(fieldName))
{
case WORD -> word.put(fieldName, value);
case ENGLISH -> english.put(fieldName, value);
case EXACT -> exact.put(fieldName, normalizeExact(value));
default ->
{
// PREFIX: fieldValues only
}
}
}
doc.withFieldValuesWord(word.isEmpty() ? null : word);
doc.withFieldValuesEnglish(english.isEmpty() ? null : english);
doc.withFieldValuesExact(exact.isEmpty() ? null : exact);
}
List<String> titles = new ArrayList<>();
List<String> subtitles = new ArrayList<>();
Map<String, String> display = new LinkedHashMap<>();
for(String fieldName : fields)
{
QuickSearchFieldRole role = tableConfig.getRole(fieldName);
if(role != QuickSearchFieldRole.TITLE && role != QuickSearchFieldRole.SUBTITLE && role != QuickSearchFieldRole.DISPLAY)
{
continue;
}
String value = displayValue(record, fieldName, MAX_ROLE_VALUE_LENGTH);
if(value == null)
{
continue;
}
switch(role)
{
case TITLE -> titles.add(value);
case SUBTITLE -> subtitles.add(value);
default -> display.put(fieldLabel(table, fieldName), value);
}
}
if(!titles.isEmpty())
{
doc.withRecordLabel(String.join(" ", titles));
}
doc.withSubtitle(subtitles.isEmpty() ? null : String.join(" - ", subtitles));
doc.withDisplayValues(display.isEmpty() ? null : display);
}
/*******************************************************************************
** The form an EXACT value is stored and matched in: trimmed, whitespace
** collapsed, lower-cased.
*******************************************************************************/
public static String normalizeExact(String value)
{
return (normalizeSearchText(value).toLowerCase(Locale.ROOT));
}
/*******************************************************************************
** A field's label, or its name.
*******************************************************************************/
private static String fieldLabel(QTableMetaData table, String fieldName)
{
if(table != null && table.getFields().containsKey(fieldName) && StringUtils.hasContent(table.getField(fieldName).getLabel()))
{
return (table.getField(fieldName).getLabel());
}
return (fieldName);
}
/*******************************************************************************
**
*******************************************************************************/
private static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List<String> fields, Map<String, Boolean> includeLabels,
QTableMetaData table, Integer maxFieldLength, String timestampField)
QTableMetaData table, Integer maxFieldLength, String timestampField, List<String> recordLabelFields)
{
if(table == null || primaryKeyField == null || table.getField(primaryKeyField) == null || QuickSearchFieldRules.isProtected(table, primaryKeyField))
{
QuickSearchFieldRules.warnExcludedFields(table);
return (null);
}
fields = QuickSearchFieldRules.indexableFields(table, fields);
String recordId = record.getValueString(primaryKeyField);
if(recordId == null)
{
@@ -200,7 +330,15 @@ public class IndexingUtils
}
String recordLabel = record.getRecordLabel();
if(!StringUtils.hasContent(recordLabel) && table != null)
boolean protectedLabel = table.getRecordLabelFields() != null
&& table.getRecordLabelFields().stream().anyMatch(f -> QuickSearchFieldRules.isProtected(table, f));
if(protectedLabel || (recordLabelFields != null && recordLabelFields.stream().anyMatch(f -> QuickSearchFieldRules.isProtected(table, f))))
{
recordLabel = null;
record = new QRecord(record);
record.setRecordLabel(null);
}
if(!StringUtils.hasContent(recordLabel) && !protectedLabel && table.getRecordLabelFields() != null)
{
try
{
@@ -213,7 +351,7 @@ public class IndexingUtils
}
Long version = null;
if(timestampField != null)
if(timestampField != null && !QuickSearchFieldRules.isProtected(table, timestampField))
{
try
{
@@ -243,6 +381,12 @@ public class IndexingUtils
*******************************************************************************/
private static String formatLabel(QRecord record, QuickSearchableTableConfig tableConfig, String fallback)
{
QTableMetaData table = lookupTable(tableConfig.getTableName());
if(tableConfig.getRecordLabelFields().stream().anyMatch(f -> QuickSearchFieldRules.isProtected(table, f)))
{
QuickSearchFieldRules.warnExcludedFields(table);
return (fallback);
}
try
{
Object[] labelValues = new Object[tableConfig.getRecordLabelFields().size()];
@@ -255,13 +399,42 @@ public class IndexingUtils
}
catch(Exception e)
{
LOG.warn("recordLabelFormat could not be applied; using default label", e, logPair("tableName", tableConfig.getTableName()));
LOG.warn("recordLabelFormat could not be applied; using default label", logPair("tableName", tableConfig.getTableName()));
return (fallback);
}
}
/*******************************************************************************
** Rebuild result presentation from allowed field values, discarding old
** aggregate labels, snippets and display values that may contain secrets.
*******************************************************************************/
public static OpenSearchDocument filterResult(OpenSearchDocument source, QuickSearchableTableConfig tableConfig)
{
if(!QuickSearchFieldRules.requiresFieldFiltering(source.getSourceTable()))
{
return (source);
}
QTableMetaData table = lookupTable(source.getSourceTable());
if(tableConfig == null || QuickSearchFieldRules.isProtected(table, table.getPrimaryKeyField()))
{
return (null);
}
QRecord record = new QRecord().withValue(table.getPrimaryKeyField(), source.getRecordId());
for(String field : QuickSearchFieldRules.indexableFields(table, tableConfig.getSearchableFields()))
{
Object value = source.getFieldValues() == null ? null : source.getFieldValues().get(field);
if(value instanceof String text)
{
record.setValue(field, text);
}
}
return (buildDocument(record, tableConfig));
}
/*******************************************************************************
**
*******************************************************************************/
@@ -0,0 +1,53 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.processes;
import com.kingsrook.qqq.backend.core.exceptions.QException;
/*******************************************************************************
** Thrown between batches when a run's cancel flag is set (Cancel Index Run),
** or indexing was paused, carrying the counts so far. The run ends CANCELLED
** with those counts; a full rebuild that is cancelled keeps the old index.
*******************************************************************************/
public class QuickSearchRunCancelledException extends QException
{
private final AbstractIndexingStep.IndexCounts counts;
/*******************************************************************************
** Constructor
*******************************************************************************/
public QuickSearchRunCancelledException(String message, AbstractIndexingStep.IndexCounts counts)
{
super(message);
this.counts = counts;
}
/*******************************************************************************
** The counts when the run stopped.
*******************************************************************************/
public AbstractIndexingStep.IndexCounts getCounts()
{
return (counts);
}
}
@@ -68,7 +68,16 @@ public class ReconcileIndexStep extends AbstractIndexingStep
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
String tableNameFilter = input.getValueString("tableName");
if(isPausedSkip(output))
{
output.addValue("recordsIndexed", 0);
output.addValue("documentsRemoved", 0L);
output.addValue("orphanDocumentsRemoved", 0L);
output.addValue("tablesSkipped", 0);
return;
}
String tableNameFilter = tableNameInput(input);
boolean allTables = tableNameFilter == null || tableNameFilter.isBlank();
Integer recordsIndexed = 0;
@@ -160,6 +169,7 @@ public class ReconcileIndexStep extends AbstractIndexingStep
if(ReconcileIndexStep.RUN_TYPE.equals(runType) == false)
{
values.put("lastFullReindexTime", Instant.now());
values.put("realTimeErrorCount", 0);
}
updateIndexRow(indexId, values);
}
@@ -181,6 +191,11 @@ public class ReconcileIndexStep extends AbstractIndexingStep
return (new ReconcileCounts(counts.indexed(), documentsRemoved));
}
catch(QuickSearchRunCancelledException e)
{
completeCancelledRun(run, e);
throw e;
}
catch(QException e)
{
if(!RUN_FAILED.equals(run.getStatus()))
@@ -0,0 +1,323 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.processes.config;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.config.QuickSearchTombstones;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexField;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
/*******************************************************************************
** The steps of Add Search Index, the valid "create" for quick_search_index:
** the admin picks an eligible table (one they can read, not yet indexed,
** not internal or secret), the update interval, the timestamp field and the
** enabled flag; the review step shows the estimated record count, the
** fields the index starts with (record-label fields as titles, then the
** text fields) and what is left out and why; saving inserts the index row
** and its field rows and queues the first build. Fields are adjusted after
** on the index record's Searchable Fields.
*******************************************************************************/
public final class AddSearchIndexSteps
{
////////////////////////////////////////////////////////////////////////
// the input is not "tableName": a process on a table gets that value //
// (the table it runs from) from the frontend //
////////////////////////////////////////////////////////////////////////
public static final String FIELD_TABLE_NAME = "searchTableName";
public static final String RECORD_TABLE = "tableName";
public static final String FIELD_INTERVAL = "basepullIntervalMinutes";
public static final String FIELD_TIMESTAMP = "basepullTimestampField";
public static final String FIELD_ENABLED = "enabled";
/*******************************************************************************
** Utility holder - no instances.
*******************************************************************************/
private AddSearchIndexSteps()
{
}
/*******************************************************************************
** Extract: the one record the input step describes.
*******************************************************************************/
public static class Extract extends AbstractExtractStep
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
getRecordPipe().addRecord(new QRecord()
.withValue(RECORD_TABLE, input.getValueString(FIELD_TABLE_NAME))
.withValue(FIELD_INTERVAL, input.getValueInteger(FIELD_INTERVAL))
.withValue(FIELD_TIMESTAMP, input.getValueString(FIELD_TIMESTAMP))
.withValue(FIELD_ENABLED, input.getValue(FIELD_ENABLED) == null || input.getValueBoolean(FIELD_ENABLED)));
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public Integer doCount(RunBackendStepInput input)
{
return (1);
}
}
/*******************************************************************************
** Transform: validate and describe the new index.
*******************************************************************************/
public static class Transform extends AbstractTransformStep
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
QuickSearchQBitConfig config = ConfigProcessSupport.config();
for(QRecord record : input.getRecords())
{
String tableName = record.getValueString(RECORD_TABLE);
String problem = validate(config, record);
if(problem != null)
{
summary.add(ConfigProcessSupport.line(Status.ERROR, problem));
continue;
}
QTableMetaData table = QContext.getQInstance().getTable(tableName);
List<QuickSearchFieldRules.DefaultField> defaults = QuickSearchFieldRules.defaultFields(table);
List<String> bullets = new ArrayList<>();
for(QuickSearchFieldRules.DefaultField field : defaults)
{
bullets.add(ConfigProcessSupport.fieldLabel(tableName, field.fieldName()) + ": " + field.role().getPossibleValueLabel() + ", weight " + field.weight());
}
Integer estimated = ConfigProcessSupport.countRecords(tableName);
summary.add(ConfigProcessSupport.line(Status.OK, ConfigProcessSupport.tableLabel(tableName) + " will be added to Quick Search; its first build indexes "
+ ConfigProcessSupport.aboutRecords(estimated) + "."));
summary.add(ConfigProcessSupport.line(Status.INFO, "It starts with " + defaults.size() + (defaults.size() == 1 ? " field" : " fields")
+ " (change them afterwards under Searchable Fields on the index):", bullets));
List<String> leftOut = new ArrayList<>();
for(QFieldMetaData field : table.getFields().values())
{
String why = QuickSearchFieldRules.whyNotIndexable(table, field.getName());
if(why != null && (why.contains("sensitive") || why.contains("hidden")))
{
leftOut.add(why);
}
}
if(!leftOut.isEmpty())
{
summary.add(ConfigProcessSupport.line(Status.WARNING, "Never indexed:", leftOut));
}
String timestamp = record.getValueString(FIELD_TIMESTAMP);
summary.add(ConfigProcessSupport.line(Status.INFO, StringUtils.hasContent(timestamp)
? "Changes are picked up every " + record.getValueInteger(FIELD_INTERVAL) + " minutes by " + ConfigProcessSupport.fieldLabel(tableName, timestamp) + ", and at once by real-time indexing."
: "The table has no timestamp field: real-time indexing and the nightly reconcile keep it current."));
if(Boolean.FALSE.equals(record.getValueBoolean(FIELD_ENABLED)))
{
summary.add(ConfigProcessSupport.line(Status.WARNING, "It is added disabled: nothing is indexed until it is enabled."));
}
if(QuickSearchTombstones.exists(tableName, null))
{
summary.add(ConfigProcessSupport.line(Status.INFO, "It was removed from Quick Search before; adding it back clears that."));
}
record.setValue("estimatedRecords", estimated);
record.setValue("fieldCount", defaults.size());
output.addRecord(record);
}
}
/*******************************************************************************
** Fill defaults and return why the record cannot be added, or null.
*******************************************************************************/
static String validate(QuickSearchQBitConfig config, QRecord record) throws QException
{
String tableName = record.getValueString(RECORD_TABLE);
if(!StringUtils.hasContent(tableName))
{
return ("Choose a table to add.");
}
String problem = QuickSearchFieldRules.whyNotEligible(QContext.getQInstance(), config, tableName);
if(problem != null)
{
return ("Cannot add " + tableName + ": " + problem + ".");
}
if(!QuickSearchFieldRules.canRead(tableName))
{
return ("Cannot add " + tableName + ": you do not have permission to read it.");
}
for(QRecord row : QuickSearchConfigPrimer.queryAll(config.getQuickSearchIndexTableName()))
{
if(tableName.equals(row.getValueString("tableName")))
{
return (ConfigProcessSupport.tableLabel(tableName) + " is already in Quick Search.");
}
}
QTableMetaData table = QContext.getQInstance().getTable(tableName);
if(QuickSearchFieldRules.defaultFields(table).isEmpty())
{
return ("Cannot add " + ConfigProcessSupport.tableLabel(tableName) + ": it has no text field to start the index with.");
}
String timestamp = record.getValueString(FIELD_TIMESTAMP);
if(!StringUtils.hasContent(timestamp))
{
record.setValue(FIELD_TIMESTAMP, QuickSearchFieldRules.defaultTimestampField(table));
}
else if(!QuickSearchFieldRules.isTimestampField(table, timestamp))
{
return ("Timestamp field [" + timestamp + "] is not a date-time field of " + ConfigProcessSupport.tableLabel(tableName) + ".");
}
Integer interval = record.getValueInteger(FIELD_INTERVAL);
if(interval == null)
{
record.setValue(FIELD_INTERVAL, config.getDefaultBasepullIntervalMinutes());
}
else if(interval < 0)
{
return ("The update interval cannot be negative (0 means every update).");
}
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
/*******************************************************************************
** Load: insert the index row and its fields, then queue (or run) the build.
*******************************************************************************/
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
QuickSearchQBitConfig config = ConfigProcessSupport.config();
for(QRecord record : input.getRecords())
{
String tableName = record.getValueString(RECORD_TABLE);
boolean enabled = !Boolean.FALSE.equals(record.getValueBoolean(FIELD_ENABLED));
QRecord inserted = QuickSearchConfigPrimer.insertOne(config.getQuickSearchIndexTableName(), new QRecord()
.withValue("tableName", tableName)
.withValue("enabled", enabled)
.withValue("basepullIntervalMinutes", record.getValueInteger(FIELD_INTERVAL))
.withValue("basepullTimestampField", record.getValueString(FIELD_TIMESTAMP))
.withValue("source", QuickSearchIndex.SOURCE_OWNER)
.withValue("ownerEdited", true)
.withValue("realTimeErrorCount", 0)
.withValue("status", enabled ? AbstractIndexingStep.STATUS_PENDING_BUILD : AbstractIndexingStep.STATUS_ACTIVE));
Integer indexId = inserted.getValueInteger("id");
List<QRecord> fields = new ArrayList<>();
int order = 10;
for(QuickSearchFieldRules.DefaultField field : QuickSearchFieldRules.defaultFields(QContext.getQInstance().getTable(tableName)))
{
fields.add(new QRecord()
.withValue("quickSearchIndexId", indexId)
.withValue("fieldName", field.fieldName())
.withValue("weight", field.weight())
.withValue("role", field.role().name())
.withValue("sortOrder", order));
order += 10;
}
QuickSearchConfigPrimer.insertAll(config.applyTablePrefix(QuickSearchIndexField.TABLE_NAME), fields);
QuickSearchTombstones.remove(tableName, null);
String build = !enabled
? "it is disabled, so nothing is indexed until it is enabled"
: ConfigProcessSupport.queueOrBuild(indexId);
summary.add(ConfigProcessSupport.line(Status.OK, "Added " + ConfigProcessSupport.tableLabel(tableName) + " to Quick Search with " + fields.size()
+ (fields.size() == 1 ? " field" : " fields") + "; " + build + "."));
output.addRecord(inserted);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(CollectionUtils.nonNullList(summary)));
}
}
}
@@ -0,0 +1,219 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.processes.config;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
/*******************************************************************************
** The steps of Apply Search Index Changes (record action on
** quick_search_index): the review step previews, per selected index, how its
** searchable fields differ from what its documents were last built with
** (fields added, removed or changed) and how many records the rebuild
** reads; executing queues the rebuild (or runs it now when no scheduler runs
** the basepull). Field edits apply to searches and new writes at once and
** mark the index NEEDS_REINDEX until this rebuilds the older documents.
*******************************************************************************/
public final class ApplyIndexChangesSteps
{
private static final Pattern FIELD_JSON = Pattern.compile("\\{\"fieldName\":\"((?:[^\"\\\\]|\\\\.)*)\"([^}]*)}");
/*******************************************************************************
** Utility holder - no instances.
*******************************************************************************/
private ApplyIndexChangesSteps()
{
}
/*******************************************************************************
** Field name to its JSON attributes (weight, label, role...), from a
** searchableFieldsJson string.
*******************************************************************************/
static Map<String, String> parseFields(String json)
{
Map<String, String> fields = new LinkedHashMap<>();
if(json != null)
{
Matcher matcher = FIELD_JSON.matcher(json);
while(matcher.find())
{
fields.put(matcher.group(1).replace("\\\"", "\"").replace("\\\\", "\\"), matcher.group(2));
}
}
return (fields);
}
/*******************************************************************************
** The differences between the fields last built and the current ones.
*******************************************************************************/
static List<String> describeChanges(String tableName, String builtJson, String currentJson)
{
Map<String, String> built = parseFields(builtJson);
Map<String, String> current = parseFields(currentJson);
List<String> changes = new ArrayList<>();
for(Map.Entry<String, String> entry : current.entrySet())
{
if(!built.containsKey(entry.getKey()))
{
changes.add("Added: " + ConfigProcessSupport.fieldLabel(tableName, entry.getKey()));
}
else if(!Objects.equals(built.get(entry.getKey()), entry.getValue()))
{
changes.add("Changed: " + ConfigProcessSupport.fieldLabel(tableName, entry.getKey()) + " (" + entry.getValue().replaceFirst("^,", "").replace("\"", "") + ")");
}
}
for(String fieldName : built.keySet())
{
if(!current.containsKey(fieldName))
{
changes.add("Removed: " + ConfigProcessSupport.fieldLabel(tableName, fieldName));
}
}
return (changes);
}
/*******************************************************************************
** Transform: preview each selected index's changes.
*******************************************************************************/
public static class Transform extends AbstractTransformStep
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
for(QRecord record : input.getRecords())
{
String tableName = record.getValueString("tableName");
String label = ConfigProcessSupport.tableLabel(tableName);
String status = record.getValueString("status");
if(Boolean.FALSE.equals(record.getValueBoolean("enabled")))
{
summary.add(ConfigProcessSupport.line(Status.WARNING, label + " is disabled; enable it first (enabling queues its build)."));
continue;
}
if(AbstractIndexingStep.STATUS_PENDING_BUILD.equals(status) || AbstractIndexingStep.STATUS_BUILDING.equals(status))
{
summary.add(ConfigProcessSupport.line(Status.INFO, label + " is already queued or building; it is skipped."));
continue;
}
QuickSearchableTableConfig tableConfig = runtime == null ? null : runtime.getTableConfig(tableName);
if(tableConfig == null)
{
summary.add(ConfigProcessSupport.line(Status.ERROR, label + " is not in this application any more; remove it instead."));
continue;
}
List<String> changes = describeChanges(tableName, record.getValueString("searchableFieldsJson"), AbstractIndexingStep.buildSearchableFieldsJson(tableConfig));
String reads = ConfigProcessSupport.aboutRecords(ConfigProcessSupport.countRecords(tableName));
if(changes.isEmpty())
{
summary.add(ConfigProcessSupport.line(Status.OK, label + ": no field changes since its last build; it will be rebuilt anyway (" + reads + ")."));
}
else
{
summary.add(ConfigProcessSupport.line(Status.OK, label + " will be rebuilt with these field changes (" + reads + "):", changes));
}
output.addRecord(record);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
/*******************************************************************************
** Load: queue (or run) each rebuild.
*******************************************************************************/
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
for(QRecord record : input.getRecords())
{
String outcome = ConfigProcessSupport.queueOrBuild(record.getValueInteger("id"));
summary.add(ConfigProcessSupport.line(Status.OK, ConfigProcessSupport.tableLabel(record.getValueString("tableName")) + ": " + outcome + "."));
output.addRecord(record);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
}
@@ -0,0 +1,152 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.processes.config;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
/*******************************************************************************
** The steps of Cancel Index Run (record action on quick_search_index_run):
** sets the run's cooperative cancel flag. The run checks it between batches
** and ends CANCELLED with its counts so far; a cancelled all-tables rebuild
** keeps the previous index in service, and a cancelled queued build waits
** for Apply Search Index Changes. Runs that are not RUNNING are skipped.
*******************************************************************************/
public final class CancelRunSteps
{
/*******************************************************************************
** Utility holder - no instances.
*******************************************************************************/
private CancelRunSteps()
{
}
/*******************************************************************************
** Transform: which runs can be cancelled.
*******************************************************************************/
public static class Transform extends AbstractTransformStep
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
for(QRecord record : input.getRecords())
{
String name = describe(record);
if(!AbstractIndexingStep.RUN_RUNNING.equals(record.getValueString("status")))
{
summary.add(ConfigProcessSupport.line(Status.INFO, name + " is not running (" + record.getValueString("status") + "); it is skipped."));
continue;
}
if(Boolean.TRUE.equals(record.getValueBoolean("cancelRequested")))
{
summary.add(ConfigProcessSupport.line(Status.INFO, name + " was already asked to stop; it is skipped."));
continue;
}
summary.add(ConfigProcessSupport.line(Status.OK, name + " (" + record.getValueString("recordsProcessed") + " records so far) will stop after its current batch and end CANCELLED."));
output.addRecord(record);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
/*******************************************************************************
** Load: set the cancel flags.
*******************************************************************************/
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
String table = ConfigProcessSupport.config().getQuickSearchIndexRunTableName();
String user = QuickSearchConfigWrites.currentUserName();
List<QRecord> updates = new ArrayList<>();
for(QRecord record : input.getRecords())
{
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("cancelRequested", true).withValue("cancelRequestedBy", user));
summary.add(ConfigProcessSupport.line(Status.OK, describe(record) + " was asked to stop; it ends CANCELLED after its current batch."));
output.addRecord(record);
}
QuickSearchConfigPrimer.updateAll(table, updates);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
/*******************************************************************************
** "Run 12 (FULL_REINDEX)".
*******************************************************************************/
static String describe(QRecord run)
{
return ("Run " + run.getValueString("id") + " (" + run.getValueString("runType") + ")");
}
}
@@ -0,0 +1,257 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.processes.config;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigWrites;
import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules;
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Shared pieces of the config processes (Add, Disable and Remove Search
** Index, Apply Search Index Changes, Cancel Index Run, Pause and Resume
** Indexing): the QBit's config and runtime, record counts for the review
** step, summary lines, and queueing (or, with no scheduler, running) a build.
*******************************************************************************/
public final class ConfigProcessSupport
{
private static final QLogger LOG = QLogger.getLogger(ConfigProcessSupport.class);
/*******************************************************************************
** Utility class - no instances.
*******************************************************************************/
private ConfigProcessSupport()
{
}
/*******************************************************************************
** The QBit's config (runtime first, then the static facade).
*******************************************************************************/
static QuickSearchQBitConfig config() throws QException
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
QuickSearchQBitConfig config = runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig();
if(config == null)
{
throw (new QException("Quick Search is not configured in this application"));
}
return (config);
}
/*******************************************************************************
** The number of records in a table, or null when it cannot be counted.
*******************************************************************************/
static Integer countRecords(String tableName)
{
try
{
return (new CountAction().execute(new CountInput(tableName)).getCount());
}
catch(Exception e)
{
LOG.debug("Could not count records for the review step", e, logPair("tableName", tableName));
return (null);
}
}
/*******************************************************************************
** The number of search documents of a table, or null when OpenSearch
** cannot be asked.
*******************************************************************************/
static Long countDocuments(String tableName)
{
try
{
QuickSearchRuntime runtime = QuickSearchRuntime.get();
return (runtime == null ? null : runtime.getClient().countDocumentsForTable(tableName));
}
catch(Exception e)
{
LOG.debug("Could not count search documents for the review step", e, logPair("tableName", tableName));
return (null);
}
}
/*******************************************************************************
** A table's label (or name).
*******************************************************************************/
static String tableLabel(String tableName)
{
QTableMetaData table = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
return (table == null ? String.valueOf(tableName) : QuickSearchFieldRules.label(table));
}
/*******************************************************************************
** A field's label and name: "Title (title)".
*******************************************************************************/
static String fieldLabel(String tableName, String fieldName)
{
QTableMetaData table = tableName == null || QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
if(table != null && table.getFields().containsKey(fieldName) && StringUtils.hasContent(table.getField(fieldName).getLabel()))
{
return (table.getField(fieldName).getLabel() + " (" + fieldName + ")");
}
return (fieldName);
}
/*******************************************************************************
** "about 1,234 records" (or "an unknown number of records").
*******************************************************************************/
static String aboutRecords(Integer count)
{
return (count == null ? "an unknown number of records" : "about " + String.format("%,d", count) + (count == 1 ? " record" : " records"));
}
/*******************************************************************************
** A summary line with a message only.
*******************************************************************************/
static ProcessSummaryLine line(Status status, String message)
{
return (new ProcessSummaryLine(status, null, message));
}
/*******************************************************************************
** A summary line with bullets of text.
*******************************************************************************/
static ProcessSummaryLine line(Status status, String message, List<String> bullets)
{
return (new ProcessSummaryLine(status, null, message).withBulletsOfText(new ArrayList<>(bullets)));
}
/*******************************************************************************
** Copy lines into a fresh list (the summary providers' return type).
*******************************************************************************/
static ArrayList<ProcessSummaryLineInterface> lines(List<? extends ProcessSummaryLineInterface> lines)
{
return (new ArrayList<>(lines));
}
/*******************************************************************************
** The quick_search_index row, or null.
*******************************************************************************/
static QRecord indexRow(Integer id) throws QException
{
GetInput getInput = new GetInput(config().getQuickSearchIndexTableName());
getInput.setPrimaryKey(id);
return (new GetAction().execute(getInput).getRecord());
}
/*******************************************************************************
** Update an index row as a system write.
*******************************************************************************/
static void updateIndexRowAsSystem(Integer id, Map<String, ? extends Serializable> values) throws QException
{
QRecord record = new QRecord().withValue("id", id);
values.forEach(record::setValue);
QuickSearchConfigWrites.asSystem(() ->
{
QuickSearchConfigPrimer.updateAll(config().getQuickSearchIndexTableName(), List.of(record));
return (null);
});
}
/*******************************************************************************
** Queue a table's build (PENDING_BUILD, run by the next scheduled basepull),
** or, when no scheduler runs the basepull, build it now. Returns what
** happened, for the result screen.
*******************************************************************************/
static String queueOrBuild(Integer indexId) throws QException
{
QuickSearchQBitConfig config = config();
updateIndexRowAsSystem(indexId, Map.of("status", AbstractIndexingStep.STATUS_PENDING_BUILD));
QuickSearchRuntime runtime = QuickSearchRuntime.get();
if(runtime != null)
{
runtime.invalidateConfig();
}
if(StringUtils.hasContent(config.getSchedulerName()) && Boolean.TRUE.equals(config.getEnableBasepullProcess()))
{
Integer seconds = config.getBasepullRepeatSeconds() == null ? 300 : config.getBasepullRepeatSeconds();
return ("its build is queued and starts with the next update (within " + seconds + " seconds)");
}
if(runtime != null && runtime.isIndexingPaused())
{
return ("its build is queued and starts when indexing is resumed");
}
QRecord row = indexRow(indexId);
try
{
boolean built = new BasepullIndexStep().buildTable(row, runtime);
QRecord after = indexRow(indexId);
return (built ? "it was built now (" + aboutRecords(after == null ? null : after.getValueInteger("recordCount")) + " indexed)" : "its build is queued");
}
catch(Exception e)
{
return ("its build failed and is queued to retry: " + e.getMessage());
}
}
}
@@ -0,0 +1,188 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.quicksearch.processes.config;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer;
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
import com.kingsrook.qbits.quicksearch.model.QuickSearchSettings;
/*******************************************************************************
** The steps of Pause Indexing and Resume Indexing (record actions on
** quick_search_settings; the process's "pause" value says which). Pausing
** stops all indexing, real-time and scheduled, on every server (within the
** settings cache time); changes made meanwhile are kept and replayed on
** resume, and search keeps serving the current index. The settings table's
** customizer stamps who paused it and when.
*******************************************************************************/
public final class SetIndexingPausedSteps
{
public static final String FIELD_PAUSE = "pause";
public static final String FIELD_REASON = "pauseReason";
/*******************************************************************************
** Utility holder - no instances.
*******************************************************************************/
private SetIndexingPausedSteps()
{
}
/*******************************************************************************
** Transform: say what pausing or resuming does.
*******************************************************************************/
public static class Transform extends AbstractTransformStep
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
boolean pause = Boolean.TRUE.equals(input.getValueBoolean(FIELD_PAUSE));
for(QRecord record : input.getRecords())
{
boolean paused = Boolean.TRUE.equals(record.getValueBoolean("indexingPaused"));
if(pause == paused)
{
summary.add(ConfigProcessSupport.line(Status.INFO, pause
? "Indexing is already paused (since " + record.getValueString("pausedAt") + " by " + record.getValueString("pausedBy") + ")."
: "Indexing is not paused."));
continue;
}
if(pause)
{
summary.add(ConfigProcessSupport.line(Status.OK, "All indexing will pause on every server (real-time and scheduled; a running rebuild stops after its current batch)."));
summary.add(ConfigProcessSupport.line(Status.INFO, "Search keeps serving the current index. Changes made while paused are kept and indexed when you resume."));
}
else
{
summary.add(ConfigProcessSupport.line(Status.OK, "Indexing will resume; " + deferredCount() + " changes made while paused will be indexed by the next update."));
}
output.addRecord(record);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
/*******************************************************************************
** Load: set the flag (the settings customizer does the rest).
*******************************************************************************/
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private final List<ProcessSummaryLine> summary = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
QuickSearchQBitConfig config = ConfigProcessSupport.config();
boolean pause = Boolean.TRUE.equals(input.getValueBoolean(FIELD_PAUSE));
for(QRecord record : input.getRecords())
{
QRecord update = new QRecord().withValue("id", record.getValue("id")).withValue("indexingPaused", pause);
if(pause)
{
update.setValue("pauseReason", input.getValueString(FIELD_REASON));
}
QuickSearchConfigPrimer.updateAll(config.applyTablePrefix(QuickSearchSettings.TABLE_NAME), List.of(update));
summary.add(ConfigProcessSupport.line(Status.OK, pause ? "Indexing is paused. Search keeps serving the current index." : "Indexing resumed."));
output.addRecord(record);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
return (ConfigProcessSupport.lines(summary));
}
}
/*******************************************************************************
** Changes deferred while paused (0 when they cannot be counted).
*******************************************************************************/
static Integer deferredCount()
{
try
{
String table = ConfigProcessSupport.config().applyTablePrefix(QuickSearchFailedEvent.TABLE_NAME);
if(QContext.getQInstance().getTable(table) == null)
{
return (0);
}
CountInput countInput = new CountInput(table);
countInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_DEFERRED)));
Integer count = new CountAction().execute(countInput).getCount();
return (count == null ? 0 : count);
}
catch(Exception e)
{
return (0);
}
}
}
Loaded 100 of 138 files, more files were not shown because too many files have changed in this diff. Show more