Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  james.maesandClaude Opus 5.5
|
e06b06000b
|
||
|
james.maes
|
2b627e767a | ||
|
james.maes
|
c5c41d486a | ||
|
james.maes
|
dce0f26287 | ||
|
james.maes
|
9941a708bb | ||
|
james.maes
|
e9ec54a3a2 | ||
|
james.maes
|
2398e62711 | ||
|
james.maes
|
12afb08deb | ||
|
james.maes
|
36bc0fb6f1 | ||
|
james.maes
|
0cd892d1a0 | ||
|
james.maes
|
000e5bb1cf | ||
|
James MaesandClaude Opus 4.8
|
7eb6d4b8e3 | ||
|
james.maesandClaude Opus 4.7
|
8d9ec2711f | ||
|
james.maes
|
fa036c8e18 | ||
|
james.maes
|
c888b771e7 |
No files matched your search
+68
-11
@@ -1,14 +1,71 @@
|
||||
version: 2.1
|
||||
setup: true
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.7.15
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: 0.7.15
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /v.*/
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*|hotfix\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/v.*/]
|
||||
|
||||
publish_hotfix_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: hotfix
|
||||
filters:
|
||||
branches:
|
||||
only: [/hotfix\/.*/]
|
||||
@@ -1,63 +0,0 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(find:*)",
|
||||
"Bash(grep:*)",
|
||||
"Bash(mvn compile:*)",
|
||||
"Bash(jar -tf:*)",
|
||||
"Bash(jar -xf:*)",
|
||||
"Bash(cat:*)",
|
||||
"Bash(mvn test:*)",
|
||||
"Bash(done)",
|
||||
"Bash(mvn verify:*)",
|
||||
"Bash(git add:*)",
|
||||
"Bash(git commit:*)",
|
||||
"Bash(git push:*)",
|
||||
"mcp__circleci-mcp-server__get_latest_pipeline_status",
|
||||
"mcp__circleci-mcp-server__list_followed_projects",
|
||||
"mcp__circleci-mcp-server__run_pipeline",
|
||||
"mcp__circleci-mcp-server__get_build_failure_logs",
|
||||
"mcp__circleci-mcp-server__config_helper",
|
||||
"mcp__github__get_file_contents",
|
||||
"Bash(mvn dependency:tree:*)",
|
||||
"Bash(mvn help:effective-pom:*)",
|
||||
"Bash(git checkout:*)",
|
||||
"Bash(ls /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/docs/*.md)",
|
||||
"Bash(git:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(open:*)",
|
||||
"Bash(WT=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/.claude/worktrees/agent-a9dc6e8e)",
|
||||
"Bash(DEST=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search)",
|
||||
"Bash(cp \"$WT/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java\" \"$DEST/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/\")",
|
||||
"Bash(mvn clean:*)",
|
||||
"Bash(jar tf:*)",
|
||||
"Bash(unzip -p /Users/james.maes/.m2/repository/com/kingsrook/qqq/qqq-backend-core/0.40.0-SNAPSHOT/qqq-backend-core-0.40.0-SNAPSHOT.jar com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java)",
|
||||
"Bash(sed -i '' 's/postInsertRecord/postInsert/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java)",
|
||||
"Bash(sed -i '' 's/postUpdateRecord/postUpdate/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java)",
|
||||
"Bash(sed -i '' 's/postDeleteRecord/postDelete/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java)",
|
||||
"Bash(docker info:*)",
|
||||
"Bash(docker run:*)",
|
||||
"Read(//var/run/**)",
|
||||
"Read(//Users/james.maes/**)",
|
||||
"Bash(echo \"DOCKER_HOST=$DOCKER_HOST\")",
|
||||
"Bash(echo \"TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=$TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE\")",
|
||||
"Bash(DOCKER_HOST=unix:///var/run/docker.sock TESTCONTAINERS_RYUK_DISABLED=true mvn verify -Dit.test=OpenSearchIntegrationTest)",
|
||||
"Bash(docker version:*)",
|
||||
"Bash(DOCKER_HOST=unix:///var/run/docker.sock mvn verify -Dit.test=OpenSearchIntegrationTest)",
|
||||
"Bash(curl -s --unix-socket /var/run/docker.sock http://localhost/v1.24/info)",
|
||||
"Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)} OS: {d.get\\(OperatingSystem,?\\)}''''\\)\")",
|
||||
"Bash(curl -s --unix-socket /Users/james.maes/Library/Containers/com.docker.docker/Data/docker-cli.sock http://localhost/v1.47/info)",
|
||||
"Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)} OS: {d.get\\(OperatingSystem,?\\)} Containers: {d.get\\(Containers,?\\)}''''\\)\")",
|
||||
"Bash(curl -s --unix-socket /Users/james.maes/.docker/run/docker.sock http://localhost/v1.47/info)",
|
||||
"Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)} NCPU: {d.get\\(NCPU,?\\)} OSType: {d.get\\(OSType,?\\)}''''\\)\")",
|
||||
"Bash(DOCKER_HOST=unix:///Users/james.maes/.docker/run/docker.sock DOCKER_API_VERSION=1.47 mvn verify -Dit.test=OpenSearchIntegrationTest -pl .)",
|
||||
"Bash(cp /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/LICENSE /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/LICENSE)",
|
||||
"Bash(cp -r /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/checkstyle/* /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/checkstyle/)",
|
||||
"Bash(gh pr:*)",
|
||||
"Bash(gh repo:*)",
|
||||
"Bash(gh api:*)",
|
||||
"Bash(python3 -c \":*)",
|
||||
"mcp__circleci-mcp-server__get_job_test_results"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
* @KofTwentyTwo
|
||||
@@ -1,23 +0,0 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,74 +0,0 @@
|
||||
name: Release candidate
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Plan a cut without changing release state
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
checks: read
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
@@ -18,3 +18,4 @@ hs_err_pid*
|
||||
dependency-reduced-pom.xml
|
||||
/.env.local
|
||||
/.cache/
|
||||
.claude/settings.local.json
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
wrapperVersion=3.3.4
|
||||
distributionType=only-script
|
||||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
profile: qbit
|
||||
orb_version: 0.7.15
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: true
|
||||
release_tag_policy: v-prefix
|
||||
test_branch_policy: non-release-no-main-hotfix
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
hotfix: true
|
||||
asciidoc: false
|
||||
@@ -0,0 +1,67 @@
|
||||
# AGENTS.md
|
||||
|
||||
This file provides guidance to Codex when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications.
|
||||
|
||||
## Build Commands
|
||||
|
||||
```bash
|
||||
./mvnw compile # Compile (always use the wrapper: Maven 3.10 cannot read the parent POM)
|
||||
./mvnw test # Run unit tests
|
||||
./mvnw test -Dtest=ClassName # Run single test class
|
||||
./mvnw package # Build JAR
|
||||
CI=true ./mvnw verify # Full verify: integration tests (Docker) and the JaCoCo gate
|
||||
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0 # other server line
|
||||
./mvnw verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT # other qqq 4.1 build (profile is active by default)
|
||||
```
|
||||
|
||||
Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
|
||||
Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
|
||||
Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped without Docker locally, failed when `CI=true`.
|
||||
|
||||
## Architecture
|
||||
|
||||
**Entry point:** `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`. QQQ's default `produce()` registers `QBitMetaData` (`com.kingsrook.qbits:quick-search`), validates the config, runs the component producers in `metadata/` (tables, processes, PVS, app), then `postProduceActions` discovers searchable tables, builds the `QuickSearchRuntime`, registers `QuickSearchRecordChangeListener` and `QuickSearchRuntimeService`, and in `FAIL_FAST` mode prepares the index. `produce(QInstance)` adds everything to the instance (0.x call style still works).
|
||||
|
||||
**Key Classes:**
|
||||
- `QuickSearchQBitConfig` - Host-facing config: connection (`opensearchUrl`), `authMode`, `tls`, timeouts, startup mode, scheduling, permission rules, search bounds, tables. List-based, null-safe `validate`.
|
||||
- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
|
||||
- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
|
||||
- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
|
||||
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
|
||||
- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
|
||||
- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
|
||||
- `metadata/*` - Component producers and the table/process helpers.
|
||||
|
||||
**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
|
||||
|
||||
**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
|
||||
|
||||
## Code Style
|
||||
|
||||
- 3-space indentation, opening braces on next line
|
||||
- Javadoc flower box comments (80-char width with `**` borders)
|
||||
- Fluent-style setters (`withX()`)
|
||||
- Use wrapper types (`Integer`, `Boolean`) not primitives
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Java 21
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom:4.1.0-RC.1` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007)
|
||||
- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
|
||||
- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
|
||||
- JUnit 5 + AssertJ + Mockito for testing
|
||||
- Testcontainers 1.21.4 for integration tests (requires Docker)
|
||||
|
||||
## Knowledge base
|
||||
|
||||
Second-brain vault notes covering this repo and the QQQ platform it plugs into:
|
||||
|
||||
- Platform hub: `$SECOND_BRAIN_VAULT/knowledge/qqq/qqq-hub.md` (start here; read `knowledge/qqq/architecture/metadata-model.md` for QBit mechanics)
|
||||
- This repo's dossier: `$SECOND_BRAIN_VAULT/knowledge/qqq/repos/qbit-quick-search.md` (reviewed at develop @ `8d9ec2711fde`, 2026-07-04)
|
||||
- Production-readiness audit + P0-P3 roadmap: `$SECOND_BRAIN_VAULT/projects/qbit-quick-search.md`
|
||||
|
||||
Note: the dossier was reviewed before the qqq 4.0 re-pin; trust this file for build and dependency facts.
|
||||
@@ -0,0 +1,64 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
|
||||
|
||||
## [1.0.0-RC.2] - 2026-10-06
|
||||
|
||||
Second release candidate, published to Maven Central from `release/1.0`. Fixes the five pre-GA findings from the PR #10 review (#13 to #17); the changes are listed under 1.0.0 below. Requires QQQ 4.1. No schema change, but two new status values: `REBUILDING` on `quickSearchIndex.status` and `AWAITING_REINDEX` on `quickSearchFailedEvent.status`; a host that sized its own `quickSearchFailedEvent.status` column needs at least 16 characters. Known gaps before GA are tracked in #12.
|
||||
|
||||
## [1.0.0-RC.1] - 2026-10-05
|
||||
|
||||
First release candidate of 1.0, published to Maven Central from `release/1.0`. It contains every change listed under 1.0.0 below. Requires QQQ 4.1 (built against 4.1.0-RC.1). Known gaps before GA are tracked in #12.
|
||||
|
||||
## [1.0.0] - Unreleased
|
||||
|
||||
Requires QQQ 4.1. Closes every blocker of the 2026-10-04 production-readiness audit (`docs/audits/2026-10-04/README.md`).
|
||||
|
||||
### Added
|
||||
- Standard QQQ 4.1 QBit contract: `QuickSearchQBitProducer implements QBitMetaDataProducer`, self-registered `QBitMetaData` (`com.kingsrook.qbits:quick-search`, real artifact version), `sourceQBitName` on produced metadata, component producers, `QuickSearchRuntimeService`.
|
||||
- Real-time indexing through `RecordChangeListenerInterface`, published after commit via `QBackendTransaction.addAfterCommitCallback`.
|
||||
- `quickSearchFailedEvent` table: failed real-time events are recorded and replayed by basepull.
|
||||
- Scheduling: `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`.
|
||||
- Full reindex of all tables builds a fresh physical index and swaps the alias; the configured index name is now an alias.
|
||||
- Mapping version 2: separate search analyzer, dynamic template mapping `fieldValues.*` as text, `_meta.quickSearchMappingVersion`; outdated indexes are detected and rebuilt by a full reindex.
|
||||
- Permission-aware search: table READ permission filter, `QueryAction` post-filter for record security locks, `tableNames`, `limitPerTable`, `tableLabel`, `totalHitsIsLowerBound`, term and page bounds.
|
||||
- Connection security: `opensearchUrl`, `authMode` (`NONE`, `BASIC`, `AWS_SIGV4`), `tls` block (CA, trust store, mTLS, hostname verification, guarded `insecureSkipVerify`), timeouts, pool size, `User-Agent`, `OpenSearchTransportCustomizer` SPI, `${env.X}` secret references, redacting `toString`, `@JsonIgnore` on secret getters.
|
||||
- `startupMode` (`FAIL_FAST` default, `DEGRADED`), `enableBasepullProcess` and `enableMaintenanceProcesses`, `basepullOverlapSeconds`, `runHistoryRetentionDays`, `maxFieldLength`, `maxSearchLimit`, `maxBulkRequestBytes`, `applyRecordSecurityLocks`, `adminPermissionRules`, `tablePermissionRules`, `tableMetaDataCustomizer`.
|
||||
- Admin table fields: `lastReconcileTime`, `lastRunStatus`, `lastErrorMessage`, `documentCount`, `realTimeErrorCount`; `quickSearchIndexRun.quickSearchIndexId` shows the table name; icons and permission rules on the app, tables and processes; unique key on `quickSearchIndex.tableName`.
|
||||
- Display values and possible-value labels are indexed; record labels fall back to QQQ's record label format; hidden fields are excluded; values are stored as strings and truncated to `maxFieldLength`.
|
||||
- External versioning (`external_gte`) from the basepull timestamp field so stale writes never overwrite newer documents.
|
||||
- Reconcile removes documents of tables no longer configured; drift between configured fields and the index row sets `status = NEEDS_REINDEX`.
|
||||
- Maven Wrapper (3.9.11), JaCoCo coverage gate actually enforced, parameterized Testcontainers image (`-Dopensearch.test.image`).
|
||||
|
||||
### Changed
|
||||
- Build targets qqq 4.1 (4.1.0-RC.1 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 is released), opensearch-java 3.10.0, JUnit from the parent BOM.
|
||||
- Basepull and full reindex page by primary key; the basepull watermark is the run start (minus overlap) with greater-than-or-equal semantics.
|
||||
- Basepull fails at the end when any table failed (one table's failure does not stop the others).
|
||||
- Query terms use the AND operator and are no longer edge-ngrammed at search time.
|
||||
- `QuickSearchOpenSearchClient.search` takes a collection of allowed tables; `deleteDocumentsForTable` returns the deleted count; all OpenSearch errors surface as `QException`.
|
||||
- `IndexingUtils.normalizeSearchText` no longer lower-cases (analyzers do).
|
||||
- `enableScheduledProcesses` is deprecated in favour of `enableBasepullProcess` and `enableMaintenanceProcesses`.
|
||||
- `QuickSearchQBitContext` is deprecated; state lives on `QuickSearchRuntime`, resolved from the `QInstance`.
|
||||
- Search `offset` counts results the session can read. With record security locks on, hits are scanned from the start in batches of at most 1,000, filtered for access and deduplicated; one call reads at most 10,000 hits over 32 requests, and in per-table mode each of T tables gets max(10,000 / T, 2 x `limitPerTable`) hits. When that share runs out `hasMore` is false and `totalHitsIsLowerBound` is true. With locks on, `totalHits` counts hits confirmed readable. Per-table mode now honours `offset` (#16).
|
||||
- A full reindex of all tables marks each table `REBUILDING` while it runs; drift detection is suspended for those tables. After a killed run, run the full reindex again (#14).
|
||||
|
||||
### Removed
|
||||
- The three table customizers (`QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`), replaced by `QuickSearchRecordChangeListener`.
|
||||
- Support for QQQ 4.0.
|
||||
|
||||
### Security
|
||||
- `opensearchUrl` with userinfo (`https://user:pass@host`) fails validation; an `${env.X}` or `${prop.X}` credential or TLS path that resolves to an empty or blank value fails transport creation and names the field, never the value; `tls.hostnameVerification=false` logs a warning (#15).
|
||||
|
||||
### Fixed
|
||||
- Rolled-back writes could be indexed; real-time failures were silent; basepull lost rows modified during a run and rows at the boundary second; unordered paging skipped or duplicated rows; full reindex blanked search; search leaked labels and snippets across tables regardless of permissions; mixed field types across tables broke indexing; non-text searchable fields failed every search; record labels were null for annotation tables; duplicate `quickSearchIndex` rows on concurrent first runs; unbounded `limit`/`offset`; unchecked `OpenSearchException`; unreachable cluster at startup left the host healthy with search broken.
|
||||
- Deletes made during a full reindex of all tables were lost from the new index; they are now captured and applied after the alias swap (#14).
|
||||
- An update was dropped without a `quickSearchFailedEvent` row when the listener's re-read failed or the optional AWS SDK was missing (`LinkageError`); both are now recorded for basepull, and a `DEGRADED` start without the AWS SDK no longer aborts host startup (#13).
|
||||
- Search pages overlapped under record locks, per-table mode ignored `offset`, and a large `maxSearchLimit` could exceed the 10,000 result window (#16).
|
||||
- A failing document count was swallowed and cleared `documentCount`; it is now logged at warn and the previous count is kept (#17).
|
||||
- A real-time insert was indexed with raw values (a foreign key as its id, the record label as "2 - email") until the next basepull. Inserts are now re-read in the write's transaction with display values and possible-value labels, as updates are; a failed re-read is recorded in `quickSearchFailedEvent` for basepull and never fails the insert (#28).
|
||||
|
||||
## [0.2.0] - 2026-03-30
|
||||
- Config-driven table indexing (`SearchableTableConfig`).
|
||||
|
||||
## [0.1.0] - 2026-03-29
|
||||
- Initial release.
|
||||
@@ -4,59 +4,41 @@ This file provides guidance to Claude Code when working with code in this reposi
|
||||
|
||||
## Project Overview
|
||||
|
||||
**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications. Current version: 0.1.0.
|
||||
**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications.
|
||||
|
||||
## Build Commands
|
||||
|
||||
```bash
|
||||
mvn compile # Compile
|
||||
mvn test # Run all tests (174 tests)
|
||||
mvn test -Dtest=ClassName # Run single test class
|
||||
mvn package # Build JAR
|
||||
mvn verify # Full verify with coverage check
|
||||
./mvnw compile # Compile (always use the wrapper: Maven 3.10 cannot read the parent POM)
|
||||
./mvnw test # Run unit tests
|
||||
./mvnw test -Dtest=ClassName # Run single test class
|
||||
./mvnw package # Build JAR
|
||||
CI=true ./mvnw verify # Full verify: integration tests (Docker) and the JaCoCo gate
|
||||
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0 # other server line
|
||||
./mvnw verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT # other qqq 4.1 build (profile is active by default)
|
||||
```
|
||||
|
||||
Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
|
||||
Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
|
||||
Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped without Docker locally, failed when `CI=true`.
|
||||
|
||||
## Architecture
|
||||
|
||||
**Entry point:** `QuickSearchQBitProducer.produce(QInstance)` validates config, discovers annotated entity classes, initializes the singleton `QuickSearchOpenSearchClient`, creates an `IndexEventPublisher`, registers all metadata into the `QInstance`, and attaches customizers to source tables.
|
||||
**Entry point:** `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`. QQQ's default `produce()` registers `QBitMetaData` (`com.kingsrook.qbits:quick-search`), validates the config, runs the component producers in `metadata/` (tables, processes, PVS, app), then `postProduceActions` discovers searchable tables, builds the `QuickSearchRuntime`, registers `QuickSearchRecordChangeListener` and `QuickSearchRuntimeService`, and in `FAIL_FAST` mode prepares the index. `produce(QInstance)` adds everything to the instance (0.x call style still works).
|
||||
|
||||
**Key Classes:**
|
||||
- `QuickSearchQBitProducer` - Orchestrator: produces tables, processes, app, and registers customizers
|
||||
- `QuickSearchQBitConfig` - Configuration (backend name, OpenSearch connection, feature flags)
|
||||
- `QuickSearchQBitContext` - Static singleton holding the live config, client, publisher, and discovered table configs
|
||||
- `QuickSearchOpenSearchClient` - All OpenSearch operations (ensure index, bulk index, delete, search)
|
||||
- `QuickSearchableTableConfig` - Runtime config for a single indexed table (fields, weights, basepull settings)
|
||||
- `QuickSearchQBitConfig` - Host-facing config: connection (`opensearchUrl`), `authMode`, `tls`, timeouts, startup mode, scheduling, permission rules, search bounds, tables. List-based, null-safe `validate`.
|
||||
- `QuickSearchRuntime` - Instance-scoped live state (discovered tables, lazy client, publisher, enabled cache), held on the config and resolved via `QuickSearchRuntime.get()` from `QContext`. `QuickSearchQBitContext` is a deprecated static facade over it.
|
||||
- `opensearch/QuickSearchOpenSearchClient` - All OpenSearch operations; the configured index name is an alias over `{alias}-v2-{epoch}` physical indexes (mapping v2, `_meta.quickSearchMappingVersion`); versioned bulk writes (`external_gte`); alias swap; wraps every error in `QException`.
|
||||
- `opensearch/OpenSearchTransportFactory` - Builds the transport for `NONE`/`BASIC` (Apache HttpClient 5 with TLS, timeouts, customizer SPI) or `AWS_SIGV4` (`AwsSdk2Transport`); resolves `${env.X}` secrets.
|
||||
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
|
||||
- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
|
||||
- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
|
||||
- `metadata/*` - Component producers and the table/process helpers.
|
||||
|
||||
**Customizers** (`customizers/` package, all three registered when `enableRealTimeIndexing = true`):
|
||||
- `QuickSearchPostInsertCustomizer` - Fires after insert, publishes index events
|
||||
- `QuickSearchPostUpdateCustomizer` - Fires after update, publishes index events
|
||||
- `QuickSearchPostDeleteCustomizer` - Fires after delete, publishes delete events
|
||||
**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
|
||||
|
||||
**Processes** (`processes/` package):
|
||||
- `AbstractIndexingStep` - Base class with shared indexing logic (query source table, build documents, publish events, update run record)
|
||||
- `BasepullIndexStep` - Queries records modified since last run timestamp
|
||||
- `FullReindexStep` - Wipes and rebuilds the index for one or all tables
|
||||
- `IndexingUtils` - Converts `QRecord` values to indexed text using field configs
|
||||
|
||||
**Publisher** (`publisher/` package):
|
||||
- `IndexEventPublisher` - Strategy interface with `publishIndexEvents()`, `publishDeleteEvents()`, `close()`
|
||||
- `SynchronousIndexEventPublisher` - Default implementation; processes events immediately via the OpenSearch client
|
||||
- `IndexEvent` / `IndexEventAction` - Event model
|
||||
|
||||
**Actions** (`actions/` package):
|
||||
- `QuickSearchAction` - Executes a paginated multi-match search across all indexed fields with per-field boosting
|
||||
- `QuickSearchInput` / `QuickSearchOutput` / `QuickSearchResult` - Search API model
|
||||
|
||||
**Tables Produced:**
|
||||
- `quickSearchIndex` - One row per indexed table; tracks enabled state, basepull interval, last run times
|
||||
- `quickSearchIndexRun` - Run history with status, record counts, and error messages
|
||||
|
||||
**Annotation Discovery:**
|
||||
- `@QuickSearchable(tableName = "...")` - Mark entity classes; defines basepull settings
|
||||
- `@QuickSearchField(weight, includeLabel)` - Mark fields to include in indexed text; takes precedence over `fields[]` array
|
||||
**Annotation Discovery:** `@QuickSearchable(tableName = "...")` on entity classes; `@QuickSearchField(weight, includeLabel)` on fields; hidden fields are excluded.
|
||||
|
||||
## Code Style
|
||||
|
||||
@@ -68,9 +50,18 @@ Unit tests exclude `*IntegrationTest.java` and `*IT.java`.
|
||||
## Dependencies
|
||||
|
||||
- Java 21
|
||||
- QQQ Backend Core 0.40.0-SNAPSHOT (via `qqq-bom-pom`)
|
||||
- OpenSearch Java Client 2.10.0
|
||||
- Apache HttpClient5 5.3
|
||||
- jackson-datatype-jsr310 2.21.0 (Instant serialization for OpenSearch client)
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom:4.1.0-RC.1` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007)
|
||||
- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
|
||||
- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
|
||||
- JUnit 5 + AssertJ + Mockito for testing
|
||||
- Testcontainers for integration tests (requires Docker)
|
||||
- Testcontainers 1.21.4 for integration tests (requires Docker)
|
||||
|
||||
## Knowledge base
|
||||
|
||||
Second-brain vault notes covering this repo and the QQQ platform it plugs into:
|
||||
|
||||
- Platform hub: `$SECOND_BRAIN_VAULT/knowledge/qqq/qqq-hub.md` (start here; read `knowledge/qqq/architecture/metadata-model.md` for QBit mechanics)
|
||||
- This repo's dossier: `$SECOND_BRAIN_VAULT/knowledge/qqq/repos/qbit-quick-search.md` (reviewed at develop @ `8d9ec2711fde`, 2026-07-04)
|
||||
- Production-readiness audit + P0-P3 roadmap: `$SECOND_BRAIN_VAULT/projects/qbit-quick-search.md`
|
||||
|
||||
Note: the dossier was reviewed before the qqq 4.0 re-pin; trust this file for build and dependency facts.
|
||||
@@ -1,86 +1,60 @@
|
||||
# QBit: Quick Search
|
||||
|
||||
[](https://github.com/QRun-IO/qbit-quick-search)
|
||||
[](https://github.com/QRun-IO/qbit-quick-search)
|
||||
[](https://www.apache.org/licenses/LICENSE-2.0)
|
||||
[](https://adoptium.net/)
|
||||
|
||||
> **OpenSearch-Powered Full-Text Search for QQQ Applications**
|
||||
> OpenSearch-backed full-text search across the tables of a QQQ application.
|
||||
|
||||
This QBit provides global search across QQQ application tables via OpenSearch. Annotate entity classes or declare tables via config, configure a connection, produce the QBit, and gain real-time indexing, scheduled basepull catch-up, per-field relevance boosting, and a pagination-aware search API.
|
||||
Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
|
||||
|
||||
## Core Capabilities
|
||||
Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services).
|
||||
|
||||
- **Real-Time Indexing**: Post-insert, post-update, and post-delete customizers index records immediately on write
|
||||
- **Basepull Safety Net**: Scheduled process catches records missed during downtime or bulk imports
|
||||
- **Field-Level Boosting**: `@QuickSearchField(weight = N)` controls relevance ranking per field
|
||||
- **Edge-Ngram Typeahead**: Custom analyzer enables prefix matching ("ord" finds "order")
|
||||
- **Paginated Search**: `QuickSearchAction` returns results with scores, highlights, total hits, and hasMore flag
|
||||
- **Config-Driven Indexing**: `SearchableTableConfig` API indexes tables from third-party QBit jars without modifying their source
|
||||
- **Extensible Publisher**: `IndexEventPublisher` interface allows swapping in RabbitMQ, Kafka, or any async transport
|
||||
- **Admin Observability**: Operational tables track index status and run history
|
||||
## What it does
|
||||
|
||||
## Open Source & Full Control
|
||||
| Capability | How |
|
||||
|---|---|
|
||||
| Real-time indexing | One `RecordChangeListenerInterface` registered on the instance; events are published after the write's transaction commits, so rolled-back writes never reach the index. Failures are recorded in `quickSearchFailedEvent` and replayed by the scheduled basepull. |
|
||||
| Basepull catch-up | Scheduled process (`schedulerName` plus `basepullRepeatSeconds`) that re-indexes rows changed since the previous run's start, with a configurable overlap, paging by primary key. |
|
||||
| Reconcile | Scheduled or on-demand process that re-indexes everything in place, then removes documents whose source row is gone, including documents of tables no longer configured. No search blackout. |
|
||||
| Full rebuild | Indexes every table into a fresh physical index and atomically swaps the alias; the old index keeps serving until the new one is complete. Also upgrades an index created by an older release. |
|
||||
| Search | `QuickSearchAction`: AND semantics, edge-ngram typeahead with a plain search analyzer, per-field weights, highlights, `tableNames` and `limitPerTable`, bounded paging, `tableLabel` on results. Only tables the session may read are searched; hits are re-read through `QueryAction` so record security locks apply. |
|
||||
| Connections | HTTP or HTTPS, basic auth, AWS IAM SigV4, custom CA or trust store, mTLS, timeouts, pool size, a transport customizer SPI for anything else. Secrets by `${env.X}` reference. |
|
||||
| Operations | `quickSearchIndex` (per-table status, counts, last run, last error), `quickSearchIndexRun` (history, purged after `runHistoryRetentionDays`), `quickSearchFailedEvent`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
|
||||
|
||||
QBit Quick Search is 100% open source under Apache 2.0. All data stays in your OpenSearch cluster.
|
||||
## Supported platforms
|
||||
|
||||
## Architecture
|
||||
Tested in CI against the Docker images named in `pom.xml` (`opensearch.test.image`).
|
||||
|
||||
### Design Principles
|
||||
| Target | Status |
|
||||
|---|---|
|
||||
| Self-managed OpenSearch 2.19.x and 3.x | Supported, security plugin disabled or HTTP basic authentication, HTTP or HTTPS with a trusted, configured or (development only) unverified certificate |
|
||||
| Amazon OpenSearch Service managed domains, OpenSearch 2.x or 3.x | Supported with fine-grained access control: either an internal-database master user with `authMode BASIC`, or IAM with `authMode AWS_SIGV4` (`awsRegion`, `awsServiceName es`, optional `awsAssumeRoleArn`). Port 443 over HTTPS. Keep `bulkBatchSize` and `maxBulkRequestBytes` under the instance type's 10 MiB payload limit on small instances. |
|
||||
| Amazon OpenSearch Serverless | Not supported in 1.0 (`_delete_by_query` and `_refresh`, used by reconcile and full rebuild, are not offered there) |
|
||||
| Elasticsearch, OpenSearch 1.x | Not supported |
|
||||
|
||||
1. **Real-time first, basepull as safety net**: Table customizers index records synchronously on every insert, update, and delete. The basepull process runs on a schedule as a catch-up mechanism for bulk imports, OpenSearch downtime, or any records the real-time path missed.
|
||||
Least-privilege role for the identity the QBit uses, on the alias and its physical indexes (`{indexName}*`): `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*`, `indices:admin/aliases`, `indices:admin/delete` (for the alias swap) and `indices:admin/mapping/get`; cluster level `cluster_composite_ops` for `_bulk`. For AWS fine-grained access control map the same to the basic-auth user or the IAM role. Treat the index as sensitive data: it holds copies of every indexed field value.
|
||||
|
||||
2. **Annotation and config-driven discovery**: Entity classes marked with `@QuickSearchable` and `@QuickSearchField` are discovered at produce time. Tables from third-party jars can be declared via `SearchableTableConfig` in code. Both paths merge during `produce()`.
|
||||
For `AWS_SIGV4` add these optional dependencies to your host (the QBit declares them `optional`):
|
||||
|
||||
3. **Extensible transport**: The `IndexEventPublisher` strategy interface decouples the write path from the indexing path. The default `SynchronousIndexEventPublisher` writes directly to OpenSearch. Consumers can provide a queue-backed implementation for async indexing.
|
||||
|
||||
### Technology Stack
|
||||
|
||||
- **Java 21** with QQQ backend modules
|
||||
- **OpenSearch 2.x** for full-text search with custom edge-ngram analyzer
|
||||
- **QQQ Framework**: Entities, processes, customizers, permissions, API layer
|
||||
|
||||
### Module Organization
|
||||
|
||||
```
|
||||
qbit-quick-search/
|
||||
src/main/java/com/kingsrook/qbits/quicksearch/
|
||||
QuickSearchQBitConfig.java -- Configuration and validation
|
||||
QuickSearchQBitProducer.java -- Produces tables, processes, customizers
|
||||
QuickSearchQBitContext.java -- Static runtime context
|
||||
QuickSearchableTableConfig.java -- Per-table index configuration
|
||||
SearchableFieldConfig.java -- Config-driven field definition
|
||||
SearchableTableConfig.java -- Config-driven table definition
|
||||
annotations/ -- @QuickSearchable, @QuickSearchField
|
||||
model/ -- QuickSearchIndex, QuickSearchIndexRun entities
|
||||
opensearch/ -- OpenSearch client, document model, bulk results
|
||||
actions/ -- QuickSearchAction, input/output DTOs
|
||||
processes/ -- AbstractIndexingStep, BasepullIndexStep, FullReindexStep
|
||||
customizers/ -- Post-insert, post-update, post-delete customizers
|
||||
publisher/ -- IndexEventPublisher interface, SynchronousIndexEventPublisher
|
||||
```xml
|
||||
<dependency><groupId>software.amazon.awssdk</groupId><artifactId>apache-client</artifactId></dependency>
|
||||
<dependency><groupId>software.amazon.awssdk</groupId><artifactId>sts</artifactId></dependency> <!-- only for awsAssumeRoleArn -->
|
||||
```
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- **Java 21+**
|
||||
- **Maven 3.8+**
|
||||
- **OpenSearch 2.x** instance
|
||||
- **QQQ Application** (this is a QBit, not a standalone application)
|
||||
|
||||
### Usage
|
||||
|
||||
#### Maven dependency
|
||||
## Getting started
|
||||
|
||||
```xml
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-quick-search</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>1.0.0-RC.1</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
#### Annotate entity classes
|
||||
`1.0.0-RC.1` is the current release candidate; `1.0.0` follows once host applications have adopted it (#12).
|
||||
|
||||
Annotate entity classes:
|
||||
|
||||
```java
|
||||
@QuickSearchable(tableName = "customer")
|
||||
@@ -97,265 +71,126 @@ public class Customer
|
||||
}
|
||||
```
|
||||
|
||||
#### Minimal setup
|
||||
Produce the QBit (after your backends, tables and scheduler are in the instance):
|
||||
|
||||
```java
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName("yourBackendName")
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withBackendName("rdbms")
|
||||
.withOpensearchUrl("https://search.example.com:443")
|
||||
.withAuthMode(QuickSearchAuthMode.BASIC)
|
||||
.withOpensearchUsername("${env.OPENSEARCH_USER}")
|
||||
.withOpensearchPassword("${env.OPENSEARCH_PASSWORD}")
|
||||
.withOpensearchIndexName("my-app-search")
|
||||
.withSchedulerName("myScheduler")
|
||||
.withSearchableEntityClasses(List.of(Customer.class, Order.class));
|
||||
|
||||
new QuickSearchQBitProducer()
|
||||
.withConfig(config)
|
||||
.produce(qInstance);
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
```
|
||||
|
||||
#### With authentication and SSL
|
||||
The producer implements QQQ's `QBitMetaDataProducer`, registers its own `QBitMetaData` (`com.kingsrook.qbits:quick-search`), and adds its tables, processes and app to the instance. Do not add a `QBitMetaData` for it yourself.
|
||||
|
||||
Tables from other QBits or jars, without annotations:
|
||||
|
||||
```java
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName("yourBackendName")
|
||||
.withOpensearchHost("search.example.com")
|
||||
.withOpensearchPort(443)
|
||||
.withUseSsl(true)
|
||||
.withOpensearchUsername("admin")
|
||||
.withOpensearchPassword("secretPassword")
|
||||
.withOpensearchIndexName("my-app-search")
|
||||
.withSearchableEntityClasses(List.of(Customer.class));
|
||||
```
|
||||
|
||||
#### With custom publisher (async indexing)
|
||||
|
||||
```java
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName("yourBackendName")
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("my-app-search")
|
||||
.withSearchableEntityClasses(List.of(Customer.class))
|
||||
.withIndexEventPublisher(new RabbitMqIndexEventPublisher(connectionFactory));
|
||||
```
|
||||
|
||||
#### Config-driven table indexing
|
||||
|
||||
When tables come from third-party QBit jars (like qbit-crm or qbit-wms), you cannot add annotations to their entity classes. Use `SearchableTableConfig` and `SearchableFieldConfig` to declare those tables as searchable in your host application config.
|
||||
|
||||
```java
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName("rdbms")
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("voyage-search")
|
||||
.withSearchableTable("crmContact", List.of(
|
||||
config.withSearchableTable(new SearchableTableConfig("crmContact", List.of(
|
||||
new SearchableFieldConfig("firstName").withWeight(2),
|
||||
new SearchableFieldConfig("lastName").withWeight(2),
|
||||
new SearchableFieldConfig("email").withWeight(3),
|
||||
new SearchableFieldConfig("phone").withWeight(1)))
|
||||
.withSearchableTable(new SearchableTableConfig("crmFormSubmission", List.of(
|
||||
new SearchableFieldConfig("email").withWeight(3),
|
||||
new SearchableFieldConfig("firstName").withWeight(1),
|
||||
new SearchableFieldConfig("message").withWeight(1)))
|
||||
.withRecordLabelFormat("%s - %s", "formType", "email"));
|
||||
new SearchableFieldConfig("email").withWeight(3)))
|
||||
.withRecordLabelFormat("%s %s", "firstName", "lastName"));
|
||||
```
|
||||
|
||||
`withSearchableTable()` accepts either a `(String tableName, List<SearchableFieldConfig> fields)` shorthand or a full `SearchableTableConfig` for advanced options.
|
||||
Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull.
|
||||
|
||||
##### SearchableFieldConfig
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `new SearchableFieldConfig(fieldName)` | Constructor; QQQ field name is required |
|
||||
| `.withWeight(int)` | Boost factor for relevance ranking (default 1) |
|
||||
| `.withIncludeLabel(boolean)` | Prefix indexed text with the field label (default false) |
|
||||
|
||||
##### SearchableTableConfig
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `new SearchableTableConfig(tableName, fields)` | Constructor; table name and at least one field required |
|
||||
| `.withBasepullIntervalMinutes(int)` | Override basepull schedule (defaults to config-level setting) |
|
||||
| `.withBasepullTimestampField(String)` | Change detection field (default `"modifyDate"`) |
|
||||
| `.withEnabledByDefault(boolean)` | Whether indexing starts active (default `true`) |
|
||||
| `.withRecordLabelFormat(String format, String... fields)` | `String.format` pattern and field names for search result display label |
|
||||
|
||||
##### Record label override
|
||||
|
||||
By default, search results use QQQ's record label for display. Config-driven tables can override this with `withRecordLabelFormat()`. The format string is passed to `String.format()` with field values pulled from the record in the order specified.
|
||||
### Searching
|
||||
|
||||
```java
|
||||
new SearchableTableConfig("crmFormSubmission", fields)
|
||||
.withRecordLabelFormat("%s - %s", "formType", "email");
|
||||
// produces labels like "Contact Form - user@example.com"
|
||||
```
|
||||
|
||||
#### Mixing annotations and config
|
||||
|
||||
Annotations and config-driven declarations can be used together. Both paths merge during `produce()`. Each table must appear in only one source; duplicate table names across annotation and config sources will produce a validation error.
|
||||
|
||||
```java
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName("rdbms")
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("my-app-search")
|
||||
.withSearchableEntityClasses(List.of(Customer.class, Order.class))
|
||||
.withSearchableTable("crmContact", List.of(
|
||||
new SearchableFieldConfig("firstName").withWeight(2),
|
||||
new SearchableFieldConfig("lastName").withWeight(2),
|
||||
new SearchableFieldConfig("email").withWeight(3)));
|
||||
```
|
||||
|
||||
## Searching
|
||||
|
||||
Once the QBit is produced, use `QuickSearchAction` to execute searches:
|
||||
|
||||
```java
|
||||
QuickSearchOutput output = new QuickSearchAction().execute(
|
||||
new QuickSearchInput()
|
||||
.withSearchTerm("widget")
|
||||
.withLimit(25)
|
||||
.withOffset(0));
|
||||
|
||||
// Check results
|
||||
System.out.println("Total hits: " + output.getTotalHits());
|
||||
System.out.println("Has more: " + output.getHasMore());
|
||||
QuickSearchOutput output = new QuickSearchAction().execute(new QuickSearchInput()
|
||||
.withSearchTerm("widget")
|
||||
.withTableNames(List.of("customer", "order")) // optional
|
||||
.withLimit(25)
|
||||
.withOffset(0));
|
||||
|
||||
for(QuickSearchResult result : output.getResults())
|
||||
{
|
||||
System.out.println(result.getTableName() + ":" + result.getRecordId()
|
||||
+ " - " + result.getRecordLabel()
|
||||
+ " (score: " + result.getScore() + ")");
|
||||
result.getTableName(); result.getTableLabel(); result.getRecordId();
|
||||
result.getRecordLabel(); result.getScore(); result.getHighlightSnippet();
|
||||
}
|
||||
output.getTotalHits(); output.getTotalHitsIsLowerBound(); output.getHasMore();
|
||||
```
|
||||
|
||||
To filter results to a single table:
|
||||
`limitPerTable` returns up to that many hits from each table instead of one ranked list. Terms shorter than 2 characters return nothing; longer than 100 characters are rejected. `limit` is capped at `maxSearchLimit` and `offset + limit` at 10,000.
|
||||
|
||||
```java
|
||||
QuickSearchOutput output = new QuickSearchAction().execute(
|
||||
new QuickSearchInput()
|
||||
.withSearchTerm("blue")
|
||||
.withTableName("customer")
|
||||
.withLimit(10));
|
||||
```
|
||||
`offset` skips N accessible results: with record security locks on, hits a lock hides are not counted, so consecutive pages never overlap or skip a visible hit, and with `limitPerTable` the offset applies within each table. With locks on, `totalHits` counts hits confirmed accessible so far; `totalHitsIsLowerBound` is true when the scan stopped before reading every candidate (page full, result window, or scan budget). A single-list search reads at most 10,000 raw hits in batches of at most 1,000. In per-table mode the T tables share that budget, each getting max(10,000 / T, 2 x `limitPerTable`) hits, so one call reads at most max(10,000, T x 2 x `limitPerTable`) raw hits. Paging within a table stops past that share of raw hits, as it does past 10,000 in a single list: `hasMore` turns false and `totalHitsIsLowerBound` stays true to signal unreachable matches.
|
||||
|
||||
### Triggering a Full Reindex
|
||||
Search runs as the current `QContext` session: tables the session cannot read are skipped, and with `applyRecordSecurityLocks` (default) hits are re-read through `QueryAction` so record security locks apply.
|
||||
|
||||
The full reindex process can be triggered programmatically:
|
||||
### Processes
|
||||
|
||||
```java
|
||||
RunBackendStepInput input = new RunBackendStepInput();
|
||||
input.addValue("tableName", "customer"); // optional: omit to reindex all tables
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep().run(input, output);
|
||||
```
|
||||
| Process | Trigger | What it does |
|
||||
|---|---|---|
|
||||
| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
|
||||
| Quick Search Reconcile Index | Cron via `reconcileCronExpression` when `schedulerName` is set; also on demand (optional `tableName` input) | Re-indexes everything in place, removes documents without a source row and documents of unconfigured tables |
|
||||
| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place |
|
||||
|
||||
Or through the QQQ admin UI via the "Full Reindex" process.
|
||||
Without `schedulerName` the processes exist but nothing runs them; the QBit logs a warning at startup.
|
||||
|
||||
## Data Model
|
||||
## Configuration reference
|
||||
|
||||
### Tables (2)
|
||||
| Field | Default | Description |
|
||||
|---|---|---|
|
||||
| `backendName` | required | QQQ backend for the operational tables |
|
||||
| `opensearchUrl` | | `https://host[:port]`; preferred over `opensearchHost`, `opensearchPort`, `useSsl`; credentials in the URL are rejected |
|
||||
| `opensearchHost`, `opensearchPort`, `useSsl` | `useSsl=false` | Legacy connection fields |
|
||||
| `opensearchIndexName` | required | Alias name owned by this QBit; lowercase, OpenSearch naming rules |
|
||||
| `authMode` | inferred | `NONE`, `BASIC` (inferred when credentials are set), `AWS_SIGV4` |
|
||||
| `opensearchUsername`, `opensearchPassword` | | BASIC credentials; `${env.X}` references recommended (an empty value counts as missing); refused over plain HTTP unless `allowPlaintextCredentials` |
|
||||
| `awsRegion`, `awsServiceName`, `awsAssumeRoleArn` | region from `AWS_REGION`; `es` | AWS_SIGV4 settings |
|
||||
| `tls` | JVM defaults | `caCertificatePath` or `trustStorePath`/`trustStorePassword`/`trustStoreType`, `keyStorePath`/`keyStorePassword`/`keyStoreType` (mTLS), `hostnameVerification` (`false` logs a warning), `insecureSkipVerify` (loopback only unless `allowInsecureInProduction`) |
|
||||
| `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections` | 5000, 60000, 30 | Transport settings |
|
||||
| `transportCustomizer` | | `QCodeReference` to an `OpenSearchTransportCustomizer` (bearer tokens, API keys, interceptors) |
|
||||
| `startupMode` | `FAIL_FAST` | `FAIL_FAST` fails `produce()` when the cluster is unreachable; `DEGRADED` boots and retries on first use |
|
||||
| `enableRealTimeIndexing` | `true` | Register the record-change listener |
|
||||
| `enableBasepullProcess`, `enableMaintenanceProcesses` | `true` | Register basepull; register full reindex and reconcile. `enableScheduledProcesses` is a deprecated alias for both |
|
||||
| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none | Scheduling |
|
||||
| `basepullOverlapSeconds` | 300 | Re-read window before the previous run start |
|
||||
| `defaultBasepullIntervalMinutes` | 5 | Per-table due interval for config-driven tables |
|
||||
| `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
|
||||
| `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
|
||||
| `maxFieldLength` | 10000 | Truncate each indexed value |
|
||||
| `maxSearchLimit` | 100 | Largest page size |
|
||||
| `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
|
||||
| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchAdmin` | Rules for the app and processes |
|
||||
| `tablePermissionRules` | `READ_WRITE_PERMISSIONS` | Rules for the operational tables |
|
||||
| `tableMetaDataCustomizer` | | Applied to every produced table (backend details, for example) |
|
||||
| `searchableEntityClasses`, `searchableTables` | at least one | What to index |
|
||||
| `indexEventPublisher` | synchronous | Replace the publisher (experimental SPI; see below) |
|
||||
|
||||
| Table | Description |
|
||||
|-------|-------------|
|
||||
| `quickSearchIndex` | Per-table index configuration, status, and basepull tracking |
|
||||
| `quickSearchIndexRun` | Individual indexing run history with record counts and error tracking |
|
||||
`@QuickSearchable(tableName, fields, basepullIntervalMinutes, basepullTimestampField, enabledByDefault)` and `@QuickSearchField(weight, includeLabel)` keep their 0.x meaning.
|
||||
|
||||
### Processes (2)
|
||||
## Operational tables
|
||||
|
||||
| Process | Description |
|
||||
|---------|-------------|
|
||||
| Basepull Index | Scheduled: queries each source table for records modified since last run, indexes in batches |
|
||||
| Full Reindex | On-demand: deletes and re-indexes all records for one or all tables |
|
||||
- `quickSearchIndex`: one row per table with `enabled` (honoured by indexing and search, cached for one minute), `basepullIntervalMinutes`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`, `NEEDS_REINDEX` when the configured fields changed since the row was created, or `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
|
||||
- `quickSearchIndexRun`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status`, counts, `errorMessage`.
|
||||
- `quickSearchFailedEvent`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. If a full reindex is killed mid-run, its tables stay `REBUILDING` and deletes keep collecting as `AWAITING_REINDEX`; no data is lost, and the recovery is to run a full reindex of all tables again, which applies the collected deletes and returns the tables to `ACTIVE`.
|
||||
|
||||
## Configuration Reference
|
||||
## Extending
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
|-------|------|---------|-------------|
|
||||
| `backendName` | `String` | required | QQQ backend for operational tables |
|
||||
| `opensearchHost` | `String` | required | OpenSearch hostname |
|
||||
| `opensearchPort` | `Integer` | required | OpenSearch port |
|
||||
| `opensearchIndexName` | `String` | required | OpenSearch index name |
|
||||
| `searchableEntityClasses` | `List<Class<?>>` | none | Entity classes with `@QuickSearchable` |
|
||||
| `searchableTables` | `List<SearchableTableConfig>` | none | Config-driven table definitions |
|
||||
| `opensearchUsername` | `String` | none | HTTP basic auth username |
|
||||
| `opensearchPassword` | `String` | none | HTTP basic auth password |
|
||||
| `useSsl` | `Boolean` | `false` | Use HTTPS |
|
||||
| `tableNamePrefix` | `String` | none | Prefix for produced table names |
|
||||
| `enableRealTimeIndexing` | `Boolean` | `true` | Register post-insert/update/delete customizers |
|
||||
| `enableScheduledProcesses` | `Boolean` | `true` | Register scheduled basepull process |
|
||||
| `defaultBasepullIntervalMinutes` | `Integer` | `5` | Default basepull schedule |
|
||||
| `bulkBatchSize` | `Integer` | `500` | Documents per OpenSearch bulk request |
|
||||
| `sourceBatchSize` | `Integer` | `1000` | Records per source table query batch |
|
||||
| `indexEventPublisher` | `IndexEventPublisher` | sync | Custom publisher for async transport |
|
||||
`IndexEventPublisher` lets a host replace the synchronous publisher with a queue. In 1.0 the SPI is experimental: `IndexEvent` carries the table, record id, action and the record snapshot, but no sequence number, so an asynchronous consumer must order INDEX and DELETE for the same record itself. `OpenSearchTransportCustomizer` customizes the HTTP transport.
|
||||
|
||||
At least one of `searchableEntityClasses` or `searchableTables` must be provided. Both can be used together.
|
||||
|
||||
## Annotations
|
||||
|
||||
### `@QuickSearchable`
|
||||
|
||||
| Attribute | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `tableName` | `String` | required | QQQ table name |
|
||||
| `fields` | `String[]` | `{}` | Fallback field list if no `@QuickSearchField` annotations |
|
||||
| `basepullIntervalMinutes` | `int` | `5` | Basepull schedule for this table |
|
||||
| `basepullTimestampField` | `String` | `"modifyDate"` | Field for change detection |
|
||||
| `enabledByDefault` | `boolean` | `true` | Active on first produce |
|
||||
|
||||
### `@QuickSearchField`
|
||||
|
||||
| Attribute | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `weight` | `int` | `1` | Boost factor for search relevance |
|
||||
| `includeLabel` | `boolean` | `false` | Prefix value with field label in indexed text |
|
||||
|
||||
## Companion QBit Integration
|
||||
|
||||
### qbit-crm
|
||||
|
||||
Index `crmContact`, `crmCompany`, and `crmDeal` for full-text global search across CRM records.
|
||||
|
||||
### qbit-wms
|
||||
|
||||
Index `wmsItem` and `wmsLocation` for warehouse item and location search.
|
||||
|
||||
## Testing
|
||||
## Building
|
||||
|
||||
```bash
|
||||
mvn test # Run all 174 unit tests
|
||||
mvn verify # Run unit tests + integration tests (requires Docker)
|
||||
mvn test -Dtest=ClassName # Run single test class
|
||||
./mvnw test # unit tests (memory backend, mocked client)
|
||||
CI=true ./mvnw verify # plus Testcontainers integration tests and the coverage gate (Docker required)
|
||||
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
|
||||
```
|
||||
|
||||
### Coverage
|
||||
Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.1 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
|
||||
|
||||
- **70%+ instruction coverage** (JaCoCo enforced)
|
||||
- **90%+ class coverage** (JaCoCo enforced)
|
||||
- All unit tests run in-memory via MemoryRecordStore (no OpenSearch required)
|
||||
- Integration tests use Testcontainers with real OpenSearch
|
||||
## Upgrading from 0.x
|
||||
|
||||
## Documentation
|
||||
|
||||
- **[QQQ Wiki](https://github.com/Kingsrook/qqq/wiki)** - Framework documentation
|
||||
- **[QBit Development Guide](https://github.com/Kingsrook/qqq/wiki/QBit-Development)** - How QBits work
|
||||
|
||||
## Contributing
|
||||
|
||||
QBit Quick Search is open source and welcomes contributions.
|
||||
|
||||
- **[Report Issues](https://github.com/QRun-IO/qbit-quick-search/issues)** - Bug reports and feature requests
|
||||
- **[QQQ Contribution Guide](https://github.com/Kingsrook/qqq/wiki/Contribution-Guidelines)** - How to contribute
|
||||
|
||||
## About Kingsrook
|
||||
|
||||
QBit Quick Search is built by **[Kingsrook](https://qrun.io)** - making engineers more productive through intelligent automation and developer tools.
|
||||
|
||||
- **Website**: [https://qrun.io](https://qrun.io)
|
||||
- **Contact**: [contact@kingsrook.com](mailto:contact@kingsrook.com)
|
||||
- **GitHub**: [https://github.com/QRun-IO](https://github.com/QRun-IO)
|
||||
See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, and run a full reindex once so the index gets the new mapping.
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the **Apache License, Version 2.0** - see the [LICENSE](LICENSE) file for details.
|
||||
Apache License, Version 2.0. See [LICENSE](LICENSE).
|
||||
+9
-10
@@ -6,18 +6,17 @@
|
||||
~ contact@kingsrook.com
|
||||
~ https://github.com/Kingsrook/
|
||||
~
|
||||
~ This program is free software: you can redistribute it and/or modify
|
||||
~ it under the terms of the GNU Affero General Public License as
|
||||
~ published by the Free Software Foundation, either version 3 of the
|
||||
~ License, or (at your option) any later version.
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ This program is distributed in the hope that it will be useful,
|
||||
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
~ GNU Affero General Public License for more details.
|
||||
~ https://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ You should have received a copy of the GNU Affero General Public License
|
||||
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<!DOCTYPE module PUBLIC
|
||||
|
||||
+9
-10
@@ -5,16 +5,15 @@
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
@@ -1,288 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2024. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.actions.customizers;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Common interface used by all (core) TableCustomizer types (e.g., post-query,
|
||||
** and {pre,post}-{insert,update,delete}.
|
||||
**
|
||||
** Note that the abstract-base classes for each action still exist, though have
|
||||
** been back-ported to be implementors of this interface. The action classes
|
||||
** will now expect this type, and call this type's methods.
|
||||
**
|
||||
*******************************************************************************/
|
||||
public interface TableCustomizerInterface
|
||||
{
|
||||
QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
|
||||
|
||||
/*******************************************************************************
|
||||
** custom actions to run after a query (or get!) takes place.
|
||||
**
|
||||
*******************************************************************************/
|
||||
default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
|
||||
{
|
||||
LOG.info("A default implementation of postQuery is running... Probably not expected!", logPair("tableName", queryInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** custom actions before an insert takes place.
|
||||
**
|
||||
** It's important for implementations to be aware of the isPreview field, which
|
||||
** is set to true when the code is running to give users advice, e.g., on a review
|
||||
** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing
|
||||
** things like storing data, you don't want to do that if isPreview is true!!
|
||||
**
|
||||
** General implementation would be, to iterate over the records (the inputs to
|
||||
** the insert action), and look at their values:
|
||||
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
|
||||
** - possibly manipulating values (`setValue`)
|
||||
** - possibly throwing an exception - if you really don't want the insert operation to continue.
|
||||
** - doing "whatever else" you may want to do.
|
||||
** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
|
||||
*******************************************************************************/
|
||||
default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (preInsertOrUpdate(insertInput, records, isPreview, Optional.empty()));
|
||||
}
|
||||
catch(NotImplementedHereException e)
|
||||
{
|
||||
LOG.info("A default implementation of preInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
|
||||
{
|
||||
return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** custom actions after an insert takes place.
|
||||
**
|
||||
** General implementation would be, to iterate over the records (the outputs of
|
||||
** the insert action), and look at their values:
|
||||
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
|
||||
** - possibly throwing an exception - though doing so won't stop the update, and instead
|
||||
** will just set a warning on all of the updated records...
|
||||
** - doing "whatever else" you may want to do.
|
||||
** - returning the list of records (can be the input list) that you want to go back to the caller.
|
||||
*******************************************************************************/
|
||||
default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (postInsertOrUpdate(insertInput, records, Optional.empty()));
|
||||
}
|
||||
catch(NotImplementedHereException e)
|
||||
{
|
||||
LOG.info("A default implementation of postInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** custom actions before an update takes place.
|
||||
**
|
||||
** It's important for implementations to be aware of the isPreview field, which
|
||||
** is set to true when the code is running to give users advice, e.g., on a review
|
||||
** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing
|
||||
** things like storing data, you don't want to do that if isPreview is true!!
|
||||
**
|
||||
** General implementation would be, to iterate over the records (the inputs to
|
||||
** the update action), and look at their values:
|
||||
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
|
||||
** - possibly manipulating values (`setValue`)
|
||||
** - possibly throwing an exception - if you really don't want the update operation to continue.
|
||||
** - doing "whatever else" you may want to do.
|
||||
** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
|
||||
**
|
||||
** Note, "old records" (e.g., with values freshly fetched from the backend) will be
|
||||
** available (if the backend supports it)
|
||||
*******************************************************************************/
|
||||
default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (preInsertOrUpdate(updateInput, records, isPreview, oldRecordList));
|
||||
}
|
||||
catch(NotImplementedHereException e)
|
||||
{
|
||||
LOG.info("A default implementation of preUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** custom actions after an update takes place.
|
||||
**
|
||||
** General implementation would be, to iterate over the records (the outputs of
|
||||
** the update action), and look at their values:
|
||||
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
|
||||
** - possibly throwing an exception - though doing so won't stop the update, and instead
|
||||
** will just set a warning on all of the updated records...
|
||||
** - doing "whatever else" you may want to do.
|
||||
** - returning the list of records (can be the input list) that you want to go back to the caller.
|
||||
**
|
||||
** Note, "old records" (e.g., with values freshly fetched from the backend) will be
|
||||
** available (if the backend supports it).
|
||||
*******************************************************************************/
|
||||
default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (postInsertOrUpdate(updateInput, records, oldRecordList));
|
||||
}
|
||||
catch(NotImplementedHereException e)
|
||||
{
|
||||
LOG.info("A default implementation of postUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Custom actions before a delete takes place.
|
||||
**
|
||||
** It's important for implementations to be aware of the isPreview param, which
|
||||
** is set to true when the code is running to give users advice, e.g., on a review
|
||||
** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing
|
||||
** things like storing data, you don't want to do that if isPreview is true!!
|
||||
**
|
||||
** General implementation would be, to iterate over the records (which the DeleteAction
|
||||
** would look up based on the inputs to the delete action), and look at their values:
|
||||
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
|
||||
** - possibly throwing an exception - if you really don't want the delete operation to continue.
|
||||
** - doing "whatever else" you may want to do.
|
||||
** - returning the list of records (can be the input list) - this is how errors
|
||||
** and warnings are propagated to the DeleteAction. Note that any records with
|
||||
** an error will NOT proceed to the backend's delete interface - but those with
|
||||
** warnings will.
|
||||
*******************************************************************************/
|
||||
default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
|
||||
{
|
||||
LOG.info("A default implementation of preDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Custom actions after a delete takes place.
|
||||
**
|
||||
** General implementation would be, to iterate over the records (ones which didn't
|
||||
** have a delete error), and look at their values:
|
||||
** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
|
||||
** - possibly throwing an exception - though doing so won't stop the delete, and instead
|
||||
** will just set a warning on all of the deleted records...
|
||||
** - doing "whatever else" you may want to do.
|
||||
** - returning the list of records (can be the input list) that you want to go back
|
||||
** to the caller - this is how errors and warnings are propagated .
|
||||
*******************************************************************************/
|
||||
default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
|
||||
{
|
||||
LOG.info("A default implementation of postDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName()));
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Optional method to override in a customizer that does the same thing for
|
||||
** both preInsert & preUpdate.
|
||||
***************************************************************************/
|
||||
default List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
|
||||
{
|
||||
throw NotImplementedHereException.instance;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Optional method to override in a customizer that does the same thing for
|
||||
** both postInsert & postUpdate.
|
||||
***************************************************************************/
|
||||
default List<QRecord> postInsertOrUpdate(AbstractActionInput input, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
|
||||
{
|
||||
throw NotImplementedHereException.instance;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
default Optional<Map<Serializable, QRecord>> oldRecordListToMap(String primaryKeyField, Optional<List<QRecord>> oldRecordList)
|
||||
{
|
||||
if(oldRecordList.isPresent())
|
||||
{
|
||||
return (Optional.of(CollectionUtils.listToMap(oldRecordList.get(), r -> r.getValue(primaryKeyField))));
|
||||
}
|
||||
else
|
||||
{
|
||||
return (Optional.empty());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
class NotImplementedHereException extends QException
|
||||
{
|
||||
private static NotImplementedHereException instance = new NotImplementedHereException();
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private NotImplementedHereException()
|
||||
{
|
||||
super("Not implemented here");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Interface to be implemented by enums of the possible supplemental types
|
||||
**
|
||||
*******************************************************************************/
|
||||
public interface SupplementalCustomizerType
|
||||
{
|
||||
/*******************************************************************************
|
||||
** allows implementing enums to return the expected class being used
|
||||
*******************************************************************************/
|
||||
public Class<?> getCodeReferenceType();
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,33 @@
|
||||
# Migrating from 0.x to 1.0
|
||||
|
||||
1.0 is a trust and platform release. Hosts that used the 0.2 fluent API keep compiling; the steps below are what changes at runtime.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- QQQ 4.1 (the QBit uses record-change listeners, after-commit callbacks and runtime services, none of which exist in 4.0). Until `qbit-build-parent` 2.1.0 is released, the QBit builds against 4.1.0-RC.1.
|
||||
- OpenSearch 2.19.x or 3.x, or an Amazon OpenSearch Service managed domain. Serverless is not supported.
|
||||
|
||||
## Required changes
|
||||
|
||||
1. **Remove any manual `QBitMetaData` registration.** The producer now registers `com.kingsrook.qbits:quick-search` itself; a second `addQBit` with that name throws at startup. (qqq-all's `DemoQBits` did this in 0.x.)
|
||||
2. **Decide the startup mode.** `startupMode` defaults to `FAIL_FAST`: `produce()` fails when OpenSearch is unreachable. Demo profiles that may run without a cluster should set `withStartupMode(QuickSearchStartupMode.DEGRADED)`.
|
||||
3. **Run a full reindex once after upgrading.** The index mapping changed (version 2: search analyzer, text-typed `fieldValues`). The QBit logs a warning when it finds an index with the old mapping; "Quick Search Full Reindex" with no table name builds a new physical index and swaps the configured name to an alias. Search keeps working during the rebuild. The identity the QBit uses needs index create, delete and alias privileges for this (see the README role).
|
||||
4. **Set `schedulerName`** (and optionally `basepullRepeatSeconds`, `reconcileCronExpression`) so basepull and reconcile actually run. In 0.x they were never scheduled.
|
||||
5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `allowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
|
||||
6. **Permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant them to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
|
||||
|
||||
## Behaviour changes to be aware of
|
||||
|
||||
- Real-time indexing happens after commit. Code that searched for a record inside the same transaction that inserted it will not find it until commit.
|
||||
- Real-time failures no longer vanish: they land in `quickSearchFailedEvent` and are replayed by the next basepull. Review that table after an OpenSearch outage.
|
||||
- Inserted and updated records are re-read inside the write's transaction, with display values and possible-value labels, before they are indexed. That is one extra query per write batch on an indexed table, and it runs the host's possible-value providers. If the re-read fails, the write still succeeds and the records are replayed by basepull.
|
||||
- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile.
|
||||
- `limit` is capped (default 100), `offset + limit` at 10,000, and search terms must be 2 to 100 characters.
|
||||
- Query semantics changed from OR over n-grams to AND over whole terms with prefix matching: "blue widget" now requires both words.
|
||||
- `enableScheduledProcesses` still works and sets both `enableBasepullProcess` and `enableMaintenanceProcesses`.
|
||||
- `QuickSearchQBitContext` is deprecated and removed in 1.1; use `QuickSearchRuntime.get()`.
|
||||
|
||||
## Removed
|
||||
|
||||
- `QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`. Hosts that referenced them (for example to compose with their own customizers) should remove those references; the record-change listener coexists with host customizers.
|
||||
- `QuickSearchOpenSearchClient.search(String term, String tableName, ...)`; use the collection overload.
|
||||
@@ -0,0 +1,50 @@
|
||||
# PLAN: Quick Search 1.0 release readiness
|
||||
|
||||
> Executed natively in-session, task by task, test-first, one commit per task on `feature/GH-1-0-release-readiness`. Spec: `docs/initiatives/quick-search-1.0.md` (PRD) and `docs/audits/2026-10-04/README.md` (blockers B1 to B25). Decisions D1 to D11 are resolved with the PRD's recommendations unless noted under "Decisions applied".
|
||||
|
||||
## Goal
|
||||
|
||||
Close every audit blocker and ship a QBit that is a standard QQQ 4.1 citizen, keeps a trustworthy index, searches safely, connects to real OpenSearch deployments, and documents all of it.
|
||||
|
||||
## Decisions applied
|
||||
|
||||
| ID | Resolution used here |
|
||||
|---|---|
|
||||
| D1 | API-only in 1.0; `QuickSearchResult` gains `tableLabel`, input gains `tableNames` and `limitPerTable`; the core SPI is proposed separately (TODO) |
|
||||
| D2 | 1.0.0-SNAPSHOT on this branch; 0.3.0 compat release remains the qqq#921 chain's job |
|
||||
| D3 | Table-level READ filter plus `QueryAction` post-filter by primary key (`applyRecordSecurityLocks`, default true) |
|
||||
| D4 | `schedulerName` null leaves processes unscheduled with a startup WARN (not an error), so qqq-all keeps booting; set it to schedule |
|
||||
| D5 | Full reindex of all tables builds a fresh physical index and swaps the alias; a single-table full reindex uses the reconcile algorithm (index in place, then delete stale) |
|
||||
| D6 | `authMode` `AWS_SIGV4` included with optional AWS SDK v2 dependencies |
|
||||
| D7 | `RecordChangeListenerInterface` plus `addAfterCommitCallback` |
|
||||
| D8 | Listener never fails the write; failures go to `quickSearchFailedEvent`, replayed by the scheduled basepull |
|
||||
| D9 | Supported platforms statement as in the audit |
|
||||
| D10 | 4.1-only: the build uses qqq 4.1.0-RC.1 through the (now default-active) `qqq-snapshot` profile until parent 2.1.0 ships; customizers are deleted |
|
||||
| D11 | `startupMode` default `FAIL_FAST`; `DEGRADED` available |
|
||||
|
||||
## Architecture
|
||||
|
||||
- `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`; component producers in the package build the two tables, three processes, failed-event table, table PVS and app; `postProduceActions` registers the record-change listener and the runtime service and resolves discovered tables.
|
||||
- `QuickSearchRuntime` (instance-scoped, held `transient` on the config, resolved through `QContext.getQInstance().getQBits()`) owns discovered tables, the lazily built client, the publisher, the enabled-table cache and lifecycle. `QuickSearchQBitContext` stays as a deprecated static facade delegating to it (tests keep working).
|
||||
- `QuickSearchRecordChangeListener` replaces the three customizers; publishes after commit when a transaction is present; records failures in `quickSearchFailedEvent`.
|
||||
- `QuickSearchOpenSearchClient` gains a transport factory (`NONE`, `BASIC`, `AWS_SIGV4`, TLS block, timeouts), mapping v2 (search analyzer, dynamic templates, `_meta` version), alias management, health, and wraps `OpenSearchException`.
|
||||
- Indexing steps page by primary key, anchor the basepull watermark at run start, honour `enabled`, sweep orphans, detect drift, purge run history, replay failed events, and use `external_gte` versioning from the timestamp field.
|
||||
- `QuickSearchAction` filters tables by permission and enabled state, bounds inputs, supports `tableNames` and `limitPerTable`, post-filters through `QueryAction`, returns `tableLabel`.
|
||||
|
||||
## Tasks (each ends with green `mvn test`, then a commit)
|
||||
|
||||
1. Build: default-active `qqq-snapshot` profile on `4.1.0-RC.1`, `revision` 1.0.0-SNAPSHOT, opensearch-java 3.10.0 with the `Refresh.True` fix, JaCoCo `prepare-agent` with explicit version, drop JUnit 5.10.0 pins (fallback: keep if JUnit 6 breaks a dependency). Files: `pom.xml`, `QuickSearchOpenSearchClient.java:427`.
|
||||
2. Config surface and runtime holder: new fields (`startupMode`, `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`, `enableBasepullProcess`, `enableMaintenanceProcesses`, `adminPermissionRules`, `tablePermissionRules`, `applyRecordSecurityLocks`, `maxSearchLimit`, `basepullOverlapSeconds`, `runHistoryRetentionDays`, `maxFieldLength`, `opensearchUrl`, `authMode`, `awsRegion`, `awsServiceName`, `awsAssumeRoleArn`, `allowPlaintextCredentials`, `tls*`, `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `transportCustomizer`), deprecated `enableScheduledProcesses` alias, null-safe validation with field-existence checks, `getDefaultBackendNameForTables`, settable `tableMetaDataCustomizer`, `@JsonIgnore` secret getters, redacting `toString`, `transient` publisher. `QuickSearchRuntime` class; `QuickSearchQBitContext` facade. Tests: `QuickSearchQBitConfigTest`, new `QuickSearchRuntimeTest`.
|
||||
3. Producer contract: `QBitMetaDataProducer`, `QBitMetaData` (artifactId `quick-search`, version from `quick-search.properties` filtered by Maven), component producers, unique key on `tableName`, permission rules, schedules, `QuickSearchRuntimeService`, `isEnabled` false without config, compat shims (`withConfig`, `getConfig`, `produce(QInstance)` adds and returns an empty output). Tests: `QuickSearchQBitProducerTest` updated plus new assertions (qbit registered, sourceQBitName, schedule, permission rules, unique key).
|
||||
4. Real-time path: `QuickSearchRecordChangeListener`, `quickSearchFailedEvent` table and entity, failed-event replay in basepull, delete customizers and their tests, new `QuickSearchRecordChangeListenerTest` (insert/update/delete, after-commit via a fake transaction, rollback discards, failure writes a failed-event row).
|
||||
5. Client: transport factory with auth modes, TLS, timeouts, user agent; secret interpretation via `QMetaDataVariableInterpreter`; mapping v2 with `_meta.mappingVersion`; `lenient` boosts and `operator=AND`; `OpenSearchException` wrapping; alias helpers (`createIndexForAlias`, `swapAlias`, `deleteIndex`, `resolvePhysicalIndex`); `health()`; `external_gte` versioning on bulk index; byte-capped bulk batches. Tests: `QuickSearchOpenSearchClientTest` (config to transport mapping, validation of modes), integration tests updated for mapping v2 and the "stella/stone" and mixed-type cases.
|
||||
6. Indexing steps: keyset paging in `AbstractIndexingStep`; basepull start-anchored watermark with overlap and `GREATER_THAN_OR_EQUALS`, `enabled` honoured, failures aggregated, run purge, failed-event replay; full reindex alias swap (all tables) or in-place (single table), error message on FAILED; reconcile orphan sweep and `enabled`; drift detection sets `status = NEEDS_REINDEX`; `IndexingUtils` display values, label fallback, hidden-field exclusion, truncation, safe label formatting. Tests: step tests updated plus new scenarios from audit 05.
|
||||
7. Search: `QuickSearchInput extends AbstractActionInput`, `tableNames`, `limitPerTable`, bounds, permission and enabled filtering, `QueryAction` post-filter, `tableLabel`, `totalHitsIsLowerBound`, `track_total_hits`. Tests: `QuickSearchActionTest` with a permission-denied session and bounds.
|
||||
8. Admin surface: process frontend steps with an optional `tableName` field backed by a discovered-tables PVS, admin table fields (`lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`, `realTimeErrorCount`), read-only tracking fields, run-to-index PVS, labels, icon, health process.
|
||||
9. Docs and CI: README rewrite, `CHANGELOG.md`, `docs/MIGRATION-1.0.md`, configuration reference, supported platforms and roles, parameterized Testcontainers image with a 2.19.6 and 3.9.0 matrix in CI, CLAUDE.md and AGENTS.md architecture updates.
|
||||
10. Close-out: full `mvn verify` with Docker, PRD and audit status columns, TODO and SESSION-STATE, second-brain save.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Parent 2.1.0 re-pin happens when qbit-bom publishes it; until then the default-active profile carries 4.1.0-RC.1 (recorded as a deviation from ADR-0007, to be reverted).
|
||||
- The qqq core `RecordSearchProviderInterface` proposal is filed as a follow-up, not implemented here.
|
||||
@@ -0,0 +1,31 @@
|
||||
# Session State
|
||||
|
||||
Updated: 2026-10-06
|
||||
|
||||
## Context
|
||||
|
||||
`1.0.0-RC.2` is released (2026-10-06). It fixes the five pre-GA findings from the PR #10 review, each on its own branch and PR into `develop`:
|
||||
|
||||
| Issue | PR | Merge commit | Fix |
|
||||
|---|---|---|---|
|
||||
| #15 | #18 | `2398e62` | Transport: warn on `hostnameVerification=false`, blank `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected |
|
||||
| #17 | #21 | `e9ec54a` | `safeCount` logs at warn and keeps the previous `documentCount` |
|
||||
| #16 | #19 | `9941a70` | Search `offset` counts readable results under record locks; bounded scan; 10,000 window cap |
|
||||
| #14 | #20 | `dce0f26` | Deletes during a full reindex captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap |
|
||||
| #13 | #22 | `c5c41d4` | Listener re-read failure and missing AWS SDK (`LinkageError`) recorded as failed events |
|
||||
|
||||
`develop` (`c5c41d4`) was merged into `release/1.0` as `b7d366c` (only difference from develop: `<revision>1.0.0-RC.1</revision>`). The orb's `publish_release_candidate` workflow succeeded on `b7d366c` (CircleCI build 82) and pushed `c11b2c2` ("Bump version to 1.0.0-RC.2 [skip ci]"). Maven Central serves the jar, pom, sources, javadoc and signatures for `com.kingsrook.qbits:qbit-quick-search:1.0.0-RC.2`.
|
||||
|
||||
## Release mechanics (qqq-orb 0.6.8, confirmed in its source)
|
||||
|
||||
- RCs publish artifacts only: no git tag and no GitHub release. The GitHub release comes when the release branch merges to `main` and a `vX.Y.Z` tag is pushed.
|
||||
- Never push a `vX.Y.Z-RC.n` tag: it matches the GA job's `v*` filter, maps back to `release/X.Y.Z`, and would publish a stray `RC.n+1`.
|
||||
- The next push to `release/1.0` publishes `1.0.0-RC.3`. Keep the release branch's `<revision>` on merge.
|
||||
- `develop` requires one approving review and the PR author cannot approve their own PR; James merges with `gh pr merge <n> --squash --admin`. The agent cannot run `gh pr merge`, and the auto-mode classifier also refused a background loop that polled PR merge state.
|
||||
- GitHub SSH times out from this machine; push over HTTPS with `git -c credential.helper='!gh auth git-credential' push https://github.com/QRun-IO/qbit-quick-search.git <ref>`.
|
||||
|
||||
## Next
|
||||
|
||||
1. qqq-all and Voyage pin `1.0.0-RC.2` and report back; their feedback decides RC.3 or GA.
|
||||
2. Remaining "Before 1.0.0 GA" items in `docs/TODO.md`: stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review), and the `qbit-build-parent` 2.1.0 re-pin.
|
||||
3. GA: merge `release/1.0` to `main` and tag `v1.0.0`.
|
||||
@@ -0,0 +1,40 @@
|
||||
# TODO
|
||||
|
||||
## Before tagging 1.0.0
|
||||
|
||||
- [ ] qqq-all and Voyage adopt `1.0.0-RC.2` (items below) and report back.
|
||||
|
||||
- [x] PR #10 (`feature/GH-1-0-release-readiness`) opened and reviewed 2026-10-05; `AGENTS.md` kept.
|
||||
- [ ] Re-pin to `qbit-build-parent` 2.1.0 when qbit-bom publishes it, and remove the `activeByDefault` on the `qqq-snapshot` profile (ADR-0007).
|
||||
- [ ] qqq-all follow-up PR: remove the manual `instance.addQBit(...)` for quick-search in `DemoQBits`, pass `withSchedulerName("demoScheduler")`, set `withStartupMode(QuickSearchStartupMode.DEGRADED)` for the full profile test, pin 1.0.0.
|
||||
- [ ] Voyage: pin 1.0.0, set `schedulerName`, switch to `opensearchUrl` and `${env.}` credentials.
|
||||
- [ ] CI: add a second test job with `-Dopensearch.test.image=opensearchproject/opensearch:3.9.0` (needs a qqq-orb parameter for extra Maven args, or a plain job).
|
||||
- [ ] Open the qqq issue proposing `RecordSearchProviderInterface` (D1); quick-search adapter lands in 1.1.
|
||||
- [x] Tracking issue #12 created and recorded in the PRD (2026-10-06).
|
||||
- [x] PR #8 merged 2026-10-05; Dependabot PR #1 closed as superseded.
|
||||
- [x] PR #10 merged and `1.0.0-RC.1` published to Maven Central from `release/1.0` (2026-10-05).
|
||||
- [ ] Upstream: tell qbit-bom that the published parent POM has `<version>${revision}</version>`, which Maven 3.10 rejects (this repo works around it with the wrapper on 3.9.11).
|
||||
|
||||
## Before 1.0.0 GA (from the PR #10 review, 2026-10-05; filed as #13 to #17 under #12)
|
||||
|
||||
- [x] #13 Listener: a failed in-transaction re-read or a missing AWS SDK (`LinkageError`) records `quickSearchFailedEvent` rows (PR #22).
|
||||
- [x] #14 Full reindex: deletes made during a rebuild are captured (`REBUILDING`, `AWAITING_REINDEX`) and applied after the alias swap (PR #20).
|
||||
- [x] #15 Transport: warn on `tls.hostnameVerification=false`, empty `${env.X}` treated as missing, userinfo in `opensearchUrl` rejected (PR #18).
|
||||
- [x] #16 Search: `offset` counts readable results under record locks, per-table `offset`, bounded scan and 10,000 window cap (PR #19).
|
||||
- [x] #17 `safeCount` logs at warn and keeps the previous `documentCount` (PR #21).
|
||||
- [x] `1.0.0-RC.2` published to Maven Central from `release/1.0` (2026-10-06).
|
||||
- [ ] Stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review): only another all-tables reindex clears it, `AWAITING_REINDEX` rows accumulate and drift detection stays off for those tables. No data is lost. Fix needs a liveness check on the `FULL_REINDEX` run record before reconcile may reset the status, plus refusing a second concurrent full reindex.
|
||||
- [ ] Drift detection still writes `NEEDS_REINDEX` into `status`, so a few-millisecond window can overwrite `REBUILDING`; the durable fix is a separate drift field (schema change, so 1.1 at the earliest).
|
||||
|
||||
## Later (1.x)
|
||||
|
||||
- [ ] Health widget for the admin app (data exists: `isHealthy`, `documentCount`, `lastRunStatus`).
|
||||
- [ ] `IndexEvent` sequence number so an async publisher can order INDEX and DELETE; promote the publisher SPI from experimental.
|
||||
- [ ] Amazon OpenSearch Serverless (needs removal of `_delete_by_query` and `_refresh` from the design).
|
||||
- [ ] Jev table routing prototype; see `docs/reference/typesafe-jev-candidates.md`.
|
||||
|
||||
## Done
|
||||
|
||||
- [x] Status assessment, deep-dive audit (25 blockers), PRD (2026-10-04).
|
||||
- [x] 4.1.0-RC.1 compatibility verified (2026-10-04).
|
||||
- [x] All 25 audit blockers implemented and tested on the release branch (2026-10-04).
|
||||
@@ -0,0 +1,93 @@
|
||||
# Audit 01: qbit-quick-search readiness for QQQ 4.1
|
||||
|
||||
Date: 2026-10-04. Read-only audit of `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` (0.2.1-SNAPSHOT, parent `qbit-build-parent` 2.0.0, qqq 4.0.0) against the QQQ tag `v4.1.0-RC.1`, the sibling qbits on their `origin/develop` branches (local checkouts are stale, Dec 2025), `qqq-all` `origin/develop` (e35f1f6, 2026-10-01) and `qbit-bom` `v2.0.0`.
|
||||
|
||||
Path abbreviations used below:
|
||||
|
||||
- `QS/` = `src/main/java/com/kingsrook/qbits/quicksearch/` in the qbit
|
||||
- `CORE/` = `qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/` at `v4.1.0-RC.1`
|
||||
- `JAV/` = `qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/` at `v4.1.0-RC.1`
|
||||
- `WH/` = `qbit-webhooks` `origin/develop:src/main/java/com/kingsrook/qbits/webhooks/`
|
||||
- `ALL/` = `qqq-all` `origin/develop:qqq-all-app/src/`
|
||||
|
||||
Taken as given: 233 unit tests and 12 Testcontainers integration tests pass on qqq 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1. Nothing below is a compile or test failure; everything is a contract, consistency or operability gap.
|
||||
|
||||
## Findings table
|
||||
|
||||
| ID | Severity | Finding | Evidence (qbit) | Evidence (qqq tag / siblings / consumers) | Recommended change | 1.0 blocker |
|
||||
|---|---|---|---|---|---|---|
|
||||
| F-01 | High | Producer is a plain class, not a `QBitMetaDataProducer`; it registers no `QBitMetaData`, so the host must stamp one by hand, and the version it stamps is a hand-copied string. | `QS/QuickSearchQBitProducer.java:60` (`public class QuickSearchQBitProducer`), `:112` (`produce(QInstance)`), `:200-203` (static context populated), `:214-215,230-232,260` (`addTable/addProcess/addApp` directly) | `CORE/model/metadata/qbits/QBitMetaDataProducer.java:43,50,56,108-189` (contract: `getQBitConfig`, `getQBitMetaData`, default `produce` that calls `qInstance.addQBit` at `:115`, pushes `QBitProductionContext` at `:117-118`, validates at `:122`, finds producers at `:124`, applies `getTableMetaDataCustomizer`/`getDefaultBackendNameForTables` at `:156-166`, stamps `sourceQBitName` at `:172-175`). Siblings: `WH/WebhooksQBitProducer.java:45,55-64` and `qbit-standard-process-trace` `StandardProcessTraceQBitProducer.java:42,90-94`, `qbit-workflows` `WorkflowsQBitProducer.java:65,77-81` all implement `QBitMetaDataProducer<C>`; only `qbit-user-role-permissions` still uses the gen-1 `QBitProducer` (`UserRolePermissionsQBitProducer.java:41`). Consumer workaround: `ALL/main/java/com/kingsrook/qqq/all/DemoQBits.java:161-163` calls `produce(instance)` then `instance.addQBit(new QBitMetaData()...withVersion("0.2.1-SNAPSHOT"))` itself; `ALL/test/java/com/kingsrook/qqq/all/FullQBitsTest.java:44-48,78-79` depends on the artifactId `quick-search` being present. | Implement `QBitMetaDataProducer<QuickSearchQBitConfig>`; return `QBitMetaData` with groupId `com.kingsrook.qbits`, artifactId `quick-search` (keep this exact string, qqq-all asserts it), version read from a build-time resource or constant kept in sync by the release step, `withNamespace(getNamespace())`, `withConfig(getQBitConfig())`. Keep `withConfig(...)`/`produce(QInstance)` as thin compatibility shims (see migration plan) so `DemoQBits.java:161` keeps compiling; the host's manual `addQBit` at `:162-163` then becomes a duplicate and should be removed in qqq-all once this ships. | Yes |
|
||||
| F-02 | High | Real-time indexing runs inside the caller's uncommitted transaction. Post-insert/update/delete customizers fire before any commit, so a rolled-back bulk/ETL write is still indexed; a committed write that OpenSearch rejected is logged and dropped. | `QS/customizers/QuickSearchPostInsertCustomizer.java:67-105` (publishes synchronously, swallows failure at `:99-102`); `QuickSearchPostUpdateCustomizer.java:82-123,149-154` (re-reads with `updateInput.getTransaction()`); `QuickSearchPostDeleteCustomizer.java:68-118` | `CORE/actions/tables/InsertAction.java:286` (`runPostInsertCustomizers` is the last thing before `return insertOutput` at `:292`); `UpdateAction.java:268`; `DeleteAction.java:370-390`. Commit happens later in the caller: `CORE/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java:204-207` commits after the actions ran, `:215-218` rolls back on exception. 4.1 adds the fix: `CORE/actions/QBackendTransaction.java:110-113` (`addAfterCommitCallback`, callbacks discarded on `rollback()` at `:89-96`), `qqq-backend-module-rdbms/.../RDBMSTransaction.java:128` (`runAfterCommitCallbacks` after JDBC commit). Reference use: `qqq-esb/src/main/java/com/kingsrook/qqq/esb/publish/EsbRecordChangeListener.java:88-97`. Not in 4.0.0 (0 hits for `addAfterCommitCallback` in `v4.0.0:CORE/actions/QBackendTransaction.java`). | When `input.getTransaction() != null`, build the events now (the record data is correct inside the transaction) but defer `publisher.publish*` via `transaction.addAfterCommitCallback(...)`; publish immediately only when there is no transaction (RDBMS without a transaction autocommits per statement, `RDBMSInsertAction.java:73-80`). Behind a `Capability`/reflection guard if 4.0 support must be kept. | Yes |
|
||||
| F-03 | High | The customizers are the wrong 4.1 hook. 4.1 adds an instance-level `RecordChangeListenerInterface` that is designed for exactly this (fires for INSERT/UPDATE/DELETE, cannot fail the write, carries old records for updates and the stored records for deletes, carries the transaction for after-commit work). The qbit's per-table `withCustomizer` approach also throws when the host already has a customizer in the slot (see F-04). | `QS/QuickSearchQBitProducer.java:571-578` | `CORE/actions/tables/listeners/RecordChangeListenerInterface.java:28-61` (contract, `appliesTo(tableName, type)` at `:56`, `onRecordsChanged` at `:61`); `RecordChangeEvent.java:30-54` (records/oldRecords/transaction semantics per type); `CORE/model/metadata/QInstance.java:1989-1996` (`withRecordChangeListener`); fire points `InsertAction.java:311-328`, `UpdateAction.java:259,488-537` (oldRecords paired by primary key), `DeleteAction.java:365,809-823`; `RecordChangeListenerHelper.java:54-70,78-102` (never fails the write). Introduced by qqq commit `8db90166a`; absent in 4.0.0 (`v4.0.0:CORE/model/metadata/QInstance.java` has 0 `RecordChangeListener` hits). | Replace the three customizers with one `QuickSearchRecordChangeListener implements RecordChangeListenerInterface`: `appliesTo` returns true only for discovered tables (so no extra old-record fetches elsewhere), `onRecordsChanged` builds INDEX events from `event.getRecords()` for INSERT/UPDATE and DELETE events from `event.getRecords()` for DELETE, and publishes after commit when `event.getTransaction() != null` (F-02). The update re-read in `QuickSearchPostUpdateCustomizer.java:132-155` stays necessary (UPDATE `records` are "as the backend returned them", which for RDBMS is the sparse input), so keep it, using `event.getTransaction()`. Register once in `postProduceActions` with `qInstance.withRecordChangeListener(new QCodeReference(...))`. Keep the customizer classes for one release as a 4.0 fallback if dual support is required. | Yes (choose F-03 or F-04; F-03 preferred) |
|
||||
| F-04 | High | `QTableMetaData.withCustomizer` throws `IllegalArgumentException` when the slot is occupied, so any host table that already has a POST_INSERT/UPDATE/DELETE customizer makes `produce` blow up (and not with a `QException`). | `QS/QuickSearchQBitProducer.java:571-578` | `CORE/model/metadata/tables/QTableMetaData.java:644-657` (`:651-654` throws `IllegalArgumentException("Attempt to add a second customizer with role ...")`); same at `v4.0.0`. Composition options: `CORE/actions/customizers/MultiCustomizer.java:66-70` (`MultiCustomizer.of(...)`) and `:79-83` (`addTableCustomizer(existing, codeRef)`); instance-level `CORE/model/metadata/QInstance.java:1927-1945` (`withTableCustomizer(role, codeRef)`), consumed in `InsertAction.java:356-362` and `UpdateAction.java:303-309` but NOT in `DeleteAction.java` (0 hits for `getTableCustomizers`), so instance-level customizers never fire for deletes. Sibling precedent: `WH/WebhooksQBitProducer.java:102-103` (instance-level, insert/update only), `WH/model/Webhook.java:154-157` (`MultiCustomizer.of`). | If customizers are kept (instead of F-03): read `table.getCustomizers().get(role)`; if absent add directly; if present and it is a `QCodeReferenceWithProperties` for `MultiCustomizer`, call `MultiCustomizer.addTableCustomizer`; otherwise replace the slot with `MultiCustomizer.of(existing, ours)` via `table.getCustomizers().put(role, ...)` (the map is mutable). Do not use `QInstance.withTableCustomizer` for this qbit: it is not per-table (it would have to filter by `QuickSearchQBitContext.getTableConfig`) and it skips deletes. | Yes (if F-03 not taken) |
|
||||
| F-05 | High | Processes are produced without `QScheduleMetaData` or `BasepullConfiguration`, so nothing ever runs the basepull or reconcile unless the host schedules it by hand; `enableScheduledProcesses` is misnamed (it only gates registration). The qbit hand-rolls last-run tracking on its own table instead of QQQ's basepull support, and the basepull window has a gap (query at T0, stamp `Instant.now()` at T1 after the batches finish; rows modified in (T0, T1] with the same second are skipped next run). | `QS/QuickSearchQBitProducer.java:509-549` (three `QProcessMetaData` with only a step); `:228-233`; `QS/QuickSearchQBitConfig.java:55` (`enableScheduledProcesses = true`); `QS/processes/BasepullIndexStep.java:202-209` (filter `GREATER_THAN lastBasepullTime`), `:268-278` (stamps `Instant.now()` after processing); `README.md:306-307` describes it as a schedule | `CORE/model/metadata/processes/QProcessMetaData.java:75,564` (`withSchedule(QScheduleMetaData)`), `:61,598` (`withBasepullConfiguration`); `CORE/model/metadata/scheduleing/QScheduleMetaData.java:40-49,89,284` (`repeatSeconds`, `cronExpression`, `schedulerName`); `CORE/scheduler/QScheduleManager.java:284-289,413-429` (processes with a schedule are set up against `getSchedule().getSchedulerName()`); `JAV/QApplicationLauncher.java:139-153,216-222` (launcher starts the scheduler only if some process/queue/automation has a schedule); `CORE/instances/QInstanceValidator.java:2255-2258,2290-2326` (a schedule must have repeat or cron, and a scheduler name that exists in the instance); `CORE/processes/implementations/basepull/BasepullConfiguration.java:33-43` (table/keyField/lastRunTimeFieldName/timestampField/seconds-to-subtract); `CORE/actions/processes/RunProcessAction.java:147-155,183-187,914` (framework reads the stored timestamp before the step, stores the new one after, keyed by process name). Consumers: `ALL/.../DemoQBits.java:63` adds `SimpleSchedulerMetaData "demoScheduler"`, `:106,171` pass `withSchedulerName("demoScheduler")` to webhooks and sftp. | Add `schedulerName` (nullable) and `basepullRepeatSeconds`/`reconcileCronExpression` (nullable) to `QuickSearchQBitConfig`; when `schedulerName` is set, attach `new QScheduleMetaData().withSchedulerName(...).withRepeatSeconds(...)` to the basepull process and a cron schedule to reconcile. Leave processes unscheduled when `schedulerName` is null (hosts that use the `scheduledJob` table can still schedule them). Validate `schedulerName` in `validate` only when `qInstance.getSchedulers()` is non-empty (see F-07). Basepull: either adopt `BasepullConfiguration` (one process per table would be needed because `BASEPULL_KEY_VALUE` is per run; or run one process that overrides `basepullKeyValue` per table) or keep the own table but stamp the run start time (captured before the first query, minus a small overlap) instead of `Instant.now()` after the loop. Rename `enableScheduledProcesses` to `enableIndexingProcesses` with a deprecated alias. | Yes |
|
||||
| F-06 | High | Search results carry no permission filtering. `QuickSearchAction` returns table/recordId/label from OpenSearch for any table the index holds, regardless of the session's table permissions or record security locks; produced tables/processes/app set no `QPermissionRules`, which is fine for the demo (instance default is `NOT_PROTECTED`) but means a host that sets a restrictive `defaultPermissionRules` silently locks the admin app, and a host that does not gets an unprotected admin app. | `QS/actions/QuickSearchAction.java:63-130` (no `PermissionsHelper` use; `:81-82` reads static context); `QS/QuickSearchQBitProducer.java:457-549` (no `withPermissionRules`) | `CORE/actions/tables/RecordSearchAction.java:103-113` (core search skips tables without `PermissionsHelper.hasTablePermission(input, table, READ)` and queries through `QueryAction` so record locks apply, `:63-67`); `CORE/instances/QInstanceEnricher.java:544-575` (null rules get `qInstance.getDefaultPermissionRules()`, else `QPermissionRules.defaultInstance()` = `NOT_PROTECTED`, `CORE/model/metadata/permissions/QPermissionRules.java:44-47`). Siblings: `qbit-workflows` processes set `withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))` explicitly (`GetWorkflowRevisionProcess.java:72` etc.); qqq-all protects webhook tables/processes itself (`ALL/test/.../FullQBitsTest.java:67-77`, `DemoUsers.protectWebhookResources`). | In `QuickSearchAction`: require a `QContext` session; filter `tableConfigs` to tables with READ permission before building the OpenSearch query (pass the filtered table list as a `terms` filter on `sourceTable`); for tables with record security locks, re-verify hits by primary key through `QueryAction` (or document that locked tables are not indexable). Add `permissionRules` to the config for the admin app/tables/processes, defaulting to `HAS_ACCESS_PERMISSION` for the processes and `READ_INSERT_EDIT_DELETE_PERMISSIONS` for the two operational tables, with `NOT_PROTECTED` opt-out, mirroring what qqq-all does for webhooks. | Yes |
|
||||
| F-07 | Medium | `validate` does a backend lookup, which works today only because the qbit is produced after backends exist. Under `QBitMetaDataProducer` the same timing holds (validate runs inside `produce` with the host's instance), but if a host lets `MetaDataProducerHelper` discover the producer by package scan, it is instantiated with a no-arg constructor and null config, and `QBitMetaDataProducer.produce` dereferences `getQBitConfig()` with no null check. | `QS/QuickSearchQBitConfig.java:73-81` (`qInstance.getBackend(backendName) == null`); `QS/QuickSearchQBitProducer.java:117-122` | `CORE/model/metadata/qbits/QBitMetaDataProducer.java:113-122` (`qBitConfig.validate(qInstance)` on the live instance; no null guard); `CORE/model/metadata/qbits/QBitConfig.java:58-75` (exceptions from `validate` are caught and only logged at `:66-69`, so a thrown NPE inside the list-based validate is swallowed, and only `errors` entries fail); `CORE/model/metadata/MetaDataProducerHelper.java:200-202,585-595` (any `MetaDataProducerInterface` in the scanned package with a no-arg constructor is instantiated and run). Known gap note: `WH/WebhooksQBitConfig.java:85-99` ("qbit validation is happening before the full instance is built"; backend and scheduler checks commented out). `QInstanceValidator` has no qbit-specific checks (0 hits for `sourceQBit`/`qbit` in `CORE/instances/QInstanceValidator.java`), so produced metadata passes as long as apps list every child in a section (`:2597-2605`; the enricher creates a default section) and schedules reference a known scheduler (`:2323-2326`). | Keep the list-based `validate`; guard null `qInstance` and null `getBackends()`; return `isEnabled() == false` from the producer when `getQBitConfig() == null` so package scanning skips it (`MetaDataProducerHelper.java:323,345` honors `isEnabled`). Add a `QInstanceValidatorPluginInterface` (registered via `QInstanceValidator.addValidatorPlugin`, `:319`) for late checks (backend exists, each searchable table exists, scheduler exists) so they run against the full instance. | No |
|
||||
| F-08 | Medium | Static `QuickSearchQBitContext` holds config/client/publisher/tables for the JVM. Two instances in one JVM (tests, namespace use, blue/green instance reload via `QInstance` hot swap) collide; customizers and steps read it instead of the instance. 4.1 gives two sanctioned places: `QBitMetaData.getConfig()` reachable from any `SourceQBitAware` table/process, and `QBitProductionContext` during production. | `QS/QuickSearchQBitContext.java:32-37` (static volatile fields), `:152-158` (`clear()` for tests); readers `QS/processes/AbstractIndexingStep.java:68-73,85-90,102-122`, `QS/customizers/*:74,89,75`, `QS/actions/QuickSearchAction.java:81-82` | `CORE/model/metadata/qbits/SourceQBitAware.java:54-75` (`getSourceQBit()` / `getSourceQBitConfig()` via `QContext.getQInstance().getQBits()`), implemented by `QTableMetaData` and `QProcessMetaData` (not `QAppMetaData`); `CORE/model/metadata/qbits/QBitProductionContext.java:48-66` (config stack during produce). Sibling pattern: `WH/WebhooksQBitConfig.java:146-157` (`getConfigValue` resolves config from a produced table's `getSourceQBitConfig()`, with a default fallback); `WH/model/WebhookEvent.java:121` (`QBitProductionContext.peekQBitConfig()` inside component producers). | Keep `QuickSearchQBitConfig` as the single source of truth and put the runtime objects (client, publisher, discovered tables) on it lazily (`transient`, built on first use, closed by a runtime service, F-10). Resolve the config in steps via `QContext.getQInstance().getProcess(input.getProcessName()).getSourceQBitConfig()`, in the listener via the discovered table's `getSourceQBitConfig()`, and in `QuickSearchAction` via an explicit `QuickSearchInput.withQBitName`/default lookup over `qInstance.getQBits()`. Keep the static accessors as deprecated delegates for one release. | No |
|
||||
| F-09 | Medium | Produce is "best effort" on OpenSearch: a missing or unreachable cluster yields a null client and publisher, the warnings are the only signal, every write then logs "publisher is null", and `QuickSearchAction` throws at query time. There is no fail-fast toggle and no health exposure. qqq-all depends on the degraded behaviour: `FullQBitsTest` builds the full instance with OpenSearch at `127.0.0.1:1`. | `QS/QuickSearchQBitProducer.java:159-167,172-182,187-195`; `QS/customizers/QuickSearchPostInsertCustomizer.java:74-79`; `QS/actions/QuickSearchAction.java:84-87`; `QS/opensearch/QuickSearchOpenSearchClient.java:94-150` (transport is built lazily; only `ensureIndexExists` at `:161` touches the network) | `JAV/QApplicationLauncher.java:118-175` (start order: server, scheduler, runtime services; a thrown `start` stops what started and fails `run`), `:192-207` (`failOnMetaDataProducerError` only reaches producer-based apps); `CORE/instances/QRuntimeServiceInterface.java:39-68` (`start(QInstance)`, `stop()`, `afterApplicationStop()`); `CORE/model/metadata/QInstance.java:2153` (`withRuntimeService`). Consumer: `ALL/test/.../FullProfileTest.java:123-124` (`OPENSEARCH_HOST=127.0.0.1`, `OPENSEARCH_PORT=1`) and `FullQBitsTest.java:41-42` (instance must define and validate with that); `ALL/main/.../QqqAllApplication.java:98` sets `withFailOnMetaDataProducerError(true)`, `:154` registers a runtime service; `qqq-all` plan: "disabled with a notice in core" (`origin/develop:docs/plan.md:80`). | Add `startupMode` to the config: `DEGRADED` (default, current behaviour, keeps qqq-all's test green) and `FAIL_FAST` (throw `QBitConfigValidationException`/`QException` from `produce` when the client or index cannot be created). Move the connect and `ensureIndexExists` out of `produce` into a `QuickSearchRuntimeService implements QRuntimeServiceInterface` registered with `qInstance.withRuntimeService(...)` in `postProduceActions`: `start` connects (fail-fast or warn per mode), `stop` closes the transport (today nothing calls `SynchronousIndexEventPublisher.close()` or `QuickSearchOpenSearchClient.close()`, `QS/publisher/SynchronousIndexEventPublisher.java:179-182`, `QS/opensearch/QuickSearchOpenSearchClient.java:595-606`). Expose a `quickSearchHealth` process or widget that reports connected/index exists/document count so the host's `/health` can show a notice. | No |
|
||||
| F-10 | Medium | Lifecycle: nothing closes the OpenSearch transport or publisher at shutdown; index creation happens during metadata production (a side effect on an external system inside `defineQInstance`). | `QS/QuickSearchQBitProducer.java:174-182`; `QS/publisher/SynchronousIndexEventPublisher.java:179-182` (only `close()` path, never invoked) | `JAV/QApplicationLauncher.java:155-168,243-273` (runtime services started after the instance is built, stopped in reverse order); `CORE/instances/QRuntimeServiceInterface.java:47-55` | Covered by F-09: runtime service owns connect/ensureIndex/close. | No |
|
||||
| F-11 | Medium | Core 4.1 ships `RecordSearchAction` and `POST /qqq/v1/search` with no provider SPI; Quick Search cannot back it, so a 4.1 host gets two incompatible "search" features (core: LIKE over `searchFields`; qbit: OpenSearch) and Next's search UI will call the core one. | `QS/actions/QuickSearchAction.java:63-130`, `QS/actions/QuickSearchInput.java`, `QuickSearchOutput`, `QuickSearchResult` (parallel model, no relation to core) | `CORE/actions/tables/RecordSearchAction.java:71-119` (`new RecordSearchAction().execute(input)` concrete class, iterates `qInstance.getTables()` with `getSearchFields()`, no code-reference indirection); `CORE/model/actions/tables/search/RecordSearchInput.java:35-40` (`searchTerm`, `tableNames`, `limitPerTable`, `inputSource`); `RecordSearchResult` (tableName, tableLabel, recordId, recordLabel); `JAV/executors/RecordSearchExecutor.java:51-57` (hard-wires `new RecordSearchAction()`); `JAV/specs/v1/RecordSearchSpecV1.java:67-68` (`POST /search`); qqq commit `7dd36c8ad`; `QTableMetaData.java:100,2016` (`searchFields`), validator `QInstanceValidator.java:1282-1298`. | Smallest core SPI (propose in qqq): `QInstance.withRecordSearchProvider(QCodeReference)` plus `interface RecordSearchProviderInterface { boolean appliesTo(QTableMetaData table); List<RecordSearchResult> search(RecordSearchInput input, QTableMetaData table, String term, int limit) throws QException; }`. `RecordSearchAction.searchTable` consults the provider when `appliesTo` is true (and the table has `searchFields` or the provider says it is searchable), still after the permission check at `:110`, so record-security filtering stays core's job. Until then: make `QuickSearchResult` shape-compatible with `RecordSearchResult` (add `tableLabel`), and set `searchFields` on indexed tables via the table customizer so Next's UI at least shows the search box; document the overlap. | No (SPI is a qqq change; the qbit-side adapter is 1.1) |
|
||||
| F-12 | Low | Namespace support is absent: produced names are prefixed with the qbit's own `tableNamePrefix`, not with the `QBitMetaData.namespace`; `getNamespace()` is not implemented. Core does not prefix names automatically either, so a namespace-aware qbit has to do it itself. | `QS/QuickSearchQBitConfig.java:169-195` (`applyPrefix`, `tableNamePrefix`); `QS/QuickSearchQBitProducer.java:208-209,220-222,246` | `CORE/model/metadata/qbits/QBitMetaData.java:60-68` (namespace is part of the qbit's unique name only); `QBitMetaDataProducer.java:62-65` (`getNamespace()` default null); qqq test qbit does no name prefixing (`qqq-backend-core/src/test/.../testqbit/metadata/SomeTableMetaDataProducer.java:59-61`); `ALL/.../DemoQBits.java:120,182` pass a namespace to gen-1 producers (`produce(instance, "shipping-geo")`). | Add `namespace` to the producer (`withNamespace`), use it as the default `tableNamePrefix` when the prefix is null, and feed it to `QBitMetaData.withNamespace`. Keep `tableNamePrefix` as an override. | No |
|
||||
| F-13 | Low | Build: the pom declares `jacoco-maven-plugin` with a `check` execution but no version and no `prepare-agent`, so the check is skipped ("missing execution data file") and the plugin version floats; the parent manages no jacoco plugin. JUnit is pinned to 5.10.0 (BOM import and direct dependency) while the parent manages `junit-bom` 6.0.1. | `pom.xml:156-181` (jacoco `check` only), `:69-77` (`junit-bom` 5.10.0 import), `:118-122,147-150` (junit-jupiter 5.10.0) | `qbit-bom` `v2.0.0:pom.xml:54-55` (`qqq-bom-pom` 4.0.0), `:62-63` (`junit-bom` 6.0.1), `:70-90` (pluginManagement contains only `maven-compiler-plugin`); no 2.1.0 tag exists yet (`git tag` ends at `v2.0.0`; `origin/main aaf52d2` "prepare build parent 2.0.0 for QQQ 4.0"); CI orb `kingsrook/qqq-orb@0.5.2` in `.circleci/config.yml:4` (parent uses 0.5.0). qqq's own BOM manages no JUnit (`v4.1.0-RC.1:qqq-bom/pom.xml` has no junit entries). | Either add `prepare-agent` (and a `prepare-agent-integration`/`report` pair for failsafe) with an explicit plugin version and make the 0.70/0.90 thresholds real, or delete the jacoco block (ADR). Drop the local `junit-bom` import and the 5.10.0 pins and inherit 6.0.1 from the parent (JUnit 6 requires Java 17+, fine on 21). When `qbit-build-parent` 2.1.0 (qqq 4.1) is released, re-pin and delete the `qqq-snapshot` profile's `qqq-bom-pom` import (ADR-0007). | No |
|
||||
| F-14 | Low | Source-table batch paging is unordered (`limit`/`skip` with no `orderBy`), so full reindex, reconcile and basepull can skip or duplicate rows on backends without a stable default order. | `QS/processes/AbstractIndexingStep.java:348-374` (no `withOrderBy`); `QS/processes/BasepullIndexStep.java:220-262` | `CORE/actions/tables/RecordSearchAction.java:165` (core orders by primary key before limiting, as a reference pattern) | Add `effectiveFilter.withOrderBy(new QFilterOrderBy(primaryKeyField))` and prefer keyset paging (`primaryKey > lastSeen`) over `skip`. | No (listed in the PRD's E2) |
|
||||
|
||||
## Answers to the audit questions
|
||||
|
||||
### 1. QBit contract
|
||||
|
||||
What must change (all evidence above, F-01, F-08, F-12):
|
||||
|
||||
1. `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>` with `getQBitConfig()`, `getQBitMetaData()` and `postProduceActions(...)` (`CORE/.../QBitMetaDataProducer.java:50,56,70`). The default `produce` scans the producer's own package for `MetaDataProducerInterface` implementations (`:124`, `getPackageNameForFindingMetaDataProducers` at `:80-101`), so the two operational tables, three processes and the app become small producers (`QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>` etc., `CORE/.../QBitComponentMetaDataProducer.java:34-69`) or `@QMetaDataProducingEntity` annotations on `QuickSearchIndex`/`QuickSearchIndexRun` (today they are plain `QRecordEntity` with `@QField`, `QS/model/QuickSearchIndex.java:32-36`). Customizer/listener registration and the OpenSearch bootstrap belong in `postProduceActions`, as webhooks does (`WH/WebhooksQBitProducer.java:89-104`).
|
||||
2. `QBitMetaData` with groupId `com.kingsrook.qbits`, artifactId `quick-search`, real version, namespace, config (`CORE/.../QBitMetaData.java:47-52`). The framework then stamps `sourceQBitName` on every produced table and process (`QBitMetaDataProducer.java:172-175`; `QTableMetaData` and `QProcessMetaData` implement `SourceQBitAware`), which is what makes F-08's static context removable.
|
||||
3. `QuickSearchQBitConfig` already implements `QBitConfig` (`QS/QuickSearchQBitConfig.java:45`); add `getDefaultBackendNameForTables()` returning `backendName` and `getTableMetaDataCustomizer()` (settable, like `WH/WebhooksQBitConfig.java:563,805`) so hosts can set RDBMS backend details the way `ALL/.../DemoQBits.java:80-88` does for the other qbits.
|
||||
|
||||
Idiomatic choice: `QBitMetaDataProducer` (3 of 4 siblings; qqq's own interface doc calls it the "2nd generation ... evolution over QBitProducer", `:39-40`). `QBitProducer` (gen 1) is only kept by user-role-permissions and qqq's test qbit.
|
||||
|
||||
qqq-all impact: `DemoQBits.java:161` uses `new QuickSearchQBitProducer().withConfig(searchConfig).produce(instance)`. `QBitMetaDataProducer.produce(QInstance)` returns `MetaDataProducerMultiOutput` instead of `void`, so that call still compiles (the return value is ignored) only if the metadata is added to the instance. The default `produce` does NOT add to the instance; it returns the multi-output and the host calls `.addSelfToInstance(instance)` (`DemoQBits.java:93,101,108,116`). To keep source compatibility: keep `withConfig(...)` as an alias of `withQBitConfig(...)`, and override `produce(QInstance)` to call the interface's default `produce`, then `addSelfToInstance(qInstance)` on the result before returning it (idempotent for hosts that also call `addSelfToInstance`, since `QInstance.addTable` etc. would throw on duplicates; so the override must either add and return an empty output, or be documented as "already added"; returning an empty `MetaDataProducerMultiOutput` after adding is the safe choice). The manual `instance.addQBit(...)` at `DemoQBits.java:162-163` will then duplicate the qbit's own `addQBit` (`QBitMetaDataProducer.java:115`); `QInstance.addQBit` behaviour on duplicate names should be checked when that qqq-all change is made (`CORE/model/metadata/QInstance.java:1800`).
|
||||
|
||||
### 2. Validation timing
|
||||
|
||||
- `QBitConfig.validate` runs at the start of `QBitMetaDataProducer.produce` (`:122`) against whatever is in the host's instance at that moment. It is not "before the full instance exists" in the sense of the host's backends: hosts add backends first (`ALL/.../FullProfileApplication.java:70-76`). The webhooks note (`WH/WebhooksQBitConfig.java:85-99`) refers to hosts that produce qbits before their own tables/schedulers; the backend lookup in this qbit (`QS/QuickSearchQBitConfig.java:79`) is safe under both the current flow and the `QBitMetaDataProducer` flow for the qqq-all ordering, but a host that produces the qbit before `addBackend` gets a false "Backend not found".
|
||||
- Under `MetaDataProducerHelper` scanning the producer would be instantiated with a null config (`MetaDataProducerHelper.java:585-595`) and `produce` would NPE at `QBitMetaDataProducer.java:113-122`; `isEnabled()` returning false when the config is null avoids that (`:145-149`).
|
||||
- `QBitConfig.validate(QInstance)` swallows exceptions from the list-based validate and only fails on collected errors (`QBitConfig.java:62-74`), so validation must add to `errors`, never throw.
|
||||
- `QInstanceValidator` 4.1 adds nothing qbit-specific. Relevant checks the produced metadata must satisfy: process steps have names and `BackendStep` code (`:2178-2197`), a schedule has repeat or cron plus an existing scheduler (`:2290-2326`), app children appear in sections (`:2597-2605`; the enricher builds a default section), table `searchFields` if set must be visible string/integer fields (`:1282-1298`). Nothing validates `sourceQBitName`.
|
||||
|
||||
### 3. Scheduling
|
||||
|
||||
- A process is scheduled by `QProcessMetaData.withSchedule(new QScheduleMetaData().withSchedulerName(name).withRepeatSeconds(n) | .withCronExpression(expr).withCronTimeZoneId(tz))` (`QProcessMetaData.java:564`, `QScheduleMetaData.java:40-49`). The host must own a `QSchedulerMetaData` (`SimpleSchedulerMetaData` or `QuartzSchedulerMetaData`, `CORE/model/metadata/scheduleing/{simple,quartz}/*.java:45-54`) and either start `QScheduleManager` itself or use `QApplicationLauncher`, which starts it only when something has a schedule (`JAV/QApplicationLauncher.java:139-153,216-222`). `QScheduleManager.start` sets up every process with a schedule (`:284-289`) under the system session supplier (`:163`), and `qqq.scheduleManager.enabled`/`onlyStartNamesMatching` gate it (`:153`, `SchedulerUtils.java:61-71`). Cron needs Quartz (`SimpleSchedulerMetaData.supportsCronSchedules`, `:56`).
|
||||
- Minimal correct way for this qbit: config fields `schedulerName`, `basepullRepeatSeconds` (default 300), `reconcileCronExpression`+`reconcileCronTimeZoneId` (default null, so no reconcile schedule unless the host asks); attach schedules only when `schedulerName` is set. Host provides the scheduler and the launcher (qqq-all already has `demoScheduler`, `DemoQBits.java:63`).
|
||||
- Basepull duplication: yes. `BasepullConfiguration` + `RunProcessAction` already persist a last-run timestamp per process key, pass it to the step as `RunBackendStepInput.getBasepullLastRunTime()` (`RunProcessAction.java:724-728`), and only store the new time when the step sets `BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD` (`:183-187`), with overlap seconds (`BasepullConfiguration.java:42-43`). `BasepullIndexStep` re-implements this per table on `quickSearchIndex.lastBasepullTime` (`:202-209,268-278`) and stamps `Instant.now()` after the loop (window gap). The qbit's per-table interval and per-table enable flag are not expressible with one `BasepullConfiguration` (its key is the process name, `RunProcessAction.java:880-885`), so the pragmatic fix is to keep the own table but (a) capture the run start before the first query and store that, minus an overlap, and (b) run the process on a `QScheduleMetaData`. Full adoption of `BasepullConfiguration` would mean one scheduled process per indexed table, which is also legitimate and gives per-table schedules for free.
|
||||
|
||||
### 4. Customizer composition
|
||||
|
||||
Confirmed: `QTableMetaData.withCustomizer` throws `IllegalArgumentException` on an occupied slot at the 4.1 tag (`QTableMetaData.java:651-654`), same at 4.0.0. Options: `MultiCustomizer.of(a, b)` / `MultiCustomizer.addTableCustomizer(existing, c)` (`MultiCustomizer.java:66-83`), or instance-level `QInstance.withTableCustomizer(role, codeRef)` (`QInstance.java:1927-1945`), which runs for every table on insert/update (`InsertAction.java:356-362`, `UpdateAction.java:303-309`) but not on delete (`DeleteAction.java` has no `getTableCustomizers` call). For per-table targeting, `MultiCustomizer` composition is the right customizer-based answer; but the better 4.1 answer is the `RecordChangeListenerInterface` (F-03), which is instance-level, filters by `appliesTo(tableName, type)`, covers delete, and never fails the write.
|
||||
|
||||
### 5. Transactions and consistency
|
||||
|
||||
- Customizers run after the backend write and before the action returns, inside the caller's transaction if one was passed (`InsertAction.java:286-292`, `UpdateAction.java:268`, `DeleteAction.java:370-390`); the `RecordChangeListenerInterface` javadoc states this explicitly (`:32-34`). ETL/bulk processes commit after the actions (`StreamedETLExecuteStep.java:204-207`) and roll back on exception (`:215-218`), so a rolled-back bulk insert is indexed today. The javalin single-record endpoints pass no transaction (`QJavalinImplementation.java:689,726,957` set up sessions only; no `openTransaction`), and the RDBMS module then uses a fresh autocommit connection (`RDBMSInsertAction.java:73-80`; only `RDBMSTransaction` sets autocommit false, `RDBMSTransaction.java:55`), so those are effectively committed before the customizer runs.
|
||||
- Fix available only in 4.1: `QBackendTransaction.addAfterCommitCallback` (`QBackendTransaction.java:110-113`), honoured by `RDBMSTransaction.commit` (`:128`), discarded on rollback (`:89-96`, `RDBMSTransaction.java:114`).
|
||||
- Update customizer receives `Optional<List<QRecord>> oldRecordList` (`UpdateAction.java:289`); records are the sparse updated records, hence the qbit's re-read (`QuickSearchPostUpdateCustomizer.java:132-155`). Delete customizer receives the old records minus those in error (`DeleteAction.java:376-390`), so primary keys are present. The listener API gives the same: UPDATE `oldRecords` paired by primary key (`UpdateAction.java:498-537`), DELETE `records` = stored records before delete (`RecordChangeEvent.java:35,41`).
|
||||
- `QContext`/`QSession` exist inside customizers and listeners: javalin calls `setupSession` then `QContext.init(qInstance, session, null, input)` (`QJavalinImplementation.java:495,573`); the scheduler runs under the system session (`QScheduleManager.java:163`). The re-read in the update path therefore runs as the writing user; for indexing that is correct (index what the user could write), but for a permission-aware re-read on behalf of other readers it is not a substitute for F-06.
|
||||
|
||||
### 6. Permissions
|
||||
|
||||
Siblings mostly leave rules null and let the enricher apply the instance default (`QInstanceEnricher.java:544-575` → `NOT_PROTECTED` unless the host sets `defaultPermissionRules`); workflows sets `NOT_PROTECTED` explicitly on its processes; qqq-all layers its own rules on the webhook tables/processes and asserts them (`FullQBitsTest.java:67-77`). `RecordSearchAction` filters by table READ permission and queries through `QueryAction` so record locks apply (`RecordSearchAction.java:103-113,168-177`). There is no provider/SPI: the action is a concrete class with no code-reference indirection, and the javalin executor instantiates it directly (`RecordSearchExecutor.java:51`). Smallest SPI sketched in F-11.
|
||||
|
||||
### 7. Application launcher and qqq-all
|
||||
|
||||
`QApplicationLauncher` (`JAV/QApplicationLauncher.java:106-184`) starts the server (which builds the instance), the scheduler if anything is scheduled, then runtime services; failures stop what started and throw. Two things this qbit should support: a `QRuntimeServiceInterface` for connect/ensure-index/close with a fail-fast or degraded mode (F-09, F-10), and health reporting the host can surface. qqq-all's core profile excludes the qbit entirely (`CoreQBitsTest.java:42`) and its README says "Quick search is disabled in core because it requires OpenSearch" (`ALL/main/.../README.md:14`); the full profile's unit test builds the instance with an unreachable OpenSearch (`FullProfileTest.java:123-124`), so the degraded mode must remain the default. qqq-all also runs with `withFailOnMetaDataProducerError(true)` (`QqqAllApplication.java:98`), which only affects `MetaDataProducerHelper`; the qbit's own `produce` must throw a `QException` with a clear message when fail-fast is requested.
|
||||
|
||||
### 8. Build and release
|
||||
|
||||
Parent `v2.0.0` pins `qqq-bom-pom` 4.0.0 and `junit-bom` 6.0.1 and manages only the compiler plugin (`qbit-bom v2.0.0:pom.xml:54-55,62-63,70-90`); no 2.1.0 tag exists yet. The qbit's jacoco `check` has no `prepare-agent` and no version (`pom.xml:156-181`), so coverage is never measured and the plugin version floats with the repository metadata; the local JUnit 5.10.0 pins (`pom.xml:69-77,118-122,147-150`) override the parent's 6.0.1. The `qqq-snapshot` profile is correct for RC checks and should be removed or re-pointed when parent 2.1.0 ships. The CI orb versions differ (0.5.2 in the qbit, 0.5.0 in the parent's own CI) but that is harmless.
|
||||
|
||||
## Minimal migration plan (ordered)
|
||||
|
||||
1. Producer contract (F-01, F-12): make `QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>`; add `getQBitMetaData()` (artifactId `quick-search`, version from a `quick-search.properties` resource filtered by Maven), `getNamespace()`; move table/process/app building into package-scanned component producers (or `@QMetaDataProducingEntity` on the two entities plus small process/app producers); move customizer registration and client bootstrap into `postProduceActions`. Keep `withConfig`, `getConfig` and a `produce(QInstance)` override that adds to the instance and returns an empty multi-output, so `qqq-all DemoQBits.java:161` keeps compiling; mark them `@Deprecated`.
|
||||
2. Config surface (F-05, F-06, F-09): add `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId`, `permissionRules` (app/tables/processes), `startupMode` (`DEGRADED` default, `FAIL_FAST`), `tableMetaDataCustomizer` setter, `getDefaultBackendNameForTables()` = `backendName`. Deprecate `enableScheduledProcesses` in favour of `enableIndexingProcesses`. Validation stays list-based and null-safe; add a `QInstanceValidatorPluginInterface` for late checks (F-07).
|
||||
3. Real-time path (F-02, F-03, F-04): add `QuickSearchRecordChangeListener implements RecordChangeListenerInterface`, registered via `qInstance.withRecordChangeListener` in `postProduceActions`; publish after commit through `event.getTransaction().addAfterCommitCallback(...)` when a transaction is present. Keep the three customizers only as a documented 4.0 fallback (`enableRealTimeIndexing` + a `legacyCustomizers` flag), composing with `MultiCustomizer` when a slot is occupied; delete them in 1.1.
|
||||
4. Runtime state (F-08, F-10): add `QuickSearchRuntimeService implements QRuntimeServiceInterface` (connect, `ensureIndexExists`, close) registered via `qInstance.withRuntimeService`; resolve the config from `SourceQBitAware.getSourceQBitConfig()` in steps and the listener; keep `QuickSearchQBitContext` as a deprecated facade over the instance-bound state for one release.
|
||||
5. Scheduling and basepull (F-05, F-14): attach `QScheduleMetaData` when `schedulerName` is set; capture the basepull run-start timestamp before the first query and store it minus an overlap; add primary-key `orderBy`/keyset paging in `querySourceTableBatch`.
|
||||
6. Permission-aware search (F-06): filter table configs by `PermissionsHelper.hasTablePermission(input, table, READ)` and pass the allowed `sourceTable` set to the OpenSearch query; verify hits for tables with record security locks via `QueryAction`; align `QuickSearchResult` with core `RecordSearchResult` (add `tableLabel`) ahead of the SPI proposal (F-11).
|
||||
7. Build (F-13): decide jacoco (bind `prepare-agent` or remove), drop the JUnit 5.10.0 pins, re-pin to `qbit-build-parent` 2.1.0 when released and retire the `qqq-snapshot` profile's BOM import; bump to 0.3.0 for the compat release, 1.0.0 after qqq-all removes its manual `addQBit`.
|
||||
8. Downstream: qqq-all PR removing `DemoQBits.java:162-163`, passing `withSchedulerName("demoScheduler")`, and asserting `instance.getQBits()` contains `quick-search` from the qbit itself; propose the `RecordSearchProviderInterface` SPI in qqq (F-11).
|
||||
@@ -0,0 +1,160 @@
|
||||
# Audit 02: OpenSearch and AWS compatibility of qbit-quick-search
|
||||
|
||||
Date: 2026-10-04. Repo reviewed read-only at `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` (branch `feature/GH-6-repin-and-drift-fix`). Compile experiments were run on a copy under the scratchpad, never in the repo.
|
||||
|
||||
Files reviewed: `src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java` (CLIENT), `OpenSearchDocument.java`, `BulkIndexResult.java`, `QuickSearchQBitConfig.java` (CONFIG), `processes/IndexingUtils.java`, `actions/QuickSearchAction.java`, `pom.xml`, and the three `*IntegrationTest.java` classes.
|
||||
|
||||
## 0. Resolved dependency facts (from `mvn dependency:tree` in the repo)
|
||||
|
||||
| Artifact | Resolved | Note |
|
||||
| --- | --- | --- |
|
||||
| org.opensearch.client:opensearch-java | 2.10.0 | Pinned in pom.xml line 24. Its own POM declares httpclient5 5.3.1, httpcore5 5.2.4, jackson 2.17.0, and a runtime dependency on opensearch-rest-client 2.12.0 (https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/2.10.0/opensearch-java-2.10.0.pom) |
|
||||
| org.opensearch.client:opensearch-rest-client | 2.12.0 (runtime) | Pulled only because the 2.x client still ships the deprecated RestClientTransport; the 3.x client POM no longer lists it (https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/3.9.0/opensearch-java-3.9.0.pom) |
|
||||
| org.apache.httpcomponents.client5:httpclient5 | 5.6.3 | Managed by the qqq BOM |
|
||||
| org.apache.httpcomponents.core5:httpcore5 / httpcore5-h2 | 5.4.3 | Managed by the qqq BOM |
|
||||
| com.fasterxml.jackson.core:* | 2.21.5 | Managed by the qqq BOM |
|
||||
| software.amazon.awssdk:* (auth, regions, apache-client, netty-nio-client, ...) | 2.41.10 | Already on the classpath, but only transitively via qqq-backend-core -> quicksight. Not something this library should rely on without declaring it |
|
||||
|
||||
Maven Central: latest opensearch-java is 3.10.0 (`<latest>3.10.0</latest>` in https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/maven-metadata.xml); the 2.x line ended at 2.26.0. The OpenSearch Java client docs now install 3.10.0 (https://docs.opensearch.org/latest/clients/java/).
|
||||
|
||||
## 1. Research findings (external)
|
||||
|
||||
### 1.1 opensearch-java client
|
||||
|
||||
* Compatibility matrix: client 2.x.0 supports OpenSearch 1.x to 2.x; client 3.x.0 supports 1.x to 3.x. JDK 8/11/17/21 for both lines. (https://github.com/opensearch-project/opensearch-java/blob/main/COMPATIBILITY.md)
|
||||
* `ApacheHttpClient5Transport` is the default and recommended transport; `RestClientTransport` and `org.opensearch.client.RestClient` are deprecated and will be removed. (https://docs.opensearch.org/latest/clients/java/)
|
||||
* OpenSearch 3.0 brings HTTP/2 and `ApacheHttpClient5Transport` switches to HTTP/2 when available; `HttpVersionPolicy` can be forced through `setHttpClientConfigCallback`. (https://github.com/opensearch-project/opensearch-java/blob/main/USER_GUIDE.md)
|
||||
* 3.0.0 breaking changes (https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/CHANGELOG.md and https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/UPGRADING.md): more conservative URL path encoding, `searchAfter` and `Hit.sort` now `FieldValue`, `listAllPit` -> `getAllPits`, several type corrections, removal of ES-only namespaces (`features`, `shutdown`, `termsEnum`, `indices.upgrade`). Nothing in the transport builders or `JacksonJsonpMapper` changed. One undocumented change does hit this repo: `DeleteByQueryRequest.Builder.refresh(...)` takes the `Refresh` enum instead of `boolean` (see empirical test below).
|
||||
* Empirical compile check (scratch copy, `opensearch.version` set to 3.10.0): exactly one compile error, CLIENT line 427 `.refresh(true)` ("boolean cannot be converted to org.opensearch.client.opensearch._types.Refresh"). After changing it to `Refresh.True`, `mvn test-compile` and all 233 unit tests pass. Resolved tree with 3.10.0: httpcore5 5.4.3, httpclient5 5.6.3 (BOM), plus a new `commons-logging:1.3.6` transitive. The 3.x client POM is built against httpclient5 5.6 / httpcore5 5.4.2, so the BOM versions are in range.
|
||||
* httpcore5/httpclient5 constraint with the current 2.10.0 client: it was built against httpclient5 5.3.1 / httpcore5 5.2.4 but runs on 5.6.3 / 5.4.3 here; the repo already carries a workaround for the 5.6 content-compression behaviour (CLIENT lines 124 to 130). Moving to the 3.x client removes that version skew.
|
||||
* Elasticsearch-compatible endpoints: the compatibility matrix lists only OpenSearch. A maintainer answer on the forum says clients generally only bridge adjacent majors (ES 7.10 <-> OpenSearch 1.x) and that 2.x clients against ES 7.10 hit renamed APIs. (https://forum.opensearch.org/t/opensearch-api-compatibility-with-elastic-search/17647). Legacy Elasticsearch 7.10 domains on AWS should be declared out of scope.
|
||||
* SigV4: `AwsSdk2Transport(SdkHttpClient, endpointWithoutScheme, "es" | "aoss", Region, AwsSdk2TransportOptions)`; credentials from the default provider chain; `aws-crt-client` is the recommended SdkHttpClient and the AWS SDK `ApacheHttpClient` is discouraged (it cannot send bodies on GET/DELETE and `AwsSdk2Transport` throws `TransportException` for such calls). (https://docs.opensearch.org/latest/clients/java/#connecting-to-amazon-opensearch-service and https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md). The AWS SDK artifacts are compileOnly in the client, so the application must declare them (release notes for #628 in https://github.com/opensearch-project/opensearch-java/releases/tag/v3.0.0).
|
||||
|
||||
### 1.2 Amazon OpenSearch Service (managed domains)
|
||||
|
||||
* Engine versions offered today: OpenSearch 3.5, 3.3, 3.1, 2.19, 2.17, 2.15, 2.13, 2.11, 2.9, 2.7, 2.5, 2.3, 1.3 to 1.0, plus legacy Elasticsearch up to 7.10. 2.3 to 2.9 left standard support in Nov 2025; 2.11 to 2.17 end Nov 2027. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html)
|
||||
* Endpoint and TLS: "OpenSearch Service only accepts connections over port 80 (HTTP) or 443 (HTTPS)" (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html); TLS 1.2 required, 1.3 recommended (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/infrastructure-security.html). No custom port; endpoints are `https://search-<domain>-<id>.<region>.es.amazonaws.com` (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/createupdatedomains.html).
|
||||
* FGAC: master user is either an IAM principal or an internal-database user. With an internal master user "you can use HTTP basic authentication (as well as IAM credentials)". If the domain access policy names IAM principals, clients must sign with SigV4; "You can't sign a request with a username and password and IAM credentials", so AWS recommends an access policy that does not require signed requests when using basic auth. Master user roles are `all_access` and `security_manager`. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html)
|
||||
* Signing service name for domains is `es`; for Serverless it is `aoss` and `x-amz-content-sha256` is required. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-comparison.html)
|
||||
* Supported operations: the per-version operation lists for managed domains include `/_bulk`, `/_delete_by_query`, `/_refresh`, `/_reindex`, `/_update_by_query` and all index-path operations, so every call this library makes is available on a managed domain. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html)
|
||||
* Request payload limits: 10 MiB on t2/t3 and `*.large` instance types, 100 MiB on xlarge and above. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html)
|
||||
* Permissions on a FGAC domain follow the security plugin model (section 1.4).
|
||||
|
||||
### 1.3 Amazon OpenSearch Serverless (AOSS)
|
||||
|
||||
* Supported API table (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html): `aoss:CreateIndex` (PUT index), `aoss:DescribeIndex` (HEAD index, GET mappings/settings), `aoss:WriteDocument` (POST _bulk, DELETE index/_doc/id, PUT index/_doc/id "for search collection types only", POST index/_doc), `aoss:ReadDocument` (GET/POST _search, _count, _msearch, PIT). `_delete_by_query`, `_update_by_query`, `_reindex` and `_refresh` do not appear anywhere in the table. Aliases exist under `aoss:CreateCollectionItems` / `UpdateCollectionItems`.
|
||||
* Collection types: Search, Vector search, Time series. Custom document IDs and upserts are only allowed for search collections. (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-overview.html and the Python note in https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html)
|
||||
* Settings: "The number of shards, number of intervals, and refresh interval are not modifiable and are handled by OpenSearch Serverless"; refresh interval is approximately 10 seconds for search and time series collections. (serverless-overview.html)
|
||||
* Clients must sign with service `aoss`, include `x-amz-content-sha256`, and "must be compatible with OpenSearch 3.x" (serverless-overview.html; the older serverless-clients.html page still says 2.x). The OpenSearch Java docs note AOSS does not support the `refresh` parameter and does not serve the root endpoint (https://docs.opensearch.org/latest/clients/java/#connecting-to-amazon-opensearch-serverless).
|
||||
* Data access policy minimum for an ingesting principal: `aoss:CreateIndex`, `aoss:WriteDocument`, `aoss:UpdateIndex` on `index/<collection>/<index>`; this library also needs `aoss:DescribeIndex` (exists check) and `aoss:ReadDocument` (search). (serverless-clients.html)
|
||||
* Quotas: 1000 indexes per collection (https://docs.aws.amazon.com/general/latest/gr/opensearch-service.html). No per-request payload limit is documented on that page.
|
||||
* Which of this library's calls fail on AOSS: the connection itself (basic auth transport, no SigV4), `deleteDocumentsForTable` (`_delete_by_query`, CLIENT line 392), `deleteDocumentsIndexedBefore` (`_delete_by_query` with `refresh=true`, CLIENT line 424), `refreshIndex` (`_refresh`, CLIENT line 579), and all indexing with custom `_id` unless the collection is of type Search. `ensureIndexExists`, bulk index/delete, single delete, and search with highlight and from/size are within the supported surface.
|
||||
|
||||
### 1.4 Self-managed OpenSearch with the security plugin
|
||||
|
||||
* Demo config and Docker image: since 2.12 a custom admin password in `OPENSEARCH_INITIAL_ADMIN_PASSWORD` is required or the node does not start; the password must be 8 to 100 chars, mixed classes, and rated "strong" by zxcvbn. `DISABLE_SECURITY_PLUGIN=true` disables the plugin; the demo config installs self-signed certificates. (https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/ and https://docs.opensearch.org/latest/security/configuration/demo-configuration/)
|
||||
* TLS with self-signed certs: the official Java client example builds an `SSLContext` (trust-all for demo, or a truststore) and installs it via `ClientTlsStrategyBuilder` + `PoolingAsyncClientConnectionManager` inside `setHttpClientConfigCallback`; the RestClient transport variant relies on the JVM truststore system properties. (https://docs.opensearch.org/latest/clients/java/#connecting-to-opensearch)
|
||||
* Auth backends: HTTP basic against the internal user database; client certificate (mTLS) via `plugins.security.ssl.http.clientauth_mode` and a `clientcert` auth domain mapped to roles by CN (https://docs.opensearch.org/latest/security/authentication-backends/client-auth/); JWT bearer tokens (HMAC, RSA, ECDSA, JWKS since 3.3) in the `Authorization` header (https://docs.opensearch.org/latest/security/authentication-backends/jwt/); API tokens with permissions bound to the token were introduced in OpenSearch 3.7 (https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-3.7.0.md).
|
||||
* Permissions for this library's operations (default action groups, https://docs.opensearch.org/latest/security/access-control/default-action-groups/): index-level `create_index` (`indices:admin/create`, `indices:admin/mapping/put`) for `ensureIndexExists`; `read` or `search` for `_search`; `write` (`indices:data/write*`, which covers index, bulk items, delete, and `delete/byquery`) and `delete`; `indices:admin/refresh*` is not in any default group except `manage` (`indices:admin/*`) and must be added for `refreshIndex`; cluster-level `cluster_composite_ops` for `indices:data/write/bulk`. The exists check (HEAD index) also needs an `indices:admin/*` read permission; verify the exact action name against the permissions reference when writing the README role.
|
||||
* Versions: 2.x is in maintenance (latest 2.19.6, maintained until 4.0 GA); 3.x is current (3.9.0 released 2026-09-29; 3.10.0 scheduled Nov 2026). (https://opensearch.org/releases/)
|
||||
|
||||
### 1.5 Testcontainers
|
||||
|
||||
* Docker Hub has `opensearchproject/opensearch` tags 2.19.0 to 2.19.6 and 3.0.0 to 3.9.0 (https://hub.docker.com/v2/repositories/opensearchproject/opensearch/tags).
|
||||
* The official module `org.opensearch:opensearch-testcontainers` is at 4.1.0 (requires Testcontainers 2.0.0+ and JDK 17+); 3.0.x works with Testcontainers 1.20.0+ and JDK 21. `OpenSearchContainer` disables security by default and offers `withSecurityEnabled()` with HTTPS plus generated credentials. (https://github.com/opensearch-project/opensearch-testcontainers/blob/main/README.md and https://repo1.maven.org/maven2/org/opensearch/opensearch-testcontainers/maven-metadata.xml)
|
||||
|
||||
### 1.6 Server-side behaviours relevant to the mapping and query design
|
||||
|
||||
* `index.max_result_window` default 10000 caps `from + size` (https://docs.opensearch.org/latest/install-and-configure/configuring-opensearch/index-settings/).
|
||||
* Search analyzer resolution order: query `analyzer`, field `search_analyzer`, index `default_search`, field `analyzer`, `standard`. The docs' edge n-gram example sets `analyzer: edge_ngram_analyzer` with `search_analyzer: standard` so the query is not n-grammed. (https://docs.opensearch.org/latest/analyzers/search-analyzers/)
|
||||
* Edge n-gram filter: `max_gram` too low means long search terms do not match; the docs suggest a `truncate` filter as search analyzer to mitigate. (https://docs.opensearch.org/latest/analyzers/token-filters/edge-ngram/)
|
||||
* Dynamic mapping: strings become `text` + `keyword` subfield, or `date` when `date_detection` (default on) matches `strict_date_optional_time`; JSON numbers become `long`/`float`; `index.mapping.total_fields.limit` defaults to 1000; `flat_object` or `dynamic_templates` are the recommended defences. (https://docs.opensearch.org/latest/mappings/ and https://docs.opensearch.org/latest/mappings/mapping-explosion/)
|
||||
|
||||
## 2. Compatibility matrix
|
||||
|
||||
| Target | Works today? | What breaks | Change needed |
|
||||
| --- | --- | --- | --- |
|
||||
| OpenSearch 2.x self-managed, security disabled (the integration-test setup) | Yes | Nothing; this is the only tested configuration (image 2.11.0) | Test against 2.19.x as well (F10) |
|
||||
| OpenSearch 2.x self-managed, security plugin, basic auth over TLS | Partly | Demo or private-CA certificates fail the JVM default trust check; no truststore or insecure option exists (CONFIG exposes only `useSsl`). Only basic auth is wired; no client-cert, JWT or API-token path | Add TLS trust configuration (F1), document required roles (F13); optional auth-header or SSLContext hooks (F3) |
|
||||
| OpenSearch 3.x self-managed | Not supported by the pinned client | opensearch-java 2.10.0 is documented for 1.x to 2.x only; the server negotiates HTTP/2 with the Apache 5 transport | Bump to opensearch-java 3.10.0; one-line change at CLIENT line 427 (F2); add 3.x to the IT matrix (F10) |
|
||||
| AWS managed domain 2.x or 3.x, FGAC with internal-database master user, HTTP basic auth | Yes for 2.x (3.x after F2) | Needs `opensearchPort = 443`, `useSsl = true`, and a domain access policy that does not require signed requests. AWS certificates are publicly trusted so TLS works. Every API the library calls is on the managed-domain supported list. Payload limit 10 MiB on small instances with count-only batching (F8); no timeouts (F9) | F2 for 3.x domains, F8, F9, README guidance |
|
||||
| AWS managed domain, IAM master user or IAM principals in the access policy (SigV4) | No | Client only does basic auth; AWS rejects unsigned requests when IAM principals are in the policy; no `AwsSdk2Transport` path, region, or service-name configuration | Add an optional SigV4 transport mode: `AwsSdk2Transport` with service `es`, `Region`, default credential chain, `aws-crt-client`; declare the AWS SDK artifacts as optional dependencies (F3) |
|
||||
| Amazon OpenSearch Serverless (AOSS) | No | Transport (SigV4 `aoss`, `x-amz-content-sha256`); `_delete_by_query` used by FullReindexStep and ReconcileIndexStep; `_refresh` used by ReconcileIndexStep; custom `_id` only on Search collections; refresh lag ~10 s; client must be 3.x | Declare unsupported for 1.0. Supporting it later means: SigV4 transport (F3), replace delete-by-query with search + bulk delete, replace refresh with a bounded wait, and require Search collections (F4) |
|
||||
| Legacy Elasticsearch 7.10 domains or other ES-compatible endpoints | Untested, not documented | Client matrix lists only OpenSearch; renamed APIs across the 7.10 to 2.x gap | Declare out of scope |
|
||||
|
||||
## 3. Findings
|
||||
|
||||
Severity: High = breaks a target platform or returns wrong results; Medium = operational risk or avoidable failure; Low = hygiene.
|
||||
|
||||
| ID | Severity | Finding | Evidence | Recommended change | 1.0 blocker |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| F1 | High | No TLS trust configuration. `useSsl` only flips the scheme; there is no truststore path, PEM CA, or insecure toggle, so self-managed clusters with demo or private-CA certificates fail the handshake. Hostname verification cannot be relaxed either | CLIENT lines 98 to 139; CONFIG lines 54, 420 to 444; official pattern in https://docs.opensearch.org/latest/clients/java/#connecting-to-opensearch | Add `trustStorePath`/`trustStorePassword` (or `caCertificatePem`) and an explicit `insecureSkipTlsVerify` flag; build an `SSLContext` and install it through `ClientTlsStrategyBuilder` + `PoolingAsyncClientConnectionManager` in the existing `setHttpClientConfigCallback`. Log a warning when verification is disabled | Yes |
|
||||
| F2 | High | opensearch-java pinned to 2.10.0 (2.x line, documented for OpenSearch 1.x to 2.x). AWS already offers 3.1 to 3.5 and 3.x is the current community line. Empirical check: bumping to 3.10.0 yields a single compile error at CLIENT line 427 (`refresh(true)` must become `refresh(Refresh.True)`); 233 unit tests pass afterwards. 3.x also drops the transitive `opensearch-rest-client` and matches the BOM's httpclient5 5.6 / httpcore5 5.4 | pom.xml line 24; COMPATIBILITY.md; CHANGELOG/UPGRADING links in section 1.1; scratch compile log | Bump `opensearch.version` to 3.10.0, fix line 427, re-run ITs; keep the `disableContentCompression()` workaround until proven unnecessary | Yes |
|
||||
| F3 | High (for AWS IAM) | Only HTTP basic auth is supported. No SigV4 (`AwsSdk2Transport`), no region or service name, no bearer-token or client-certificate hooks. An IAM master user or any IAM principal in the domain access policy makes the domain unreachable | CLIENT lines 109 to 137; CONFIG lines 52, 53; fgac.html; clients/java docs | Introduce an `authMode` (BASIC, AWS_SIGV4, NONE) plus `awsRegion`, `awsServiceName` (default `es`), optional credentials provider; build `AwsSdk2Transport` with `AwsCrtHttpClient`. Declare `software.amazon.awssdk:aws-crt-client` and `auth` as `<optional>true</optional>` rather than relying on the quicksight transitive. Optionally expose a `Consumer<HttpAsyncClientBuilder>` hook for JWT/API-token headers and mTLS | No (declare unsupported in README) |
|
||||
| F4 | High (for AOSS) | AOSS cannot be targeted: SigV4 `aoss` missing; `_delete_by_query` (CLIENT 392, 424) and `_refresh` (CLIENT 579) are not in the AOSS supported-operations table; custom `_id` requires a Search collection; `refresh=true` parameter unsupported | serverless-genref.html; serverless-overview.html; CLIENT lines 388 to 445, 575 to 585 | Declare AOSS unsupported for 1.0. For a later release: F3, implement table wipe and stale-document removal as search (PIT or `search_after`) + bulk delete, replace `refreshIndex` with a polling wait, document the Search-collection requirement and the `aoss:CreateIndex/DescribeIndex/WriteDocument/ReadDocument/UpdateIndex` data-access policy | No |
|
||||
| F5 | High | Edge n-gram analyzer (min 2, max 20) is set as the field `analyzer` with no `search_analyzer`, so the query string is also n-grammed. A search for "search" becomes se, sea, sear, searc, search and matches every document containing any token starting with "se", inflating hits and flattening relevance. Tokens longer than 20 chars only index their first 20 grams, and single-character terms never match | CLIENT lines 173 to 192 (`searchableText` property), 472 to 475 (must clause on `searchableText`); search-analyzers doc | Add a second analyzer (`standard` tokenizer + `lowercase`) and set it as `search_analyzer` on `searchableText`; consider `truncate` to 20 in the search analyzer or `preserve_original` so long terms still match; keep `min_gram 2` but document it. Requires index recreation (bump an index-version suffix or document a full reindex) | Yes |
|
||||
| F6 | High | `fieldValues` is a dynamic `object` that receives raw `QRecord` values (`Integer`, `BigDecimal`, `Instant`, `LocalDate`, `Boolean`, `String`). Dynamic mapping turns them into `long`/`float`/`date`/`boolean`; the first table to index a field name fixes its type for the whole index, so a `status` that is a string in one table and an integer in another causes `mapper_parsing_exception` on the second table. The boost `multi_match` clauses target `fieldValues.<field>` without `lenient: true`, so a non-numeric or non-date search term against a numeric or date `fieldValues` field makes the whole search request fail. Field count also grows with every configured field across all tables toward `index.mapping.total_fields.limit` (1000) | IndexingUtils lines 144 to 156; CLIENT lines 192, 525 to 565; mappings docs (date detection on by default; dynamic rules); mapping-explosion doc | Map `fieldValues` with a `dynamic_templates` rule that maps every `fieldValues.*` string to `text` (edge n-gram analyzer + standard search analyzer) and set `date_detection: false`; convert values to strings in `IndexingUtils` (`getValueString`) so types can never conflict; set `lenient(true)` on the boost `multi_match` queries; consider `flat_object` if per-field boosting is not needed | Yes |
|
||||
| F7 | Medium | `limit` and `offset` are unbounded; `from + size > 10000` returns a 400 (`index.max_result_window`) and large `size` values are accepted verbatim | QuickSearchAction lines 46 to 79; CLIENT 497 to 502; index-settings doc | Cap `limit` (for example 100) and reject or clamp `offset + limit > 10000` with a clear error; document that deep paging is not supported | No |
|
||||
| F8 | Medium | Bulk batching is count-based only (`bulkBatchSize` default 500) with no byte cap. On AWS small instance types the HTTP payload limit is 10 MiB; wide records with large text fields can exceed it and the whole batch fails with 413 | CLIENT lines 241 to 274; CONFIG line 58; AWS limits.html | Flush a batch when its serialized size approaches a configurable byte limit (default 5 MiB) in addition to the count; surface 413 clearly | No |
|
||||
| F9 | Medium | No connect, response or connection-request timeouts and no pool sizing; a hung node blocks the caller indefinitely; single host only, no failover list | CLIENT lines 105 to 139; no `RequestConfig`, `setConnectionManager`, or timeout call anywhere in `src/main/java` | Add `connectTimeoutMs`, `responseTimeoutMs`, `maxConnections` to CONFIG and apply via `setDefaultRequestConfig` and `PoolingAsyncClientConnectionManagerBuilder`; accept a list of hosts (`ApacheHttpClient5TransportBuilder.builder(HttpHost...)`) | Yes |
|
||||
| F10 | Medium | Integration tests pin `opensearchproject/opensearch:2.11.0` (Oct 2023). Nothing exercises 2.19.x or any 3.x, although AWS offers 2.19 and 3.1 to 3.5 and the community line is 3.9 | OpenSearchIntegrationTest line 76; IndexDriftIntegrationTest line 81; ConfigDrivenIndexingIntegrationTest line 76; releases page; Docker Hub tags | Parameterize the image via a system property (`-Dopensearch.image=...`) and run two CI jobs: `2.19.6` and `3.9.0`. Optionally adopt `org.opensearch:opensearch-testcontainers` 3.0.2 (Testcontainers 1.20+, JDK 21) now and 4.1.0 after moving to Testcontainers 2.x | Yes |
|
||||
| F11 | Low | ITs disable security via `plugins.security.disabled=true` while also passing `OPENSEARCH_INITIAL_ADMIN_PASSWORD=Admin123!`. Harmless today, but if security is ever enabled that password is unlikely to pass the zxcvbn "strong" check required since 2.12, and the documented image switch is `DISABLE_SECURITY_PLUGIN=true` | the three IT classes lines 79 to 85; demo-configuration doc | Use `DISABLE_SECURITY_PLUGIN=true` (and `DISABLE_INSTALL_DEMO_CONFIG=true`); drop the password, or when adding a secured IT, generate a strong one and connect over HTTPS with F1 in place | No |
|
||||
| F12 | Low | `opensearch-rest-client:2.12.0` and `httpcore5` 5.2.x come in transitively from the 2.10.0 client even though `RestClientTransport` is never used; version skew with the BOM's httpclient5 5.6.3 already required the compression workaround | dependency tree; CLIENT lines 124 to 130 | Resolved by F2; otherwise exclude `opensearch-rest-client` | No |
|
||||
| F13 | Low | No documented least-privilege role for the security plugin or AWS FGAC | none in repo | README: index pattern `<opensearchIndexName>`, groups `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*` and an `indices:admin/*` read for the exists check; cluster `cluster_composite_ops`. For AWS FGAC map the same to the basic-auth user; for AOSS list the five `aoss:*` permissions | No |
|
||||
| F14 | Low | Composite `_id` is `sourceTable + ":" + recordId`; `recordId` is mapped `keyword` (correct for term filters and mixed PK types). A table name containing ":" would be ambiguous, and `_id` is limited to 512 bytes | OpenSearchDocument lines 184 to 187; CLIENT line 188 | Validate table names against ":" in CONFIG; keep `keyword` | No |
|
||||
| F15 | Info | Against 3.x servers the Apache 5 transport negotiates HTTP/2 automatically; nothing in the code forces a version | USER_GUIDE.md HTTP/2 note | No change; mention in README; `HttpVersionPolicy.FORCE_HTTP_1` is available if a proxy misbehaves | No |
|
||||
|
||||
## 4. Recommended "supported platforms" statement for the 1.0 README
|
||||
|
||||
Supported (tested in CI):
|
||||
|
||||
* Self-managed OpenSearch 2.19.x and 3.x (latest minor), reached over HTTP or HTTPS with a certificate the JVM trusts or a configured truststore, with the security plugin disabled or with HTTP basic authentication against the internal user database. Required role: `create_index`, `read`, `write`, `delete` on the quick-search index plus `indices:admin/refresh*`, and the cluster-level `cluster_composite_ops` group.
|
||||
* Amazon OpenSearch Service managed domains running OpenSearch 2.x or 3.x with fine-grained access control, an internal-database master user (or a mapped internal user), HTTP basic authentication, `opensearchPort = 443`, `useSsl = true`, and a domain access policy that does not require signed requests. Keep `bulkBatchSize` small enough for the instance type's HTTP payload limit (10 MiB on t3 and `*.large`).
|
||||
|
||||
Not supported in 1.0:
|
||||
|
||||
* IAM / SigV4 authentication to Amazon OpenSearch Service (IAM master user or IAM principals in the access policy).
|
||||
* Amazon OpenSearch Serverless (requires SigV4 with service `aoss` and does not offer `_delete_by_query` or `_refresh`, which the full-reindex and reconcile processes use).
|
||||
* Legacy Elasticsearch 7.10 domains and other Elasticsearch-compatible endpoints.
|
||||
* OpenSearch 1.x.
|
||||
|
||||
Pre-conditions for the statement above: F1, F2, F5, F6, F9 and F10 must land first; the statement should name the exact image tags the CI matrix runs.
|
||||
|
||||
## 5. Source list
|
||||
|
||||
* https://github.com/opensearch-project/opensearch-java/blob/main/COMPATIBILITY.md
|
||||
* https://docs.opensearch.org/latest/clients/java/
|
||||
* https://github.com/opensearch-project/opensearch-java/blob/main/USER_GUIDE.md
|
||||
* https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md
|
||||
* https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/CHANGELOG.md
|
||||
* https://raw.githubusercontent.com/opensearch-project/opensearch-java/main/UPGRADING.md
|
||||
* https://github.com/opensearch-project/opensearch-java/releases/tag/v3.0.0
|
||||
* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/maven-metadata.xml
|
||||
* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/2.10.0/opensearch-java-2.10.0.pom
|
||||
* https://repo1.maven.org/maven2/org/opensearch/client/opensearch-java/3.9.0/opensearch-java-3.9.0.pom
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/infrastructure-security.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-operations.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-overview.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-comparison.html
|
||||
* https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html
|
||||
* https://docs.aws.amazon.com/general/latest/gr/opensearch-service.html
|
||||
* https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
|
||||
* https://docs.opensearch.org/latest/security/configuration/demo-configuration/
|
||||
* https://docs.opensearch.org/latest/security/access-control/default-action-groups/
|
||||
* https://docs.opensearch.org/latest/security/authentication-backends/client-auth/
|
||||
* https://docs.opensearch.org/latest/security/authentication-backends/jwt/
|
||||
* https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-3.7.0.md
|
||||
* https://opensearch.org/releases/
|
||||
* https://hub.docker.com/v2/repositories/opensearchproject/opensearch/tags
|
||||
* https://github.com/opensearch-project/opensearch-testcontainers/blob/main/README.md
|
||||
* https://docs.opensearch.org/latest/install-and-configure/configuring-opensearch/index-settings/
|
||||
* https://docs.opensearch.org/latest/analyzers/search-analyzers/
|
||||
* https://docs.opensearch.org/latest/analyzers/token-filters/edge-ngram/
|
||||
* https://docs.opensearch.org/latest/mappings/
|
||||
* https://docs.opensearch.org/latest/mappings/mapping-explosion/
|
||||
* https://forum.opensearch.org/t/opensearch-api-compatibility-with-elastic-search/17647
|
||||
@@ -0,0 +1,156 @@
|
||||
# Audit 03: Authentication, secrets handling, and connection security
|
||||
|
||||
Repo: `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1` at `7cd94ae` (branch `feature/GH-6-repin-and-drift-fix`), read-only.
|
||||
QQQ reference: `/Users/james.maes/Git.Local/QRunIO/qqq` at tag `v4.1.0-RC.1`.
|
||||
Resolved runtime versions (from `mvn -B dependency:tree`): opensearch-java 2.10.0, opensearch-rest-client 2.12.0 (runtime), httpclient5 5.6.3, httpcore5 5.4.3, jackson 2.21.5, log4j 2.25.5, AWS SDK v2 2.41.10 (via qqq-backend-core), AWS SDK v1 1.12.797 (via qqq-backend-core).
|
||||
|
||||
Paths below are relative to `src/main/java/com/kingsrook/qbits/quicksearch/` unless stated otherwise.
|
||||
|
||||
## Findings
|
||||
|
||||
| ID | Severity | Finding | Evidence | Recommended change | 1.0 blocker |
|
||||
|----|----------|---------|----------|--------------------|-------------|
|
||||
| A1 | Critical | `QuickSearchAction` returns `tableName`, `recordId`, `recordLabel` and a `highlightSnippet` (fragments of `searchableText`, which is the concatenation of every configured field value) for every indexed table, with no check of the caller's session, table permission, or record security locks. The caller-supplied `tableName` filter is also unchecked. Any authenticated (or anonymous, if the instance allows it) caller can enumerate labels and field values across all indexed tables. | `actions/QuickSearchAction.java:81-121` (reads client and all `discoveredTables` from the static context, maps every hit straight to output); `opensearch/QuickSearchOpenSearchClient.java:466-510` (only filter is the optional caller-supplied `sourceTable` term); `processes/IndexingUtils.java:83,136-165` (searchableText and fieldValues built from all configured fields) | See section "A1 fix" below: make `QuickSearchInput` extend `AbstractActionInput`, compute the set of tables the session may read with `PermissionsHelper.hasTablePermission(input, table, READ)`, reject or ignore a requested `tableName` outside that set, pass the allowed set into the client as a `terms` filter on `sourceTable`, then post-filter hits per table through `QueryAction` by primary key so record security locks apply. Add negative tests. | Yes |
|
||||
| A2 | High | HTTP basic credentials are sent in cleartext when `useSsl` is false (the default), and nothing in `validate()` objects to username/password combined with plain HTTP. The README example and the qqq-all `DemoQBits` default `OPENSEARCH_SSL` to false. | `QuickSearchQBitConfig.java:54` (`useSsl = false`), `:103-112` (only checks that username and password are set together); `opensearch/QuickSearchOpenSearchClient.java:98-99,113-119`; `README.md:303`; qqq-all `DemoQBits.java:156-158` | In `validate()`, add an error when `authMode` is BASIC (or username/password set) and TLS is off, unless an explicit `allowPlaintextCredentials` (dev only) is true. For 1.0 change the default scheme to HTTPS (see config sketch) so that opting into plaintext is deliberate. | Yes |
|
||||
| A3 | High | The password lives as a plain `String` on a `Serializable` `QBitConfig`, and the downstream host attaches that same object to `QBitMetaData` which is stored in `QInstance.getQBits()`. Anything that serializes the `QInstance` (for example `QInstanceAdapter.qInstanceToJson`, used by dev tooling and tests) or any code that walks `getQBits()` can read the password through the public getter. No `@JsonIgnore`, no redaction, no `toString()` override (the default `Object.toString()` happens not to leak it, but only by accident). | `QuickSearchQBitConfig.java:52-53,389-392`; qqq `QBitMetaData.java` (field `QBitConfig config`, `getConfig()`); qqq `QInstance.java:114,1800-1836`; qqq `adapters/QInstanceAdapter.java:46,57`; qqq-all `DemoQBits.java:157,162-163`; qqq-all `FullQBitsTest.java:78` (casts the config back out of `getQBits()`) | Do not keep the raw secret on the metadata object. Options, cheapest first: (1) mark `getOpensearchPassword()` (and any token/secret getter) `@JsonIgnore` and add a `toString()` that redacts; (2) accept `${env.X}` / `${prop.X}` references and resolve them with `QMetaDataVariableInterpreter` only inside the client constructor, so the config object stores the reference string, not the value (this is exactly how `RDBMSBackendMetaData.enrich()`, `MongoDBBackendMetaData.enrich()` and `S3BackendMetaData.enrich()` handle `password`/`secretKey`); (3) longer term, resolve credentials through a `QCodeReference` provider so no secret ever sits on metadata. Do (1) and (2) for 1.0. | Yes |
|
||||
| A4 | Medium | Only two auth modes exist: none and HTTP basic. There is no AWS SigV4 support, which blocks Amazon OpenSearch Service domains using IAM (and OpenSearch Serverless entirely), and there is no way to inject a bearer token, API-key header, or client certificate without forking the client. | `opensearch/QuickSearchOpenSearchClient.java:94-147` (hard-wires `ApacheHttpClient5TransportBuilder` + `BasicCredentialsProvider`) | Add an `authMode` enum (NONE, BASIC, AWS_SIGV4) and a transport-customizer SPI for the rest. `AwsSdk2Transport` ships in opensearch-java 2.10.0 and the AWS SDK v2 core modules are already on the compile classpath via qqq-backend-core, so SigV4 is mostly wiring (details in section 1). | Yes (SigV4); No (bearer, API key, mTLS via SPI) |
|
||||
| A5 | Medium | TLS trust is not configurable. The transport uses `SSLContext.getDefault()` (JVM trust store) and the httpclient5 default hostname verifier, so certificates are verified correctly by default, but a private CA, a custom trust store, or client-certificate material can only be supplied through JVM system properties. There is no explicit, loud "insecure" switch for local development, which pushes developers toward global `-Djavax.net.ssl.trustStore` hacks or disabling the security plugin. | opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` bytecode: `ClientTlsStrategyBuilder.create().setSslContext(SSLContext.getDefault())`; `opensearch/QuickSearchOpenSearchClient.java:105-137` never sets a TLS strategy | Add `tls.caCertificatePath` (PEM) or `tls.trustStorePath` + `tls.trustStorePassword` (reference-interpreted), `tls.hostnameVerification` (default true), and `tls.insecureSkipVerify` (default false; refuse unless host is loopback or `allowInsecureInProduction` is set; log a WARN at startup). Build an `SSLContext` via `SSLContextBuilder.loadTrustMaterial(...)` and pass it through `ClientTlsStrategyBuilder` on a `PoolingAsyncClientConnectionManager`. | No (strongly recommended) |
|
||||
| A6 | Medium | The three produced processes (`quickSearchBasepullIndex`, `quickSearchFullReindex`, `quickSearchReconcileIndex`), the two operational tables, and the `quickSearchAdmin` app carry no `QPermissionRules`, so they inherit `QInstance.defaultPermissionRules` (QQQ default is `NOT_PROTECTED`). In a host that has not tightened the instance default, any user can trigger a full reindex (which wipes the index first, a search blackout) or read/edit the run tables. | `QuickSearchQBitProducer.java:457-549` (tables and processes built without `withPermissionRules`), `:246-260` (app); `grep PermissionRules src/main` returns nothing; qqq `QInstance.java:140` (`defaultPermissionRules = QPermissionRules.defaultInstance()`); qqq `QJavalinProcessHandler.java:404,418` (middleware enforces whatever rules the process declares) | Set explicit rules in the producer: processes and app `HAS_ACCESS_PERMISSION`, tables `READ_WRITE_PERMISSIONS`, with a stable `permissionBaseName` (for example `quickSearchAdmin`) so one grant covers the admin surface. Make it overridable via an `adminPermissionRules` config field; never default to `NOT_PROTECTED`. | Yes |
|
||||
| A7 | Medium | Transport timeouts are the opensearch-java defaults: connect 1 s, response 30 s, max 10 connections per route / 30 total, automatic httpclient retries disabled. None are configurable. 1 s connect is tight for cross-region or cold AWS endpoints; 30 s response is marginal for `deleteByQuery(... refresh=true)` on large tables (`deleteDocumentsIndexedBefore`, `deleteDocumentsForTable`) and for 500-document bulk requests on a slow cluster. There is no explicit user-agent. | opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` constants `DEFAULT_CONNECT_TIMEOUT_MILLIS=1000`, `DEFAULT_RESPONSE_TIMEOUT_MILLIS=30000`, `DEFAULT_MAX_CONN_PER_ROUTE=10`, `DEFAULT_MAX_CONN_TOTAL=30`, and `HttpAsyncClientBuilder.disableAutomaticRetries()` in its `createHttpClient`; `opensearch/QuickSearchOpenSearchClient.java:105-137` sets no `RequestConfigCallback` | Add `connectTimeoutMillis` (default 5000), `responseTimeoutMillis` (default 60000), `maxConnections` (default 30) to config; apply via `setRequestConfigCallback` and `PoolingAsyncClientConnectionManagerBuilder`. Set a default header `User-Agent: qbit-quick-search/<version>` so cluster-side logs can attribute traffic. Keep retries off at the HTTP layer; the indexing steps already record per-item failures. | No |
|
||||
| A8 | Medium | The OpenSearch client is never closed on host shutdown. `QuickSearchOpenSearchClient` is `Closeable` and `SynchronousIndexEventPublisher.close()` calls `client.close()`, but nothing invokes `publisher.close()` or `client.close()` outside tests; the static `QuickSearchQBitContext` has no lifecycle hook. The pooled connections and the httpclient5 I/O reactor thread leak until JVM exit, and a hot-reload of the `QInstance` (which re-runs `produce`) creates a second transport while the first stays open. | `QuickSearchQBitContext.java:34-37,152-158`; `QuickSearchQBitProducer.java:159-203`; `publisher/SynchronousIndexEventPublisher.java:179-181`; `opensearch/QuickSearchOpenSearchClient.java:595-612` | In `produce`, close any previously stored client/publisher before replacing them, and register a `Runtime.getRuntime().addShutdownHook` (or the QQQ instance shutdown hook if the host exposes one) that calls `QuickSearchQBitContext.shutdown()`. | No |
|
||||
| A9 | Low | Search terms are logged at DEBUG (`"term", normalizedTerm`). Search terms are frequently PII (names, emails, order numbers). Not a secret leak, but worth knowing before enabling DEBUG in production. | `actions/QuickSearchAction.java:89` | Log term length or a hash at DEBUG; log the raw term only at TRACE. | No |
|
||||
| A10 | Low | `QuickSearchQBitConfig implements QBitConfig` (which `extends Serializable`) but holds a non-`Serializable` `IndexEventPublisher` and a `List<Class<?>>`. Java serialization of the config (which QQQ permits by contract) would throw `NotSerializableException` when a custom publisher is set. Not a security issue, but it is the same contract that A3 relies on. | `QuickSearchQBitConfig.java:45,60,63`; qqq `QBitConfig.java` (`public interface QBitConfig extends Serializable`) | Mark `indexEventPublisher` `transient` or replace it with a `QCodeReference`. | No |
|
||||
| A11 | Low | `limit` and `offset` are unbounded. A caller can request `limit=100000`; OpenSearch rejects `from+size > index.max_result_window` (10000) with an error that surfaces as a `QException`, and smaller but large pages pull `fieldValues` for every hit into heap. Combined with A1 this makes bulk enumeration trivial. | `actions/QuickSearchAction.java:78-79` | Clamp `limit` to a `maxLimit` (default 100) and `offset + limit` to 10000, and document it. | No |
|
||||
| A12 | Info | No committed production credentials. Test fixtures use placeholders (`"user"`, `"secret"`, `"pass"`, `"admin"`) and the Testcontainers `OPENSEARCH_INITIAL_ADMIN_PASSWORD=Admin123!` with `plugins.security.disabled=true`, so the password is never used. No `.env`, properties, or YAML with secrets in the tree; `.gitignore` excludes `.env` and `.env.local`. The README example hard-codes `"secretPassword"` as a literal, which teaches the wrong pattern. | `src/test/.../QuickSearchQBitConfigTest.java:229-278,509-555`; `src/test/.../QuickSearchOpenSearchClientTest.java:89-90`; `src/test/.../OpenSearchIntegrationTest.java:75-81` (and the two other integration tests); `README.md:115-125` | Change the README example to `.withOpensearchPassword("${env.OPENSEARCH_PASSWORD}")` once A3 lands. | No |
|
||||
| A13 | Info | Logging and exception messages do not include the host, port, username or password. `validate()` emits only field names, `backendName` and table names; the client's `QException`s include `indexName`, document ids, the search term, and the underlying `e.getMessage()` (which for connection failures from httpclient5 can include `host:port`, acceptable). | `QuickSearchQBitConfig.java:73-159`; `opensearch/QuickSearchOpenSearchClient.java:145,199,222,351,376,401,443,508,583`; `QuickSearchQBitProducer.java:121,166,180` | Keep as is; add the `toString()` redaction from A3 so a future `LOG.info("config", config)` cannot regress this. | No |
|
||||
| A14 | Info | Supply chain: no concrete CVE affects the resolved runtime set. httpclient5 5.6.3 is past the fix for CVE-2025-27820 (PSL/hostname-verification bypass in 5.4.0 to 5.4.2, fixed 5.4.3; see https://nvd.nist.gov/vuln/detail/CVE-2025-27820 and https://github.com/advisories/GHSA-73m2-qfq3-56cx). jackson-core 2.21.5 is past CVE-2025-52999 (nesting-depth DoS, fixed 2.15.0; https://nvd.nist.gov/vuln/detail/CVE-2025-52999). log4j 2.25.5 has no open advisories. No published advisory was found for opensearch-java 2.10.0 or opensearch-rest-client 2.12.0 (searched NVD and GitHub advisories on 2026-10-04). Note, not a CVE: qqq-backend-core still drags in AWS SDK for Java v1 1.12.797, which reached end of support on 2025-12-31 (https://aws.amazon.com/blogs/developer/announcing-end-of-support-for-aws-sdk-for-java-v1-x-on-december-31-2025/); that is a qqq concern, not this repo's, and this QBit should use SDK v2 only. | `pom.xml:22` (`opensearch.version 2.10.0`), `:74-83` (httpclient5 and jackson unpinned, versions from the qqq BOM); `mvn -B dependency:tree` output | Nothing to pin. Consider moving opensearch-java to the 2.x line that matches the target cluster (the client's `opensearchVersion` is 2.12.0) when the next qqq BOM bump happens; keep httpclient5 and jackson BOM-managed as ADR-0007 already requires. | No |
|
||||
|
||||
## 1. Auth modes: today versus 1.0
|
||||
|
||||
| Mode | Supported today | What it needs | Notes |
|
||||
|------|-----------------|---------------|-------|
|
||||
| None (local dev, security plugin disabled) | Yes (omit username/password) | Nothing | Should stay the default only for `localhost`; see A2. |
|
||||
| HTTP basic (security plugin internal user, or Amazon OpenSearch fine-grained access control with an internal master user) | Yes | Nothing new; fix A2 (require TLS) and A3 (`${env.}` references) | Current implementation uses `BasicCredentialsProvider` scoped to the host, which is correct. Consider pre-emptive auth (`BasicScheme` in a `HttpClientContext`) to save one 401 round trip per connection; optional. |
|
||||
| AWS IAM SigV4, service `es` (Amazon OpenSearch Service domain with IAM) | No | `org.opensearch.client.transport.aws.AwsSdk2Transport` (already inside opensearch-java 2.10.0); `software.amazon.awssdk:sdk-core`, `auth`, `regions`, `http-client-spi` (already compile-scope via qqq-backend-core at 2.41.10); one `SdkHttpClient` implementation declared explicitly, `software.amazon.awssdk:apache-client` (currently only transitive runtime scope) or `url-connection-client`; `software.amazon.awssdk:sts` if assume-role is wanted (not currently on the classpath) | `new AwsSdk2Transport(httpClient, endpointHost, "es", Region.of(region), AwsSdk2TransportOptions.builder().setCredentials(provider).build())`. Default provider is `DefaultCredentialsProvider` (env, profile, container, instance role). Assume-role: `StsAssumeRoleCredentialsProvider`. Reference: https://github.com/opensearch-project/opensearch-java/blob/main/guides/auth.md and https://docs.opensearch.org/latest/clients/java/ |
|
||||
| AWS IAM SigV4, service `aoss` (OpenSearch Serverless) | No | Same as above with signing service name `"aoss"` | Serverless rejects some operations this QBit uses; verify `refreshIndex()` and `deleteByQuery(... refresh=true)` against the supported-operations list before claiming support (https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html). Reference for `aoss`: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html |
|
||||
| Bearer / JWT token | No | `ApacheHttpClient5TransportBuilder.setDefaultHeaders(new BasicHeader("Authorization", "Bearer " + token))` for a static token; a `Supplier<String>` plus an `HttpRequestInterceptor` for rotating tokens | The security plugin supports JWT as an HTTP authenticator (https://docs.opensearch.org/latest/security/authentication-backends/authc-index/). Static tokens fit a header field; rotation needs the SPI below. |
|
||||
| API key header | No | Same mechanism as bearer, custom header name and value | Not a native OpenSearch security-plugin concept (OpenSearch has no Elasticsearch-style API keys); treat as "arbitrary static header", covered by the SPI. |
|
||||
| Client certificate mTLS | No | `SSLContextBuilder.loadKeyMaterial(keyStore, keyPassword)` plus `loadTrustMaterial`, passed through `ClientTlsStrategyBuilder` to a `PoolingAsyncClientConnectionManager` set on the `HttpAsyncClientBuilder` | Security plugin side: `plugins.security.ssl.http.clientauth_mode: OPTIONAL` or `REQUIRE` (https://docs.opensearch.org/latest/security/authentication-backends/client-auth/). Fits naturally under the `tls` block (`keyStorePath`, `keyStorePassword`). |
|
||||
|
||||
Recommended minimum for 1.0: `authMode` enum with three values, NONE, BASIC, AWS_SIGV4 (with `awsRegion`, `awsServiceName` defaulting to `es`, optional `awsAssumeRoleArn`), plus one escape hatch, `transportCustomizer` as a `QCodeReference` to an interface such as
|
||||
|
||||
public interface OpenSearchTransportCustomizer
|
||||
{
|
||||
default ApacheHttpClient5TransportBuilder customizeTransport(ApacheHttpClient5TransportBuilder b) { return b; }
|
||||
default HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder b) { return b; }
|
||||
}
|
||||
|
||||
That covers local, self-managed with the security plugin, and both AWS flavours natively, and lets bearer, API key and mTLS be added by hosts without a fork. Do not add BEARER, API_KEY or MTLS enum values in 1.0; they would each need a token-refresh or key-store story that the SPI defers.
|
||||
|
||||
The AWS path is a different `OpenSearchTransport` implementation, so `QuickSearchOpenSearchClient` should build an `OpenSearchTransport` from a small factory keyed on `authMode` and hold the interface type rather than `ApacheHttpClient5Transport` (`opensearch/QuickSearchOpenSearchClient.java:77`).
|
||||
|
||||
Least privilege for whatever identity is used: the QBit needs, on its one index, `indices:admin/create`, `indices:admin/exists` (or `indices:admin/get`), `indices:admin/refresh`, `indices:data/write/index`, `indices:data/write/bulk*`, `indices:data/write/delete`, `indices:data/write/delete/byquery`, and `indices:data/read/search`. It needs no cluster-level privileges. Document a role template; the index holds copies of field values from every indexed table, so the index itself should be treated as sensitive data.
|
||||
|
||||
## 2. TLS
|
||||
|
||||
- Can trust be configured: no. The transport is built with `SSLContext.getDefault()` and the httpclient5 default hostname verifier (confirmed from the opensearch-java 2.10.0 `ApacheHttpClient5TransportBuilder` bytecode). Certificates are verified against the JVM trust store by default, hostname verification is on, and there is no code path to weaken or extend either.
|
||||
- Custom CA or trust store: only through `-Djavax.net.ssl.trustStore` system properties on the host JVM, which is global and awkward in containers. Add the `tls` block from the sketch.
|
||||
- Insecure flag for dev: none exists. This is good for safety but in practice developers running OpenSearch with its default self-signed demo certs will reach for a global workaround. Provide `tls.insecureSkipVerify` with guard rails (refuse for non-loopback hosts unless an explicit second flag is set; WARN at startup) rather than leaving it to ad hoc JVM flags.
|
||||
- `useSsl` default false: acceptable only for a local, no-auth cluster. It is not acceptable combined with credentials (A2). For 1.0 prefer a single `url` (or `scheme`) field defaulting to `https`, and make `validate()` reject BASIC or AWS_SIGV4 over `http` unless `allowPlaintextCredentials` is set. AWS endpoints are HTTPS only, so AWS_SIGV4 should force `https`.
|
||||
|
||||
## 3. Secrets handling
|
||||
|
||||
- Storage: `opensearchPassword` is a plain `String` on a `Serializable` config (`QuickSearchQBitConfig.java:53`). `QuickSearchQBitProducer.produce` stores the same object in the static `QuickSearchQBitContext` (`QuickSearchQBitProducer.java:200`), and the downstream qqq-all host attaches it to `QBitMetaData` and `QInstance.addQBit` (`DemoQBits.java:162-163`). The password therefore lives in three places for the JVM lifetime.
|
||||
- Exposure through QQQ metadata endpoints: not today. `MetaDataAction` builds `MetaDataOutput` from `QFrontendTableMetaData`, `QFrontendProcessMetaData`, `QFrontendAppMetaData`, widgets, `environmentValues`, and supplemental metadata; it never touches `QInstance.getQBits()` or backend metadata, so `/metaData` does not carry QBit configs. Two caveats: `QInstanceAdapter.qInstanceToJson` (qqq `adapters/QInstanceAdapter.java:46,57`) serializes the whole `QInstance` including `qBits`, and Jackson will emit `opensearchPassword` through its public getter; and `MetaDataOutput.environmentValues` is sent to the frontend and is populated from every `QQQ_ENV_*` environment variable (qqq `QInstance.java:190-216`), so an operator must never name an OpenSearch secret `QQQ_ENV_...`. Document that.
|
||||
- Logging: no `LOG` call prints host, port, username or password (full list in A13). `validate()` and `produce()` print only field names, `backendName`, class names and table names. There is no `toString()` on the config, so an accidental `LOG.info("...", config)` would print `QuickSearchQBitConfig@hash` today; add an explicit redacting `toString()` so that stays true if someone adds one.
|
||||
- Recommendation (ordered): (1) interpret `opensearchUsername`, `opensearchPassword` and any future token or store-password fields with `QMetaDataVariableInterpreter.interpret()` at client-construction time only, so hosts write `"${env.OPENSEARCH_PASSWORD}"` and the metadata object never holds the value; this is the established QQQ pattern (`RDBMSBackendMetaData.enrich()`, `MongoDBBackendMetaData.enrich()`, `S3BackendMetaData.enrich()`); `${prop.X}` and `${literal.X}` come for free, and `??` null-coalescing lets `"${env.X}??${prop.X}"` work. (2) `@JsonIgnore` on the secret getters and a redacting `toString()`. (3) For 1.0 documentation, state that the raw password form is deprecated and will be rejected (or at least warned about) in a later release.
|
||||
|
||||
## 4. QQQ-side authorization
|
||||
|
||||
### A1 fix (search)
|
||||
|
||||
Exact leak: `QuickSearchAction.execute` reads every discovered table from `QuickSearchQBitContext` and returns `tableName`, `recordId`, `recordLabel` and `highlightSnippet` for every hit (`actions/QuickSearchAction.java:81-121`). The snippet is cut from `searchableText`, which `IndexingUtils.buildSearchableText` builds from all configured fields (`processes/IndexingUtils.java:83,136`). So a user with read permission on `order` but not `customer` who searches "smith" receives customer labels and fragments of customer field values (emails, phone numbers, whatever was configured). The caller-supplied `tableName` filter is passed straight into the query (`QuickSearchOpenSearchClient.java:481-487`) so it can also be used to target one table deliberately.
|
||||
|
||||
Minimal fix, in order:
|
||||
|
||||
1. Make `QuickSearchInput extends AbstractActionInput` (`actions/QuickSearchInput.java:25` is a plain class today). `PermissionsHelper.hasTablePermission(AbstractActionInput, String, TablePermissionSubType)` needs it, and it gives the action the session via `QContext`.
|
||||
2. Compute `allowedTables = discoveredTables.filter(t -> PermissionsHelper.hasTablePermission(input, t.getTableName(), TablePermissionSubType.READ))`. QQQ 4.1's own `RecordSearchAction` does exactly this check (qqq `actions/tables/RecordSearchAction.java:110`); mirror it, including the `Capability.TABLE_QUERY` check.
|
||||
3. If `input.tableName` is set and not in `allowedTables`, return the empty output (do not throw; do not reveal that the table exists). If `allowedTables` is empty, return empty without calling OpenSearch.
|
||||
4. Change `QuickSearchOpenSearchClient.search` to take `Collection<String> allowedTables` and always add `filter: { terms: { sourceTable: allowedTables } }` to the bool query, in addition to the optional single-table term. Only build boost `should` clauses for allowed tables.
|
||||
5. Optional but recommended (initiative D3 already decided yes): group hits by `sourceTable`, run one `QueryAction` per table with `primaryKeyField IN (ids)` and `shouldOmitHiddenFields`, keep only hits whose id came back. `QueryAction` itself does not check table permission (it only calls `ActionHelper.validateSession`, qqq `QueryAction.java:127`), so step 2 is still required; what the re-read adds is record security locks (`RecordSecurityLock` handling in `JoinsContext`), which is why `RecordSearchAction` goes through `QueryAction` too (qqq `RecordSearchAction.java:66,177`). Over-fetch (for example `limit * 2`, capped) and document that `totalHits` is an upper bound.
|
||||
6. Tests: a session without READ on table B searching a term that matches rows in A and B gets only A hits; a session with a record security lock sees only its own rows after step 5; a requested `tableName` the session cannot read yields empty.
|
||||
|
||||
### Processes, tables and app
|
||||
|
||||
Today none of the produced metadata declares permission rules (A6). For 1.0 the producer should set, by default and overridable through config:
|
||||
|
||||
- `quickSearchBasepullIndex`, `quickSearchFullReindex`, `quickSearchReconcileIndex`: `QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName("quickSearchAdmin")`. Full reindex deletes the table's documents before rebuilding (search blackout), so it must not be runnable by arbitrary users.
|
||||
- `quickSearchIndex`, `quickSearchIndexRun`: `READ_WRITE_PERMISSIONS` (operators may need to flip `enabled` or edit intervals; everyone else should not see run error messages, which can include backend exception text).
|
||||
- `quickSearchAdmin` app: `HAS_ACCESS_PERMISSION` with the same base name so a single `quickSearchAdmin.hasAccess` grant lights up the whole admin surface.
|
||||
- Scheduled runs: when D4 lands (`schedulerName` required), basepull runs under the scheduler's system session, not a user's, so the process permission only gates manual triggering. Say so in the README.
|
||||
|
||||
The customizers run inside the acting user's transaction and session; they index what that user inserted or updated and delete what that user deleted, so they need no extra checks. `QuickSearchPostUpdateCustomizer` re-reads via `QueryAction` in the same session (`customizers/QuickSearchPostUpdateCustomizer.java:154`), which is correct.
|
||||
|
||||
## 5. Transport hygiene
|
||||
|
||||
| Aspect | Current value | Source | Assessment |
|
||||
|--------|---------------|--------|------------|
|
||||
| Connect timeout | 1000 ms | opensearch-java builder default | Too tight for AWS or cross-AZ; make configurable, default 5 s |
|
||||
| Response (socket) timeout | 30 000 ms | builder default | Bounded, good; configurable, default 60 s for bulk and delete-by-query |
|
||||
| Connection-request (pool wait) timeout | httpclient5 `RequestConfig.DEFAULT` (3 min) | not set | Bounded; fine |
|
||||
| Max connections | 10 per route / 30 total | builder default | Fine for one host; expose `maxConnections` |
|
||||
| HTTP-level retries | disabled (`disableAutomaticRetries()`) | builder | Good; indexing steps handle failures per item |
|
||||
| Node failover | single `HttpHost`, dead-host denylist with backoff | `QuickSearchOpenSearchClient.java:99`; transport `DeadHostState` | OK for one endpoint; allow a list of hosts for self-managed clusters |
|
||||
| Compression | client-side decompression disabled, transport handles it | `QuickSearchOpenSearchClient.java:122-131` | Correct workaround for httpclient5 5.6 |
|
||||
| User-Agent | opensearch-java default | not set | Add `qbit-quick-search/<version>` |
|
||||
| Close on shutdown | never called in production | A8 | Fix |
|
||||
| Keep-alive / idle eviction | httpclient5 defaults | not set | Acceptable |
|
||||
|
||||
## 6. Supply chain
|
||||
|
||||
See A14. Resolved compile/runtime set has no concrete published CVE at the audit date. The one cited fix the stack already has is CVE-2025-27820 in httpclient5 5.4.0 to 5.4.2 (resolved 5.6.3 is clear). Keep httpclient5 and jackson BOM-managed (ADR-0007). `h2 2.2.220`, Testcontainers 1.21.4, JUnit, AssertJ and Mockito are test scope only.
|
||||
|
||||
## Recommended 1.0 auth configuration
|
||||
|
||||
Field names on `QuickSearchQBitConfig` (fluent `withX` setters as today). All `String` values that can hold secrets or paths are run through `QMetaDataVariableInterpreter.interpret()` inside the client factory, never at `validate()` or `produce()` time, so the config object stores references.
|
||||
|
||||
// Connection
|
||||
opensearchUrl String "https://search.example.com:443" (preferred; replaces host/port/useSsl)
|
||||
opensearchHost/Port String/Integer (kept for compatibility; deprecated)
|
||||
useSsl Boolean (kept; ignored when opensearchUrl is set)
|
||||
opensearchHosts List<String> optional, same scheme, for self-managed multi-node clusters
|
||||
opensearchIndexName String (unchanged)
|
||||
|
||||
// Authentication
|
||||
authMode enum OpenSearchAuthMode { NONE, BASIC, AWS_SIGV4 } default NONE
|
||||
opensearchUsername String BASIC only; "${env.OPENSEARCH_USER}"
|
||||
opensearchPassword String BASIC only; "${env.OPENSEARCH_PASSWORD}"; @JsonIgnore getter; redacted in toString()
|
||||
awsRegion String AWS_SIGV4; "${env.AWS_REGION}" if omitted
|
||||
awsServiceName String AWS_SIGV4; "es" (default) or "aoss"
|
||||
awsAssumeRoleArn String AWS_SIGV4, optional; uses StsAssumeRoleCredentialsProvider when set
|
||||
allowPlaintextCredentials Boolean default false; validate() errors on BASIC over http unless true
|
||||
|
||||
// TLS
|
||||
tls.caCertificatePath String PEM bundle to trust in addition to or instead of the JVM store
|
||||
tls.trustStorePath String JKS/PKCS12 alternative to caCertificatePath
|
||||
tls.trustStorePassword String reference-interpreted
|
||||
tls.keyStorePath String client certificate for mTLS (optional)
|
||||
tls.keyStorePassword String reference-interpreted
|
||||
tls.hostnameVerification Boolean default true
|
||||
tls.insecureSkipVerify Boolean default false; validate() rejects for non-loopback hosts unless allowInsecureInProduction is true; WARN at startup
|
||||
|
||||
// Transport
|
||||
connectTimeoutMillis Integer default 5000
|
||||
responseTimeoutMillis Integer default 60000
|
||||
maxConnections Integer default 30
|
||||
transportCustomizer QCodeReference<OpenSearchTransportCustomizer> escape hatch for bearer, API key, custom interceptors
|
||||
|
||||
// Authorization inside QQQ
|
||||
searchRequiresTableReadPermission Boolean default true (no switch to turn it off in 1.0; listed so the behaviour is documented)
|
||||
applyRecordSecurityLocks Boolean default true; when true, hits are re-read through QueryAction by primary key
|
||||
adminPermissionRules QPermissionRules default HAS_ACCESS_PERMISSION, base name "quickSearchAdmin", applied to the app and the three processes
|
||||
tablePermissionRules QPermissionRules default READ_WRITE_PERMISSIONS, applied to quickSearchIndex and quickSearchIndexRun
|
||||
|
||||
`validate()` additions: `authMode` consistency (BASIC needs both username and password; AWS_SIGV4 needs a resolvable region and forbids username/password; NONE with username/password set is an error, not a silent fallback), TLS consistency (trust store needs password or must be PKCS12 without one; `insecureSkipVerify` rules above), and the plaintext-credentials rule. Validation messages must never echo values, only field names, which is already the convention in the file.
|
||||
|
||||
Deprecation path: keep `withOpensearchHost/Port/UseSsl/Username/Password` working for 0.x hosts; infer `authMode = BASIC` when username is set and `authMode` is null, and log a one-line WARN recommending `${env.}` references when a password value does not start with `${`.
|
||||
@@ -0,0 +1,213 @@
|
||||
# Audit 04: Feature completeness for Quick Search 1.0
|
||||
|
||||
Scope: `qbit-quick-search` at `feature/GH-6-repin-and-drift-fix` (content-identical to `develop` 000e5bb), 29 main classes, 29 test classes (233 unit tests, 12 integration tests). Compared against the draft PRD `docs/initiatives/quick-search-1.0.md`, qqq-all `DemoQBits.java`, Voyage, and QQQ `v4.1.0-RC.1` `RecordSearchAction`.
|
||||
|
||||
Paths below are relative to `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1/src/main/java/com/kingsrook/qbits/quicksearch/` unless stated. Core paths are relative to `/Users/james.maes/Git.Local/QRunIO/qqq` at tag `v4.1.0-RC.1`.
|
||||
|
||||
Verdict in one line: the QBit has a working happy path (index, bulk, search, reconcile) but is missing most of what makes a search feature trustworthy in a multi-table, multi-user, long-running host: no scheduling, no permissions, no timeouts, no mapping/config drift handling, no admin UI affordances, a single global static context, and a search query shape that was never tuned. The PRD covers roughly half of the gaps found here.
|
||||
|
||||
## 1. Area-by-area table
|
||||
|
||||
| # | Area | Status | Evidence | What 1.0 needs |
|
||||
|---|---|---|---|---|
|
||||
| 1a | Real-time insert/update/delete customizers | Partial | `customizers/QuickSearchPostInsertCustomizer.java:66-105`, `...PostUpdateCustomizer.java:81-155` (re-reads full record in the update's transaction), `...PostDeleteCustomizer.java:67-118`. All three swallow every exception at WARN (`:99-102`, `:117-120`, `:112-115`) and return records unchanged. Registration throws when the slot is occupied (`QuickSearchQBitProducer.java:571-579`; core `QTableMetaData.java:651-653` throws `IllegalArgumentException`). | `MultiCustomizer` composition (PRD E1). Decide and document the failure contract (swallow vs. fail write), plus a failed-event record so swallowed failures are not lost (see 4). Honour `quickSearchIndex.enabled` (see 3). |
|
||||
| 1b | Bulk inserts with many records | Partial | Insert customizer builds one `IndexEvent` per record and the publisher bulk-indexes in `bulkBatchSize` chunks (`publisher/SynchronousIndexEventPublisher.java:93-127`, `opensearch/QuickSearchOpenSearchClient.java:252-272`). Whole batch runs inside the host's insert call; no size-based (bytes) batching, no async. A 50k-row bulk load blocks the insert for the full OpenSearch round trips. | Document that bulk loads should disable real-time indexing or rely on basepull; add byte-size cap per bulk request (OpenSearch `http.max_content_length` default 100 MB). |
|
||||
| 1c | Null/blank searchable fields | Complete | `processes/IndexingUtils.java:96-99` skips null/blank in text; `:150-156` omits null from `fieldValues`; `:138-141` returns null doc when PK null. Unit tested (`IndexingUtilsTest`). | Nothing. |
|
||||
| 1d | Very large text fields | Missing | No truncation or length cap anywhere (`IndexingUtils.buildSearchableText`); `recordLabel`/`searchableText` mapped as plain `text` (`QuickSearchOpenSearchClient.java:189-190`). A CLOB field is sent whole; edge-ngram 2..20 on a 1 MB field explodes token count. Highlight returns default fragments only. | Per-field `maxLength` in `SearchableFieldConfig`/`@QuickSearchField` with a sane default (e.g. 10k chars), documented. |
|
||||
| 1e | Tables without a `modifyDate`-style field | Missing | Default `basepullTimestampField = "modifyDate"` (`annotations/QuickSearchable.java:54`, `SearchableTableConfig.java:34`). `QuickSearchQBitConfig.validate` (`:73-157`) never checks that the timestamp field or the searchable fields exist on the table. Basepull then fails per table at query time, caught at `BasepullIndexStep.java:128-132` and logged. qqq-all's `customer` table config does not declare one (`qqq-all DemoQBits.java:159-160`). | Validate field existence at produce (fail fast); allow `basepullTimestampField = null` meaning "no incremental basepull, reconcile only". |
|
||||
| 1f | Non-integer primary keys | Partial | `recordId` is a String everywhere (`OpenSearchDocument.java:35`, `QuickSearchResult.java:28`); keyset paging uses `GREATER_THAN` on the PK (`ReconcileIndexStep.java:222-238`), which works for strings/UUIDs only if the backend sorts them consistently. Untested. | Integration test with a STRING PK table; document ordering assumption. |
|
||||
| 1g | Composite keys | Missing | Single `primaryKeyField` model (`QuickSearchableTableConfig.java:25`). | Out of scope for 1.0; state it in README. |
|
||||
| 1h | Table removed from config (orphaned documents) | Missing | Reconcile iterates `getDiscoveredTables()` only (`ReconcileIndexStep.java:98`); documents for a table no longer configured are never deleted and are still returned by an unfiltered search (`QuickSearchOpenSearchClient.java:481-487` filter only when `tableName` given). `quickSearchIndex` rows for removed tables stay `ACTIVE`. | Reconcile (or a startup check) should delete-by-query every `sourceTable` not in the discovered set and mark the index row `ORPHANED`/disabled. |
|
||||
| 1i | Table renamed / prefix changed | Missing | `sourceTable` is the QQQ table name; `tableNamePrefix` applies only to the two operational tables and processes (`QuickSearchQBitConfig.applyPrefix`). Renaming a source table leaves old documents behind (same as 1h). Changing `opensearchIndexName` silently starts an empty index (`ensureIndexExists` creates it, `:173-192`) with no reindex trigger. | Same mechanism as 1h plus an "index is empty but source has rows" warning in the admin table. |
|
||||
| 1j | Config changes to fields/weights/analyzer | Missing | `searchableFieldsJson` is written once when the index row is created (`AbstractIndexingStep.java:167-175`) and never compared again. `ensureIndexExists` returns early if the index exists (`QuickSearchOpenSearchClient.java:167-171`), so a pre-existing index with a different analyzer or mapping is used as-is. Weights are query-time boosts on `fieldValues.*` (`:526-565`), so weight changes need no reindex, but field additions do. | Compare current config to `searchableFieldsJson` at produce/basepull and set `status = NEEDS_REINDEX`; verify the live mapping's analyzer settings and refuse to start (or warn loudly) on mismatch. |
|
||||
| 1k | Index name collisions on a shared cluster | Missing | One flat index per config, name is whatever the host passes (`QuickSearchQBitConfig.java:51`). Two apps with the same default would share documents; there is no `appId`/environment field in the document or filter. `fieldValues` is a single dynamic object shared by every table (`:192`): two tables with the same field name but different types (e.g. `code` INTEGER in one, STRING in another) produce `mapper_parsing_exception` bulk-item failures for the second table. | Document required index naming per environment; namespace `fieldValues` per table (`fieldValues.<table>.<field>`) or map all values as `keyword`/`text` strings to avoid type conflicts. |
|
||||
| 1l | Basepull catch-up | Partial | Works per enabled row (`BasepullIndexStep.java:74-134`). Defects: never scheduled (no `withSchedule` anywhere; `QuickSearchQBitProducer.java:510-518`); `lastBasepullTime` is set to `Instant.now()` after the run (`:268-279`), not the run's start, so rows modified during the run but after their page was read are skipped forever; `GREATER_THAN` on a seconds-precision DB column vs. millisecond `Instant` misses same-second rows; limit/skip paging without order-by (`AbstractIndexingStep.java:358-359`) is nondeterministic under concurrent writes. | PRD E2 covers scheduling and ordered paging. Add: capture `runStart` before querying and use it (minus a small overlap) as the new watermark; use `GREATER_THAN_OR_EQUALS` with the overlap. |
|
||||
| 1m | Full reindex | Partial | Wipes the table's documents first (`FullReindexStep.java:152`) causing a search blackout; limit/skip paging without order-by (`:162`); does not honour `enabled`; `recordCount` written only on success (`:191-204`). | PRD E2 (alias swap) covers the blackout. |
|
||||
| 1n | Reconcile | Complete for its design | Keyset paging (`ReconcileIndexStep.java:222-238`), refresh then delete-stale by `indexedAt` (`:191-193`), aborts stale-delete on any failure (`:195-198`). Relies on cross-node clock sync (class Javadoc `:62-63`). Not scheduled. Does not handle orphaned tables (1h). | Schedule it (PRD E2); add orphan-table sweep. |
|
||||
| 2a | Term normalisation and short/empty terms | Partial | `IndexingUtils.normalizeSearchText` lowercases and collapses whitespace (`:58-66`); blank returns empty output (`QuickSearchAction.java:67-74`). No minimum length: a 1-char term yields zero edge-ngram tokens (min_gram 2) so it silently returns nothing; no maximum length (core caps at 100). | Enforce min 2 / max 100 characters with a clear message, matching core. |
|
||||
| 2b | Multi-word queries | Missing (untuned) | `multi_match` on `searchableText` with no `operator`, no `minimum_should_match`, no `type` (`QuickSearchOpenSearchClient.java:472-475`). Default is OR, so "blue widget" matches every document containing "bl", "blu", "wi", ... and ranks by ngram overlap. | Set `operator = AND` (or `minimum_should_match = "2<75%"`) and type `best_fields` or `cross_fields`; cover with an integration test using two-word terms. |
|
||||
| 2c | Phrase matching | Missing | No `match_phrase` clause; quoted input is passed through as-is. | Optional for 1.0; document absence. |
|
||||
| 2d | Prefix/typeahead behaviour | Partial (defective) | `searchableText` uses the edge-ngram analyzer for both indexing and search; no `search_analyzer` is set (`:190`). The query term is therefore also edge-ngrammed: "order" becomes or/ord/orde/order, so "or" matches as strongly as the full word and relevance degrades with term length. The boost should-clauses hit `fieldValues.*`, which are dynamically mapped with the standard analyzer, so field weights only apply to whole-token matches, never to prefix matches. | Set `search_analyzer: standard` (lowercase only) on `searchableText`; either map `fieldValues.*` with the same edge-ngram analyzer via a dynamic template or drop field-level boosting into per-field sub-fields of `searchableText`. |
|
||||
| 2e | Fuzziness | Missing | None configured. | 1.x (PRD explicitly out of scope). |
|
||||
| 2f | Per-table result limits and grouping | Missing | Single flat ranked list; `tableName` filter accepts one table (`QuickSearchInput.java:29`). One hot table swamps the others. Core has `limitPerTable` (default 5, max 25). | Add `tableNames` (list) and `limitPerTable` (use an OpenSearch `terms` aggregation with `top_hits`, or `collapse`); needed to map onto core `RecordSearchInput`. |
|
||||
| 2g | Result fields | Partial | `QuickSearchResult` has `tableName, recordId(String), recordLabel, score, highlightSnippet` (`actions/QuickSearchResult.java:27-31`). No `tableLabel`, no typed `recordId`, no route/URL hint, no display fields, no `relation` on total. Core `RecordSearchResult` has `tableLabel` and `Serializable recordId`. | Add `tableLabel` (from `QInstance.getTable(name).getLabel()`), typed `recordId`, and optionally the matched field name. |
|
||||
| 2h | Max page depth | Missing | `limit` has no upper bound (`QuickSearchAction.java:78`); `from + size > 10000` hits `index.max_result_window` and surfaces as a raw QException. | Cap `limit` (e.g. 100) and `offset + limit` (e.g. 10000); return a clean error. |
|
||||
| 2i | Highlight quality | Partial | Highlight only on `searchableText` with defaults (`QuickSearchOpenSearchClient.java:491-492`); fragments joined with "..." (`QuickSearchAction.java:104-109`). Snippets include the `fieldName: ` prefix when `includeLabel` is set (`IndexingUtils.java:106-108`), exposing internal field names; no `pre/post_tags` control; nothing stops a snippet from a field the user may not see (PRD E3 mentions this). | Configurable tags and fragment size; strip label prefixes or use the QQQ field label. |
|
||||
| 2j | Sorting options | Missing | Score only. | Fine for 1.0; document. |
|
||||
| 2k | Total hit accuracy | Missing | `track_total_hits` not set, so `totalHits` caps at 10,000 and `hasMore` is computed from the capped value (`QuickSearchAction.java:123-124`); `relation` is ignored. | Set `track_total_hits` to a bounded number and expose `totalHitsIsLowerBound`. |
|
||||
| 2l | Permissions / session | Missing | No `PermissionsHelper`, no `QContext` use anywhere in `actions/` or `opensearch/`. Core `RecordSearchAction.java` checks `TablePermissionSubType.READ` and `Capability.TABLE_QUERY` per table and personalises fields. | PRD E3. |
|
||||
| 3a | `quickSearchIndex` fields | Partial | `id, tableName, enabled, basepullIntervalMinutes, basepullTimestampField, searchableFieldsJson, lastBasepullTime, lastFullReindexTime, recordCount, status, createDate, modifyDate` (`QuickSearchQBitProducer.java:456-471`). `status` is always "ACTIVE" (`AbstractIndexingStep.java:175`) and never updated; no `lastError`, `lastRunStatus`, `documentCount` (from OpenSearch), `lastReconcileTime`, `indexName`. No PVS, no sections, no `isRequired`, no `isEditable=false` on system-managed fields, no labels beyond defaults. | Add `lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`; make tracking fields read-only; add field sections and a tableName PVS. |
|
||||
| 3b | `quickSearchIndexRun` fields | Partial | `quickSearchIndexId, runType, status, startTime, endTime, recordsProcessed, recordsIndexed, errorCount, errorMessage` (`:482-497`). `errorMessage` is a plain STRING holding only the first error (`BasepullIndexStep.java:254-257`); no per-item error table; `quickSearchIndexId` has no PVS/join so the admin UI shows an integer. | Join/PVS to `quickSearchIndex`; store error list (JSON or child table); set `errorMessage` max length. |
|
||||
| 3c | Run records for every path | Partial | Written for basepull, full reindex, reconcile (`createRunRecord`/`completeRunRecord`). Not written for real-time customizer failures (they only WARN) nor for "skipped, not due" basepull iterations. Nothing is scheduled, so there are no scheduled runs today. | A failed-event table or a `REALTIME_FAILURE` run row per failed publish (see 4c). |
|
||||
| 3d | Retention/cleanup of run history | Missing | No purge; unbounded growth once scheduled (every 5 minutes per table). | Scheduled purge (e.g. keep 30 days or N rows per index) in the same scheduled process. |
|
||||
| 3e | `enabled` honoured at runtime | Partial | Only basepull filters `enabled = true` (`BasepullIndexStep.java:89-92`). Customizers, full reindex, reconcile and search ignore it, so disabling a table in the admin UI still indexes it on every write and still returns its rows. | Customizers and processes consult the index row (cached, refreshed per run); search excludes disabled tables via a `terms` filter. |
|
||||
| 3f | Status reporting / counts | Partial | `recordCount` updated only on full reindex/reconcile success; no live document count; no health field. | See 3a. |
|
||||
| 3g | Admin widgets or dashboards | Missing | App is two tables plus three processes (`QuickSearchQBitProducer.java:247-258`); no widgets, no icon. | A small dashboard widget (per-table status, last run, failures) is a 1.x nicety; an icon and labels are 1.0 polish. |
|
||||
| 3h | Process input screens | Missing | All three processes are a single backend step with no frontend step and no input `QFieldMetaData` (`:510-549`), yet full reindex and reconcile read `tableName` from input (`FullReindexStep.java:73`, `ReconcileIndexStep.java:93`). From the admin UI there is no way to supply it, so UI-triggered runs always hit every table. README's "or through the QQQ admin UI" is therefore only half true. | Add a frontend step with an optional `tableName` field backed by a PVS of discovered tables, plus a summary screen showing counts (reconcile already emits `recordsIndexed`/`documentsRemoved`). |
|
||||
| 3i | Labels/help text | Partial | Table and process labels set; no field labels/help, no app icon/section. | Polish pass. |
|
||||
| 4a | Logging | Partial | QLogger with structured pairs; failures at WARN. Basepull logs "not due" at INFO per table per run (`BasepullIndexStep.java:118-120`), which is noisy every 5 minutes. | Downgrade to DEBUG; add one INFO summary per run. |
|
||||
| 4b | Metrics / health check | Missing | No counters, no timers, no `ping`/cluster-health call; the client constructor never connects (`QuickSearchOpenSearchClient.java:94-147`), so "client created" says nothing about reachability. | Expose a health method (indices.exists or cluster health) and surface it in the admin table or a QQQ health endpoint; count publish failures. |
|
||||
| 4c | OpenSearch down at startup vs. runtime | Partial (hazardous) | Startup: client build rarely fails; `ensureIndexExists` failure is WARN-and-continue (`QuickSearchQBitProducer.java:174-181`), publisher is still created (`:192-195`), host boots "healthy". Runtime: every host insert/update/delete calls OpenSearch synchronously; failures are swallowed (customizers) so the host write succeeds and the index silently diverges; `QuickSearchAction` throws. No timeouts are configured on the transport (no `setConnectTimeout`/`setResponseTimeout`/`RequestConfig` anywhere), so a black-holed host blocks each write for the OS connect timeout (tens of seconds to minutes) and a slow cluster blocks indefinitely. | Fail-fast at produce with opt-out (PRD E1). Add explicit connect/response timeouts in config (e.g. 2s/10s). Record failed events (see 4e). |
|
||||
| 4d | Retry/backoff | Missing | None; a transient 429/503 on bulk is a hard failure. | Bounded retry with jitter on bulk 429/5xx inside the client (sync path), documented. |
|
||||
| 4e | Dead-letter / failed-event capture | Missing | Failed real-time events vanish after a WARN. Basepull only recovers rows whose `modifyDate` moves; a failed delete is recovered only by reconcile. | A `quickSearchFailedEvent` table (table, recordId, action, error, attempts) replayed by the scheduled process; this is the cheapest correct answer without an async queue. |
|
||||
| 4f | Circuit breaking | Missing | None. | Simple breaker in `SynchronousIndexEventPublisher` (open after N consecutive failures, half-open after T seconds; skip and record events while open). With timeouts (4c) this is what keeps a down cluster from slowing every host write. |
|
||||
| 5a | Index name per environment | Partial | Host supplies the name; qqq-all reads `OPENSEARCH_INDEX` (`DemoQBits.java:155`). No validation of OpenSearch naming rules (lowercase, no spaces). | Validate name; document per-environment convention. |
|
||||
| 5b | Analyzer customisation | Missing | Hard-coded `standard` tokenizer + `lowercase` + `edge_ngram(2,20)` (`QuickSearchOpenSearchClient.java:176-185`). | Expose `minGram/maxGram` and an optional analyzer-settings hook (see 6c). |
|
||||
| 5c | Refresh policy | Missing | Bulk requests use default (near-real-time, ~1s); only tests call `refreshIndex`. Hosts that read-after-write will not see the row. | `refreshPolicy` config (none/wait_for) for the real-time path; document the default. |
|
||||
| 5d | `sourceBatchSize` vs `bulkBatchSize` | Complete but undocumented | Source page size for QQQ queries (`:1000`) vs documents per bulk request (`:500`) (`QuickSearchQBitConfig.java:58-59`). README table describes both correctly. | Note interaction (one source page = ceil(1000/500) bulk calls). |
|
||||
| 5e | Timeouts | Missing | See 4c. | Add. |
|
||||
| 5f | Multiple QBit instances / multiple indexes | Missing | Static `QuickSearchQBitContext` (`:34-37`); a second `produce()` overwrites the first; customizers and steps resolve state statically. | PRD E1 (instance-scoped holder). |
|
||||
| 5g | Excluded tables | Missing | No way to exclude, only to include; `enabledByDefault=false` still registers customizers and creates the row. | Fine if `enabled` is honoured (3e). |
|
||||
| 5h | Per-table record label strategy | Partial | `recordLabelFormat` only on config-driven tables (`SearchableTableConfig.java:36-37`); annotation path uses `QRecord.getRecordLabel()` which is null unless the query generated it (`IndexingUtils.java:142`). Customizers receive records from Insert/Update actions where `recordLabel` is typically not populated, so real-time-indexed documents often have a null `recordLabel`. Core falls back to `QValueFormatter.formatRecordLabel`. | Fall back to `QValueFormatter.formatRecordLabel(table, record)` in `buildDocument`; allow label format on the annotation too. |
|
||||
| 5i | Field type handling (dates, decimals, booleans, PVS labels) | Missing | Raw values only: `getValueString` for text (`IndexingUtils.java:94`) and raw `Object` into `fieldValues` (`:150-154`). Dates index as ISO instants, booleans as "true", PVS fields as their raw id. No display values, no PVS translation. Core sets `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` on its queries. | Index display values (translate PVS, format dates) into `searchableText` and keep raw values in `fieldValues`; batch processes already go through `QueryAction` so this is a flag; customizers need a re-read with display values (update already re-reads). |
|
||||
| 5j | Flag semantics | Partial | `enableScheduledProcesses=false` removes all three processes including the manual full reindex and reconcile (`QuickSearchQBitProducer.java:228-232`), so a host cannot have on-demand processes without the basepull. | Split into `enableBasepullProcess` and `enableMaintenanceProcesses`, or always register processes and gate only the schedule. |
|
||||
| 6a | `IndexEventPublisher` SPI | Partial | `publishIndexEvents`, `publishDeleteEvents`, `close()` (`publisher/IndexEventPublisher.java:40-60`). `IndexEvent` carries `tableName, recordId, action, QRecord` (`IndexEvent.java:30-33`): the full record snapshot, not a built document, so an async consumer can rebuild the document later via `IndexingUtils.buildDocument(record, tableConfig)` only if it has the table config (currently only through the static context). There is no event timestamp, sequence or record version, so a queue consumer cannot order an INDEX after a later DELETE. `close()` is never invoked by the producer or any shutdown hook. The publisher receives no `QSession`/tenant information. | Add `occurredAt` (Instant) and an optional `version` to `IndexEvent`; pass the table config map (or a resolver) into the publisher constructor contract; register `close()` with the host lifecycle; document ordering expectations. |
|
||||
| 6b | Document builder hook | Missing | `IndexingUtils.buildDocument` is static and called directly from publisher and steps. | `DocumentBuilder` interface with a default implementation, settable on `QuickSearchQBitConfig` (lets hosts add computed text, strip PII, index display values). |
|
||||
| 6c | Custom analyzer hook | Missing | Hard-coded in `ensureIndexExists`. | `IndexSettingsCustomizer` (consumer of `CreateIndexRequest.Builder`) or at least `minGram/maxGram/searchAnalyzer` config. |
|
||||
| 6d | Result post-processing hook | Missing | `QuickSearchAction.execute` maps hits inline (`:95-121`). | `ResultCustomizer` to add route hints or filter; also where permission filtering would plug in. |
|
||||
| 7a | Unit coverage | Partial | 233 unit tests across all packages; happy paths and null/empty guards well covered (see test list in section 4). JaCoCo `check` is configured (`pom.xml:188-212`) but with no `prepare-agent` execution, so the 0.70/0.90 gates never run (PRD notes this). | Bind `prepare-agent`; keep gates. |
|
||||
| 7b | Integration coverage | Partial | 12 tests in `OpenSearchIntegrationTest`, `IndexDriftIntegrationTest`, `ConfigDrivenIndexingIntegrationTest`. Every search uses the single whole-word term "widget" or "zzzznonexistent"; no prefix, multi-word, boost-ordering, highlight-content, pagination-beyond-first-page-correctness, basepull, or scheduling test. | See section 4 list. |
|
||||
| 7c | README promises not met | Partial | "Scheduled basepull" (README:14, 30, 287) is not scheduled. "Prefix indexed text with the field label" (README:171, 331) prefixes the field *name* (`IndexingUtils.java:108`). "Or through the QQQ admin UI" for per-table reindex (README:254, 271) has no input screen. Dependency snippet shows `0.2.0-SNAPSHOT` (README:79). qbit-crm/qbit-wms companion section (README:333-341) describes integrations that do not exist. "70%+ instruction coverage (JaCoCo enforced)" (README:362) is not enforced. "Admin Observability" (README:20) is two raw tables. | PRD E5 rewrite; add the items above to its checklist. |
|
||||
| 7d | CHANGELOG / migration notes | Missing | No `CHANGELOG.md` at repo root; no migration doc. | PRD E5. |
|
||||
| 7e | Configuration reference accuracy | Partial | README table (291-312) matches `QuickSearchQBitConfig` fields but omits the `enableScheduledProcesses` side effect (5j), the `indexEventPublisher` lifecycle, and says nothing about timeouts, refresh, or required cluster privileges. | Fix with E5. |
|
||||
| 7f | Javadoc on public API | Mostly complete | Flower-box Javadoc on nearly every public member. Gaps: `IndexingUtils.buildDocument` documents `fieldWeights` as "reserved for future use" (`:130`) although weights drive query boosts; `QuickSearchQBitProducer` public constants (`:54-57`) undocumented; no `package-info.java`; `IndexEventPublisher` does not state thread-safety or ordering guarantees. | Small pass with E5. |
|
||||
|
||||
## 2. Comparison with QQQ 4.1 core `RecordSearchAction`
|
||||
|
||||
Source: `qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/RecordSearchAction.java`, `model/actions/tables/search/RecordSearch{Input,Output,Result}.java`, `QTableMetaData.searchFields` (`QTableMetaData.java:100, 2013-2060`), Javalin `RecordSearchExecutor` (passes `QInputSource.USER`).
|
||||
|
||||
Core has, Quick Search lacks:
|
||||
|
||||
| Capability | Core | Quick Search |
|
||||
|---|---|---|
|
||||
| Table-level READ permission check | `PermissionsHelper.hasTablePermission(..., READ)` per table; unauthorised tables skipped silently (no probing) | None |
|
||||
| Capability check (`TABLE_QUERY`) | Yes | N/A (reads from index) but no equivalent gate |
|
||||
| Field personalisation; hidden/masked fields excluded | `TableMetaDataPersonalizerAction`, `getIsHidden`, `needsMasked` | None; whatever is configured is indexed and highlighted |
|
||||
| Record-level security locks | Inherited from `QueryAction` | None |
|
||||
| Per-table limit | `limitPerTable` default 5, max 25 | Global `limit`, no max |
|
||||
| Multi-table filter | `tableNames` list | Single `tableName` |
|
||||
| Term validation | Blank -> `QBadRequestException`; max 100 chars | Blank -> empty output; no max |
|
||||
| Query timeout | 30s per table | None |
|
||||
| Read-replica hint | `QueryHint.MAY_USE_READ_ONLY_BACKEND` | N/A |
|
||||
| Result shape | `tableName, tableLabel, Serializable recordId, recordLabel` | `tableName, String recordId, recordLabel, score, highlightSnippet`; no `tableLabel` |
|
||||
| Record label fallback | `QValueFormatter.formatRecordLabel` | Null when the record had no label |
|
||||
| Display values / PVS translation | `shouldGenerateDisplayValues`, `shouldTranslatePossibleValues` | Raw values |
|
||||
| Deterministic ordering | Order by PK | Score (fine) |
|
||||
| Integer term matching | Integral fields match numeric terms exactly | Numbers indexed as text tokens; "42" prefix-matches "4200" |
|
||||
| Metadata declaration | `QTableMetaData.withSearchFields(...)` is first-class metadata the Next UI can read | Separate annotation/config model; `searchFields` on the table is ignored |
|
||||
| UI surface | `POST /qqq/v1/search` and the Next UI search box | None |
|
||||
|
||||
Quick Search has, core lacks:
|
||||
|
||||
| Capability | Quick Search | Core |
|
||||
|---|---|---|
|
||||
| Relevance scoring and field weights | BM25 with per-field boosts | None (PK order) |
|
||||
| Prefix/typeahead | Edge-ngram (with the defect in 2d) | `CONTAINS` (LIKE `%term%`), which is a full scan on large tables |
|
||||
| Highlights | Yes | No |
|
||||
| Pagination and total | `offset/limit`, `totalHits`, `hasMore` | Top-N per table only |
|
||||
| Cross-table ranking | Single ranked list | Per-table lists concatenated |
|
||||
| Performance isolation | Query hits OpenSearch, not the OLTP backend | One LIKE query per searchable table per keystroke |
|
||||
| Operational history | Run tables | None |
|
||||
| Async-capable transport | `IndexEventPublisher` SPI | N/A |
|
||||
|
||||
Integration consequence (PRD E4/D1): to back the Next UI, a `QuickSearchRecordSearchProvider` must accept `RecordSearchInput` (`tableNames`, `limitPerTable`, `inputSource`) and return `RecordSearchResult` (`tableLabel`, typed `recordId`). Quick Search today cannot satisfy `limitPerTable` or `tableNames` without the changes in 2f, and cannot honour the permission semantics core users will assume without E3. Also, a host that sets `QTableMetaData.searchFields` for core and `@QuickSearchable` for the QBit now maintains two field lists; 1.0 should at least default the QBit's field list from `searchFields` when present.
|
||||
|
||||
## 3. Prioritised gap list
|
||||
|
||||
Severity: P0 = blocks a credible 1.0; P1 = should ship in 1.0; P2 = 1.0 if cheap, else 1.x; P3 = 1.x.
|
||||
|
||||
| ID | Sev | Gap | Evidence | Recommended scope |
|
||||
|---|---|---|---|---|
|
||||
| G01 | P0 | No connect/response timeouts on the OpenSearch transport; a down or black-holed cluster blocks every host write for the OS timeout, indefinitely for a hung response | `QuickSearchOpenSearchClient.java:94-147`; grep for timeout in `src/main` returns nothing | 1.0. Add `connectTimeoutMs`/`responseTimeoutMs` config with conservative defaults. Not in PRD. |
|
||||
| G02 | P0 | Search ignores permissions, hidden/masked fields, record locks | `actions/QuickSearchAction.java`, no `PermissionsHelper` in repo; core does all three | 1.0 (PRD E3). Extend E3 to exclude hidden/masked fields at index time, not just at search time. |
|
||||
| G03 | P0 | Basepull and reconcile never scheduled | No `withSchedule`; `QuickSearchQBitProducer.java:510-518` | 1.0 (PRD E2, D4). |
|
||||
| G04 | P0 | Edge-ngram analyzer applied at query time too (no `search_analyzer`); multi_match defaults to OR with no `minimum_should_match`; field boosts never apply to prefix matches | `QuickSearchOpenSearchClient.java:190, 472-475, 526-565` | 1.0. Set `search_analyzer`, `operator=AND`, dynamic template for `fieldValues.*`. Requires a reindex (mapping change) so it must land before 1.0's first index. Not in PRD. |
|
||||
| G05 | P0 | Startup is best-effort; host boots healthy with search broken | `QuickSearchQBitProducer.java:160-181` | 1.0 (PRD E1). |
|
||||
| G06 | P1 | Real-time failures are swallowed with no record; no retry, no breaker, no dead-letter | `customizers/*:99-120`; `SynchronousIndexEventPublisher` | 1.0: failed-event table replayed by the scheduled process, plus bounded retry on bulk 429/5xx. Breaker can be 1.x if timeouts (G01) ship. Not in PRD. |
|
||||
| G07 | P1 | `quickSearchIndex.enabled` ignored by customizers, full reindex, reconcile and search | Only `BasepullIndexStep.java:89-92` filters on it | 1.0. Not in PRD. |
|
||||
| G08 | P1 | Basepull watermark set after the run; same-second rows and rows modified mid-run are skipped forever | `BasepullIndexStep.java:268-279`, `:206-209` | 1.0 alongside E2 ordered paging; capture start time, apply overlap. Not in PRD (E2 mentions only ordering). |
|
||||
| G09 | P1 | `fieldValues` shared dynamic object across tables causes mapping type conflicts when two tables share a field name with different types | `QuickSearchOpenSearchClient.java:192`; `IndexingUtils.java:150-154` | 1.0 (mapping change, same reindex as G04): namespace per table or map as keyword. Not in PRD. |
|
||||
| G10 | P1 | Static JVM-global context; one QBit instance per JVM | `QuickSearchQBitContext.java:34-37` | 1.0 (PRD E1). |
|
||||
| G11 | P1 | Customizer slot collision fails produce | `QuickSearchQBitProducer.java:571-579`; core `QTableMetaData.java:651-653` | 1.0 (PRD E1). |
|
||||
| G12 | P1 | Full reindex blackout; unordered paging | `FullReindexStep.java:152, 162` | 1.0 (PRD E2, D5). |
|
||||
| G13 | P1 | No validation that searchable fields and the basepull timestamp field exist on the table; silent PK fallback to "id" for tables missing from the QInstance | `QuickSearchQBitConfig.java:73-157`; `QuickSearchQBitProducer.java:349, 418` | 1.0 (fold into E1 fail-fast validation). Partly in PRD ("fail-fast config validation") but field existence is not named. |
|
||||
| G14 | P1 | Orphaned documents when a table is removed from config or renamed; reconcile only sweeps configured tables | `ReconcileIndexStep.java:98`; `QuickSearchOpenSearchClient.java:481-487` | 1.0 (small addition to reconcile). Not in PRD. |
|
||||
| G15 | P1 | No config-drift detection: `searchableFieldsJson` never compared; existing index mapping/analyzer never verified | `AbstractIndexingStep.java:167-175`; `QuickSearchOpenSearchClient.java:167-171` | 1.0: compare and set `NEEDS_REINDEX`; verify analyzer on existing index and warn. Not in PRD. |
|
||||
| G16 | P1 | Result model lacks `tableLabel`, typed `recordId`, `tableNames`/`limitPerTable` input; cannot map onto core `RecordSearchResult` cleanly | `actions/QuickSearchInput.java:28-31`, `QuickSearchResult.java:27-31` | 1.0 (needed for PRD E4 even if the SPI slips). PRD E4 names the mapping but not the input-side gaps. |
|
||||
| G17 | P1 | Processes have no input screen; `tableName` cannot be supplied from the admin UI | `QuickSearchQBitProducer.java:510-549`; `FullReindexStep.java:73` | 1.0: frontend step with optional tableName PVS. Not in PRD. |
|
||||
| G18 | P1 | `totalHits` capped at 10,000 silently; `limit`/`offset` unbounded | `QuickSearchAction.java:78-79, 123-124` | 1.0: bound limit/offset, set `track_total_hits`, expose lower-bound flag. Not in PRD. |
|
||||
| G19 | P1 | Raw values indexed; PVS ids, booleans, dates not searchable by their display form; `recordLabel` often null on real-time path | `IndexingUtils.java:94, 142, 150-154` | 1.0: index display values, fallback to `QValueFormatter.formatRecordLabel`. Not in PRD. |
|
||||
| G20 | P1 | `enableScheduledProcesses=false` also removes manual full reindex and reconcile | `QuickSearchQBitProducer.java:228-232` | 1.0: split the flag (backwards compatible: keep old name as alias). Not in PRD. |
|
||||
| G21 | P2 | Run history grows unbounded once scheduled; no purge | No purge code | 1.0 if trivial (one delete-by-filter in the scheduled step), else 1.1. Not in PRD. |
|
||||
| G22 | P2 | `quickSearchIndex` lacks `lastRunStatus`, `lastError`, `documentCount`, `lastReconcileTime`; `status` never changes; no PVS/joins/sections | `QuickSearchQBitProducer.java:456-497`; `AbstractIndexingStep.java:175` | 1.0 polish with E5; schema additions are cheap before 1.0 and painful after. Not in PRD. |
|
||||
| G23 | P2 | Health check and metrics absent | No ping/health call | 1.0: a health method surfaced in the admin table; metrics 1.x. Not in PRD. |
|
||||
| G24 | P2 | No term min/max length; 1-char term returns nothing silently | `QuickSearchAction.java:67-74` | 1.0 (trivial, align with core 100-char cap). Not in PRD. |
|
||||
| G25 | P2 | Large text fields unbounded | `IndexingUtils.buildSearchableText` | 1.0: per-field `maxLength` default. Not in PRD. |
|
||||
| G26 | P2 | `IndexEvent` has no timestamp/version; publisher `close()` never called; publisher cannot resolve table configs without the static context | `publisher/IndexEvent.java:30-33`; `IndexEventPublisher.java:60` | 1.0 if the SPI is to be stable at 1.0 (PRD says "the SPI stays"): add `occurredAt`, pass config resolver, wire `close()`. Otherwise mark SPI experimental in 1.0. Not in PRD. |
|
||||
| G27 | P2 | No document-builder, analyzer or result hooks | `IndexingUtils` static; `ensureIndexExists` hard-coded | 1.1 unless needed for E3 (a result hook is the natural place for permission post-filtering). Not in PRD. |
|
||||
| G28 | P2 | Refresh policy not configurable; read-after-write misses | Bulk requests default | 1.1; document the 1s default in 1.0. Not in PRD. |
|
||||
| G29 | P2 | Highlight snippets expose field names via `includeLabel` prefix; README says "label" | `IndexingUtils.java:106-108` | 1.0 docs fix at least; use QQQ field label in 1.0 if cheap. |
|
||||
| G30 | P3 | Fuzziness, phrase queries, synonyms, language analyzers, sort options | Not present | 1.x (PRD out of scope). |
|
||||
| G31 | P3 | Composite keys, multi-index layouts | Not present | 1.x (PRD out of scope). |
|
||||
| G32 | P1 | Tests: JaCoCo gate inert; integration suite exercises one whole-word term only; no basepull, scheduling, permission, typeahead, multi-word, boost-order, non-integer PK, OpenSearch-down, mapping-conflict tests | `pom.xml:188-212`; test list in section 4 | 1.0 (PRD E1 covers the agent; add the concrete tests below). |
|
||||
| G33 | P1 | Docs: README promises (scheduled basepull, admin UI per-table reindex, coverage enforced, qbit-crm/wms integrations, field "label"), no CHANGELOG or migration notes | `README.md:14, 30, 79, 171, 254, 271, 287, 333-341, 362` | 1.0 (PRD E5). |
|
||||
|
||||
## 4. Test inventory and concrete untested behaviours
|
||||
|
||||
Unit (233): `QuickSearchQBitConfigTest` (34), `QuickSearchQBitProducerTest` (23), `IndexingUtilsTest` (19), `QuickSearchActionTest` (16), `QuickSearchOpenSearchClientTest` (13, constructor/unavailable-server only), `SynchronousIndexEventPublisherTest` (12), `BasepullIndexStepTest` (9), `QuickSearchPostUpdateCustomizerTest` (9), `QuickSearchQBitContextTest` (9), `FullReindexStepTest` (8), `QuickSearchPostDeleteCustomizerTest` (8), `BulkIndexResultTest` (8), `QuickSearchableTableConfigTest` (8), `QuickSearchDTOsTest` (7), `QuickSearchAnnotationsTest` (7), `ReconcileIndexStepTest` (6), `QuickSearchPostInsertCustomizerTest` (6), `SearchableFieldConfigTest` (6), `SearchableTableConfigTest` (6), `IndexEventTest` (4), `OpenSearchDocumentTest` (4), `QuickSearchIndexTest` (4), `QuickSearchIndexRunTest` (4), `RequiresDockerConditionTest` (3).
|
||||
|
||||
Integration (12, Testcontainers OpenSearch 2.11): `OpenSearchIntegrationTest` (index created, full reindex, search returns, pagination, no results), `ConfigDrivenIndexingIntegrationTest` (discovery, full reindex, label format, no results), `IndexDriftIntegrationTest` (partial update keeps fields, delete with missing doc, reconcile repairs orphan/missing/stale).
|
||||
|
||||
Concretely untested:
|
||||
|
||||
1. Query shape: no test asserts the OpenSearch request (operator, boosts, highlight, from/size). `QuickSearchOpenSearchClient.search` is only exercised end-to-end with "widget".
|
||||
2. Typeahead: README's "ord finds order" is never tested; nor that "or" does not outrank "order".
|
||||
3. Multi-word terms and `minimum_should_match` behaviour.
|
||||
4. Field weight ordering (a weight-3 field match ranks above a weight-1 match).
|
||||
5. Highlight fragment content against a real cluster.
|
||||
6. Basepull against a real cluster; watermark semantics; timestamp filter with seconds-precision columns; rows modified mid-run.
|
||||
7. Any scheduling (nothing to test yet).
|
||||
8. Permission denial (nothing to test yet).
|
||||
9. Non-integer and string primary keys through reconcile keyset paging.
|
||||
10. OpenSearch down at runtime during a host insert (only a mocked publisher-throws test exists); produce with an unreachable host beyond `ensureIndexExists` failing.
|
||||
11. Pre-existing index with a different mapping/analyzer.
|
||||
12. Two tables sharing a field name with different types (mapping conflict).
|
||||
13. `totalHits` above 10,000; `offset + limit` above `max_result_window`.
|
||||
14. `enableScheduledProcesses=false` behaviour; `tableNamePrefix` applied to processes/app end to end.
|
||||
15. Duplicate `@QuickSearchable(tableName)` across two entity classes (only annotation-vs-config duplicates are checked, `QuickSearchQBitProducer.java:144`).
|
||||
16. Large text fields; `includeLabel` text appearing in snippets.
|
||||
17. Publisher `close()` lifecycle.
|
||||
18. `enabled=false` row: customizers/search still index and return (would currently fail if written as a negative test, which is the point).
|
||||
|
||||
## 5. Items the draft PRD does not mention
|
||||
|
||||
The PRD's five epics cover: 4.1 contract and fail-fast (E1), scheduling, ordered paging, alias-swap reindex, per-item run errors (E2), permissions (E3), core SPI and result mapping (E4), docs and release (E5). It does not mention:
|
||||
|
||||
1. Transport timeouts (G01). The single largest operational risk; without it, "OpenSearch down" slows every host write.
|
||||
2. Search query quality: `search_analyzer`, `operator`/`minimum_should_match`, boosts not applying to prefixes (G04). This changes the mapping and should be bundled with the alias-swap work so 1.0 starts with the right index.
|
||||
3. `fieldValues` cross-table mapping conflicts (G09). Same reindex.
|
||||
4. Honouring `enabled` outside basepull (G07).
|
||||
5. Basepull watermark correctness (G08). E2 says "order-by primary key" but not the start-time watermark or overlap.
|
||||
6. Failed-event capture, retry, circuit breaker (G06). E2's "run records capture per-item errors" covers batch runs only; real-time failures still vanish.
|
||||
7. Orphaned documents for removed/renamed tables (G14).
|
||||
8. Config/mapping drift detection and `NEEDS_REINDEX` status (G15).
|
||||
9. Validation that configured fields and the timestamp field exist on the table; `basepullTimestampField = null` for tables without one (G13). E1 says "fail-fast config validation" generically.
|
||||
10. Process input screens with a table PVS (G17). Without it, the "Or through the QQQ admin UI" README promise stays false after E5.
|
||||
11. `totalHits` cap, `limit`/`offset` bounds, term length bounds (G18, G24).
|
||||
12. Display values and PVS translation at index time; `recordLabel` fallback (G19). Core does both; a UI backed by Quick Search will show different labels than core for the same record.
|
||||
13. Input-side parity with core (`tableNames`, `limitPerTable`) and `tableLabel` on results (G16). E4 mentions only the output mapping.
|
||||
14. The `enableScheduledProcesses` flag removing manual processes (G20).
|
||||
15. Run-history retention (G21) and the admin table's missing status/error/count fields (G22). Schema changes are cheap before 1.0.
|
||||
16. Health check (G23).
|
||||
17. `IndexEvent` ordering metadata and publisher lifecycle (G26). The PRD keeps the SPI but does not say whether it is stable; as it stands an async implementation cannot be correct.
|
||||
18. Large-field truncation (G25) and refresh policy (G28).
|
||||
19. Defaulting the QBit's field list from `QTableMetaData.searchFields` so hosts do not maintain two lists.
|
||||
20. Voyage as a consumer: the local Voyage checkout (`/Users/james.maes/Git.Local/QRunIO/Voyage`, develop, main, circleci-project-setup) contains no reference to Quick Search or OpenSearch on any ref; the PRD's "Voyage (pins 0.2.0)" stakeholder row should be re-verified or dropped. qqq-all is the only confirmed consumer (`DemoQBits.java:151-163`, config-driven `customer` name/email, hand-registered `QBitMetaData` 0.2.1-SNAPSHOT, no `schedulerName` while the SFTP QBit on the same line uses `withSchedulerName("demoScheduler")`).
|
||||
|
||||
## 6. Suggested adjustments to the epics
|
||||
|
||||
- E1: add field-existence validation, OpenSearch naming validation, timeouts in config, split of `enableScheduledProcesses`.
|
||||
- E2: add start-time watermark with overlap, orphan-table sweep in reconcile, config-drift detection, run-history purge, failed-event table replayed by the scheduled run. Bundle the mapping changes (search_analyzer, operator, fieldValues namespacing, display values) into the alias-swap work so the first 1.0 index is right.
|
||||
- E3: add index-time exclusion of hidden/masked fields; honour `enabled` in search.
|
||||
- E4: add `tableNames`, `limitPerTable`, `tableLabel`, typed `recordId`, term bounds; default fields from `searchFields`.
|
||||
- E5: add process input screens, admin table fields, health method, README corrections listed in 7c, and an explicit "SPI stability" statement for `IndexEventPublisher`.
|
||||
@@ -0,0 +1,91 @@
|
||||
# 05. Correctness and silent-failure review: qbit-quick-search
|
||||
|
||||
Scope: every file under `src/main/java/com/kingsrook/qbits/quicksearch` and `src/test/java`, read in full, cross-checked against QQQ `v4.1.0-RC.1` (`InsertAction`, `UpdateAction`, `DeleteAction`, `QueryAction`, `QueryInput`, `QRecord`, `ValueUtils`, `TableCustomizerInterface`, `RDBMSQueryAction`, `RDBMSInsertAction`, `RDBMSUpdateAction`, `RDBMSDeleteAction`, `RDBMSTransaction`, `QInstance`, `ProcessLockUtils`) and the opensearch-java 2.10.0 jar in the local Maven repo. Read-only; no repo file was modified.
|
||||
|
||||
Paths below are relative to `/Users/james.maes/Git.Local/QRunIO/qbit-quick-search-b1/src/main/java/com/kingsrook/qbits/quicksearch/` unless prefixed with `qqq:` (relative to `/Users/james.maes/Git.Local/QRunIO/qqq/` at tag `v4.1.0-RC.1`).
|
||||
|
||||
## Framework facts this review depends on (verified)
|
||||
|
||||
- POST_INSERT / POST_UPDATE / POST_DELETE customizers run synchronously, in the calling thread, immediately after the backend DML and before `execute()` returns (`qqq:qqq-backend-core/.../actions/tables/InsertAction.java:286`, `UpdateAction.java:268`, `DeleteAction.java:370-390`).
|
||||
- When the caller passes a `QBackendTransaction`, the RDBMS module runs the DML on that transaction's connection (`qqq:qqq-backend-module-rdbms/.../RDBMSInsertAction.java:73-79`, `RDBMSUpdateAction.java:85-91`, `RDBMSDeleteAction.java:87-93`). `RDBMSTransaction` sets `autoCommit=false` (`RDBMSTransaction.java:55`) and only the caller's later `commit()` (`:87`) or `rollback()` (`:137`) ends it. So the customizers run inside an open transaction.
|
||||
- A customizer exception never propagates to the caller; QQQ catches it and attaches a `QWarningMessage("An error occurred after the insert/update/delete: ...")` to each output record (`InsertAction.java:344-350`, `UpdateAction.java:291-297`, `DeleteAction.java:418-426`).
|
||||
- Post-delete receives the pre-fetched old records minus those with errors (`DeleteAction.java:376`); post-update receives the (possibly sparse) updated records plus `Optional<oldRecordList>`.
|
||||
- RDBMS paging emits `LIMIT n OFFSET m` and emits `ORDER BY` only when the filter has order-bys (`qqq:qqq-backend-module-rdbms/.../RDBMSQueryAction.java:340-354`). There is no default ordering, so unordered paging is not stable.
|
||||
- `QueryAction` sets `QRecord.recordLabel` only when `queryInput.getShouldGenerateDisplayValues()` is true (`qqq:.../QueryAction.java:503-506`); the default is `false` (`qqq:.../QueryInput.java:56`).
|
||||
- `QRecord.getValueString` is `ValueUtils.getValueAsString`, which returns `String.valueOf(value)` for Instant, LocalDate, BigDecimal, Boolean, and the possible-value id (not the label) for enums (`qqq:.../ValueUtils.java:67-90`).
|
||||
- `QInstance.addTable` throws `IllegalArgumentException` on a duplicate name (`qqq:.../QInstance.java addTable`).
|
||||
- `org.opensearch.client.opensearch._types.OpenSearchException extends java.lang.RuntimeException` (javap on opensearch-java 2.10.0). The client wrapper only catches `IOException`.
|
||||
|
||||
## Findings
|
||||
|
||||
| ID | Severity | Finding | Failure scenario | Evidence | Recommended fix | 1.0 blocker |
|
||||
|----|----------|---------|------------------|----------|-----------------|-------------|
|
||||
| F01 | Critical | Real-time indexing happens inside the caller's open transaction, before commit. | Host code inserts an order inside a `QBackendTransaction`, the post-insert customizer indexes it, the host then rolls back: OpenSearch now holds a document for a row that never existed, and it is only removed by the next reconcile (basepull cannot see deletes). Post-update is worse: it re-reads the row through the same transaction (`setTransaction`) and indexes uncommitted values. | `customizers/QuickSearchPostInsertCustomizer.java:97`; `customizers/QuickSearchPostUpdateCustomizer.java:114,151`; `customizers/QuickSearchPostDeleteCustomizer.java:109`; `qqq:InsertAction.java:286`; `qqq:RDBMSInsertAction.java:73-79`; `qqq:RDBMSTransaction.java:55,87` | QQQ has no post-commit hook, so: (a) when `input.getTransaction() != null`, buffer events keyed by the transaction object and flush them from a wrapper the host calls after `commit()` (ship a `QuickSearchTransactionListener` helper), or at minimum (b) document loudly that real-time indexing is pre-commit and that reconcile is the correction path; and make reconcile the recommended scheduled job. | Yes (fix or document; today the README implies the index tracks the table) |
|
||||
| F02 | Critical | Real-time indexing failures are fully silent to the host. | OpenSearch is down for ten minutes; every insert/update/delete in that window logs one WARN line and succeeds normally; no run record, no counter, no record warning. QQQ would have attached a `QWarningMessage` to the returned records if the exception had been allowed to escape, but the customizers swallow it first. The index silently drifts until someone runs reconcile. | `customizers/QuickSearchPostInsertCustomizer.java:99-102`; `QuickSearchPostUpdateCustomizer.java:117-120`; `QuickSearchPostDeleteCustomizer.java:112-115`; `publisher/SynchronousIndexEventPublisher.java:160-171` (throwIfAnyFailed is caught by the customizers and goes nowhere); `qqq:InsertAction.java:344-350` | Remove the blanket catch in the three customizers (QQQ already guarantees the write is not failed; the host gets a warning on the records). Additionally record the failure durably: increment a failure counter on the `quickSearchIndex` row (new `realTimeErrorCount`/`lastRealTimeError`), or write the failed (table, id) pairs to a small dead-letter table that basepull/reconcile drains. | Yes |
|
||||
| F03 | High | Basepull and FullReindex page with `LIMIT/OFFSET` and no `ORDER BY`. | While FullReindex pages through 1M rows, a concurrent insert or delete shifts the physical order: a row moves from page 101 to page 100 (already read) and is never indexed, or is read twice. Because FullReindex first deletes every document for the table, a skipped row disappears from search until the next full run. Postgres/MySQL do not guarantee a stable order for repeated unordered `LIMIT/OFFSET` queries even without writes. | `processes/AbstractIndexingStep.java:358-359`; `processes/BasepullIndexStep.java:218-262`; `processes/FullReindexStep.java:158-185`; `qqq:RDBMSQueryAction.java:340-354` | Replace offset paging with the keyset loop already written in `processes/ReconcileIndexStep.java:222-238` (order by PK ascending, `PK > lastKey`). For basepull, keyset on PK with the timestamp criterion added. Delete `querySourceTableBatch` once both callers migrate. | Yes |
|
||||
| F04 | High | Basepull window is anchored at the end of the run and compared against the application clock, so records modified during the run are lost. | Run starts at 10:00:00, pages for 90 s; a row updated and committed at 10:00:30 is on a page already read with its old values, or not yet visible; at 10:01:30 `lastBasepullTime = now()`; the next run filters `modifyDate > 10:01:30` and never sees the 10:00:30 change. Any DB/app clock skew widens or flips the gap. `GREATER_THAN` also drops rows whose `modifyDate` equals the boundary exactly. | `processes/BasepullIndexStep.java:198-209` (filter), `:268-279` (`lastBasepullTime = Instant.now()` after the loop); `processes/FullReindexStep.java:196` (same end-of-run stamp) | Capture `windowStart = Instant.now()` before the first query and persist that (as `ReconcileIndexStep.java:142,274` already does), subtract a configurable overlap (default: the basepull interval, or 5 minutes) to absorb skew and long transactions, and use `GREATER_THAN_OR_EQUALS`. Re-indexing is idempotent, so overlap is cheap. | Yes |
|
||||
| F05 | High | No document versioning; last writer to OpenSearch wins, not last committer. | Thread A updates row 7 (v1) and, in its post-update, reads v1 back; thread B updates row 7 (v2), commits, indexes v2; thread A's bulk arrives last and overwrites with v1. Same race between FullReindex/Reconcile reading an old page and a real-time write that lands between the read and the bulk. Also applies to delete-then-reinsert of the same id across threads. | `opensearch/QuickSearchOpenSearchClient.java:263-267` (`IndexOperation` with no `version`/`versionType`/`ifSeqNo`); `processes/IndexingUtils.java:164` (`indexedAt = now()` is informational only) | Use external versioning: `versionType(External)` with `version = modifyDate.toEpochMilli()` (or a DB version column) so stale writes are rejected with 409 and counted, not applied. Short of that, document that basepull/reconcile is the convergence mechanism (true only once F04 is fixed). | No, if F04 is fixed and the limitation is documented |
|
||||
| F06 | High | Search-time analysis uses the edge_ngram analyzer, so query terms are n-grammed too. | Mapping sets `analyzer("quick_search_analyzer")` with no `search_analyzer`, so OpenSearch analyzes the query with the same edge_ngram filter. Searching `stella` becomes `st OR ste OR stel OR stell OR stella` and matches every document containing any token that starts with `st`. Scores still favour the full term, but `totalHits`, `hasMore`, and pages 2+ are mostly noise. | `opensearch/QuickSearchOpenSearchClient.java:177-190` | Add a second analyzer (`standard` tokenizer + `lowercase`) and set it as `searchAnalyzer` on `searchableText`. Requires index recreation (bump the index name or add a mapping-version check in `ensureIndexExists`, which currently returns early if the index exists at `:165-171` and never reconciles mappings). | Yes |
|
||||
| F07 | High (behaviour of OpenSearch assumed from its documented default `lenient=false`; verify with the Docker test) | Boost clauses query `fieldValues.<field>` for every configured field, and `fieldValues` is a dynamic object; a non-string searchable field breaks every search. | A table marks `quantity` (Integer) or `createDate` (Instant) as a `@QuickSearchField`. The first indexed document maps `fieldValues.quantity` as `long`. The next search for `widget` issues `multi_match` against that field with a non-numeric term; OpenSearch returns `query_shard_exception: failed to create query ... For input string: "widget"` and the whole bool query (not just the should clause) fails. Separately, dynamic mapping is first-value-wins: a `sku` whose first value looks like a date maps as `date`, and later SKUs fail per-item with `mapper_parsing_exception`. | `opensearch/QuickSearchOpenSearchClient.java:192` (dynamic `object`), `:536-561` (per-field multi_match, no `lenient`); `processes/IndexingUtils.java:150-155` (raw `Object` values copied into `fieldValues`) | Either set `.lenient(true)` on the should-clause multi_match queries, or define a `dynamic_templates` rule mapping everything under `fieldValues.*` as `text` with the search analyzer, or stringify values when building `fieldValues`. | Yes |
|
||||
| F08 | High | `recordLabel` is null for every annotation-configured table. | `buildDocument` copies `record.getRecordLabel()`, but the records come from `QueryAction` calls that never set `shouldGenerateDisplayValues`, and from `InsertAction` output, neither of which populates the label. Only tables with `recordLabelFormat` (config-driven path) get a label. Search results for annotation-based tables show no label, which is the primary display field. | `processes/IndexingUtils.java:142`; `processes/AbstractIndexingStep.java:361-365`; `processes/ReconcileIndexStep.java:233-237`; `customizers/QuickSearchPostUpdateCustomizer.java:149-154`; `qqq:QueryAction.java:503-506`; `qqq:QueryInput.java:56` | In `buildDocument`, when `recordLabelFormat` is absent, call `QValueFormatter.formatRecordLabel(QContext.getQInstance().getTable(tableName), record)`; or set `shouldGenerateDisplayValues(true)` on every source query and translate possible values (also fixes F13). | Yes |
|
||||
| F09 | High | FullReindex wipes the table's documents first; a mid-run failure leaves a partially empty index with no automatic recovery. | `deleteDocumentsForTable` runs, then OpenSearch returns 429 on batch 40 of 1000: `OpenSearchException` (unchecked) escapes `indexDocuments`, the run is marked FAILED, 96 percent of the table is unsearchable until someone reruns. Also `deleteByQuery` is called with default `conflicts=abort` and no `refresh`, so a real-time write racing the delete raises `version_conflict_engine_exception` and fails the run after a partial wipe. The FAILED run record from the bulk-item path carries `errorMessage = null`. | `processes/FullReindexStep.java:152` (wipe), `:160-185` (loop), `:210` (null errorMessage on FAILED), `:215-220`; `opensearch/QuickSearchOpenSearchClient.java:388-403` (no `conflicts(Proceed)`, no `refresh`) | Prefer the reconcile algorithm (index into place, then delete stale) for the "full" process too, or build into a fresh index and swap an alias. At minimum: pass the first bulk error into `completeRunRecord`, set `conflicts(Proceed)`, and document the blackout. | Yes for the errorMessage and conflicts fixes; the wipe design should be documented or replaced before 1.0 |
|
||||
| F10 | Medium | `OpenSearchException` (unchecked) bypasses every `catch(IOException)` in the client and reaches `QuickSearchAction` callers raw. | Someone deletes the index (`index_not_found_exception`), or a client sends `offset=20000` (`max_result_window`), or F07 fires: `QuickSearchAction.execute` throws a `RuntimeException` rather than the declared `QException`; UI/API layers that only handle `QException` surface a 500 with the raw OpenSearch JSON. In the steps and customizers the `catch(Exception)` blocks do catch it, so only the search path is exposed. | `opensearch/QuickSearchOpenSearchClient.java:197,220,349,374,399,441,506,581` (all `catch(IOException)`); `actions/QuickSearchAction.java:91` | Catch `OpenSearchException` alongside `IOException` in every client method and wrap in `QException` with `error().reason()`. | Yes (small) |
|
||||
| F11 | Medium | Search input has no upper bound on `limit` or `offset`. | `limit=100000` builds a 100k-hit response (memory and `max_result_window` error); `offset+limit > 10000` is rejected by OpenSearch and surfaces as F10. `totalHits` is also capped at 10000 by `track_total_hits` default, so `hasMore` is wrong beyond that. | `actions/QuickSearchAction.java:78-79`; `opensearch/QuickSearchOpenSearchClient.java:497-502` | Clamp `limit` to a configurable max (e.g. 100) and reject or clamp `offset` so `from + size <= 10000`; set `trackTotalHits(true)` only if exact counts matter. | Yes (small) |
|
||||
| F12 | Medium | `quickSearchIndex.enabled` is honoured only by basepull; the row's `basepullTimestampField` is never read. | An admin disables a table in the UI; the next FullReindex or Reconcile still indexes it. An admin edits `basepullTimestampField` on the row; basepull keeps using the annotation value. | `processes/FullReindexStep.java:90-96`; `processes/ReconcileIndexStep.java:98-106`; `processes/BasepullIndexStep.java:113` (read) vs `:207` (uses `tableConfig.getBasepullTimestampField()`) | Check `enabled` in all three steps (or make the field read-only and document). Pick one source of truth for the timestamp field and remove the other. | No (document) |
|
||||
| F13 | Medium | Basepull on a table whose config was removed but whose row and QQQ table still exist silently scans everything and indexes nothing. | `getTableConfig` returns null; the filter stays null (full scan), `buildDocument` is skipped for every row, the run completes `COMPLETED` with `recordsProcessed=N, recordsIndexed=0`, and `lastBasepullTime` advances. | `processes/BasepullIndexStep.java:184,202-209,236-240,268` | If `tableConfig == null`, mark the run FAILED (or the row `status=INACTIVE`) and skip. | No |
|
||||
| F14 | Medium | `ensureIndexRowExists` assumes a unique constraint on `tableName` that the produced metadata does not declare. | Two replicas run the first basepull together; both see no row, both insert; without a DB unique index there is no exception, the recheck is never reached, and the table now has two rows. Basepull then processes the table twice per cycle (two run records, two `lastBasepullTime` stamps); FullReindex and Reconcile use `get(0)` arbitrarily. | `processes/AbstractIndexingStep.java:143-195`; `QuickSearchQBitProducer.java:457-475` (no `withUniqueKey`) | Add `.withUniqueKey(new UniqueKey("tableName"))` to the table metadata so QQQ's pre-insert unique check fires, and ship DDL with a unique index. | Yes (metadata only) |
|
||||
| F15 | Medium | Per-table basepull failures are logged at WARN and the process still reports success. | OpenSearch rejects one table's bulk every run; the scheduler sees a green process forever. | `processes/BasepullIndexStep.java:124-132` | Collect failures and throw (or set an output value and a FAILED status on the step) after the loop; expose counts via `output.addValue` as Reconcile does. | Yes (small) |
|
||||
| F16 | Medium | `produce()` tolerates OpenSearch being unreachable and starts with `client=null, publisher=null`. | Misconfigured host/port in production: the app boots healthy, every write logs "publisher is null", every search throws "client is not initialized". Nothing in the operational tables records this. | `QuickSearchQBitProducer.java:159-167,172-182,187-195`; `customizers/*.java:74-79` | Add `failFastOnStartup` (default true) to the config, or at least retry client creation lazily on first use and surface a health check. | No (document) |
|
||||
| F17 | Medium | Static singleton context: re-produce leaks transports, and two QInstances in one JVM share one context. | `produce()` called twice (tests, qqq-all, hot reload): the first `ApacheHttpClient5Transport` and its IO reactor threads are never closed (only `clear()` nulls references; main code never calls `publisher.close()` or `client.close()` outside tests). A second QInstance overwrites `config/client/discoveredTables`, so instance A's customizers index into instance B's index. Calling `produce()` twice on the same QInstance throws from `addTable` after the static context has already been replaced. | `QuickSearchQBitContext.java:34-37,152-158`; `QuickSearchQBitProducer.java:200-215`; `qqq:QInstance.java addTable` | Close the previous client when replacing it; register a shutdown hook or expose `QuickSearchQBit.shutdown()`; document the one-QInstance-per-JVM constraint; longer term key the context by `QInstance` identity. | No (document; close-on-replace is trivial and worth doing) |
|
||||
| F18 | Medium | `String.format(recordLabelFormat, ...)` with `""` substituted for null throws on non-`%s` specifiers and poisons the whole batch. | `recordLabelFormat="%s #%d"` and `orderNumber` null: `IllegalFormatConversionException` (unchecked) escapes `buildDocument`; in `SynchronousIndexEventPublisher.publishIndexEvents` this happens before any bulk call, so every document in the batch (all tables) is dropped and the customizer logs one WARN. In the steps it fails the table's run. | `processes/IndexingUtils.java:194-200`; `publisher/SynchronousIndexEventPublisher.java:113-120` | Validate the format at `produce()` time against the field types; wrap per-record `buildDocument` in try/catch, count as a failure, continue. | No |
|
||||
| F19 | Medium | Indexed text is raw `String.valueOf`: PVS ids instead of labels, ISO instants, `true/false`, unbounded length, no HTML stripping. | A `status` PVS field indexes `3` not `Shipped`, so users cannot search by status label; a `notes` CLOB of 5 MB is sent whole (500 per bulk can exceed `http.max_content_length` and fail the batch with a 413 that surfaces as F10 in the steps). | `processes/IndexingUtils.java:94,150-155`; `qqq:ValueUtils.java:67-90` | Query with `shouldGenerateDisplayValues(true)` and `shouldTranslatePossibleValues(true)` and index `getDisplayValue(field)`; truncate each value to a configurable max (e.g. 32 KB); strip tags for `HTML` adorned fields. | No |
|
||||
| F20 | Low | Public `buildSearchableText` dereferences `includeLabels` without a null check. | Any external caller passing `null` for `includeLabels` gets an NPE; the table-config overload guards it, the 6-arg overload does not. | `processes/IndexingUtils.java:83-115` (`:106`) vs `:190` | Null-guard inside `buildSearchableText`. | No |
|
||||
| F21 | Low | Document ids are `sourceTable:recordId`, not namespaced by `tableNamePrefix` or instance. | Two apps pointed at the same `opensearchIndexName` with a table named `order` overwrite each other's documents, and either app's FullReindex/Reconcile deletes the other's documents (`deleteDocumentsForTable`, `deleteDocumentsIndexedBefore` filter on `sourceTable` only). Changing `tableNamePrefix` does not change ids (good: no orphan churn). A table name containing `:` can collide with another (table, id) pair only in contrived cases. | `opensearch/OpenSearchDocument.java:184-186`; `opensearch/QuickSearchOpenSearchClient.java:392-397,428-437` | Document one index per QInstance (validate at produce time if the index already holds a different app's marker document), or include an app/prefix token in `sourceTable`. | No |
|
||||
| F22 | Low | Reconcile's orphan detection depends on `indexedAt` written by whichever node indexed the document, using that node's clock. | Node B's clock is 3 minutes behind node A; node A starts reconcile at T; node B inserts a row at T+1 min and indexes it with `indexedAt = T-2 min`; reconcile deletes it as an orphan. Basepull repairs it only if its `modifyDate` (DB clock) is `> reconcileStart` and the row still exists. The comment at `:62-63` acknowledges the assumption. The orphan delete itself uses `delete_by_query` with `conflicts(Proceed)` and `refresh`, so there is no 10000-document cap and no scroll needed. | `processes/ReconcileIndexStep.java:142,192`; `opensearch/QuickSearchOpenSearchClient.java:420-445`; `processes/IndexingUtils.java:164` | Replace the time cutoff with a per-run token (`reconcileRunId` stored on each document; delete `sourceTable = X AND reconcileRunId != current`), or subtract a skew margin from the cutoff. | No |
|
||||
| F23 | Low | `RUNNING` run records are never swept; `completeRunRecord` failures are WARN-only. | JVM killed mid-run (or DB blip at completion): the row stays `RUNNING` indefinitely; operators cannot tell a hung run from a dead one. | `processes/AbstractIndexingStep.java:264-292,310-334` | On step start, mark `RUNNING` rows older than N hours as `ABANDONED`; or add a heartbeat column. | No |
|
||||
| F24 | Low | FullReindex aborts remaining tables on the first table failure; basepull continues. Inconsistent semantics. | A reindex-all with 5 tables fails on table 2; tables 3 to 5 are untouched and no run records exist for them. | `processes/FullReindexStep.java:90-96,215-220` vs `processes/BasepullIndexStep.java:124-132` | Pick one policy (continue and aggregate is friendlier for a reindex-all) and apply it to all three steps. | No |
|
||||
| F25 | Low | `normalizeSearchText` lowercases client-side even though both index and (intended) search analyzers lowercase; harmless but misleading once F06 is fixed. The highlight join and `totalHits` fallback are correct: `fragments` emptiness is checked before `String.join("...")`, and when `total()` is null `totalHits = results.size()` so `hasMore` is false. | None. | `actions/QuickSearchAction.java:76,102-110,123-124` | Leave the whitespace collapsing, drop the lowercase, or keep as belt-and-braces with a comment. | No |
|
||||
|
||||
### Items checked and found acceptable
|
||||
|
||||
- `ReconcileIndexStep` keyset paging works for non-numeric keys and gaps: both the `ORDER BY` and the `GREATER_THAN` comparison are evaluated by the database, so string collation is consistent; a null primary key fails fast (`processes/ReconcileIndexStep.java:178-185`). Not verified for `byte[]` (binary UUID) keys.
|
||||
- `ReconcileIndexStep` when the orphan delete throws after a successful re-index: run FAILED, index row untouched, documents are all current; the next run simply repeats. Acceptable.
|
||||
- `ReconcileIndexStep` race with a concurrent real-time update (stale page wins, F05) is repaired by the next basepull because `lastBasepullTime` is set to `reconcileStart` (`:274`), provided F04's overlap is in place.
|
||||
- `deleteDocuments` treats `not_found` as success (bulk delete items with status 404 carry no `error`), matching the comment at `opensearch/QuickSearchOpenSearchClient.java:283-284`.
|
||||
- `BulkIndexResult` accounting per item is correct; partial bulk application is reported, not hidden, on the batch path.
|
||||
- Post-delete correctly skips records with errors and null ids (`customizers/QuickSearchPostDeleteCustomizer.java:95-99`); post-update correctly re-reads the full row rather than indexing the sparse update (`QuickSearchPostUpdateCustomizer.java:132-155`), though through the open transaction (F01).
|
||||
|
||||
## Missing tests
|
||||
|
||||
Unit tests use `MemoryBackendModule` with a mocked `QuickSearchOpenSearchClient`; integration tests use the memory backend plus a Testcontainers OpenSearch. No test uses an RDBMS backend or a `QBackendTransaction`, so none of the transaction, ordering, or SQL-paging hazards are exercised.
|
||||
|
||||
Covered today (for calibration): post-update sparse-record re-read (`QuickSearchPostUpdateCustomizerTest.testUpdateAction_partialUpdate_reindexesFullRecord`, `IndexDriftIntegrationTest.testPartialUpdate_unchangedFieldsStaySearchable`); publisher throws on item failures (`SynchronousIndexEventPublisherTest.testPublish*_itemFailures_throwsQException`); customizers swallow publisher exceptions (`*CustomizerTest.*_publisherThrows_returnsRecordsWithoutThrowing`, which is testing the F02 behaviour as desired); reconcile keyset paging, stale delete skipped on errors, run FAILED (`ReconcileIndexStepTest`, `IndexDriftIntegrationTest.testReconcile_repairsOrphanMissingAndStaleDocuments`); basepull due/not-due and FAILED on client throw (`BasepullIndexStepTest`); full reindex wipe-then-index ordering and FAILED on throw (`FullReindexStepTest`); search mapping, defaults, hasMore (`QuickSearchActionTest`); config-driven `recordLabelFormat` label (`ConfigDrivenIndexingIntegrationTest.testSearchResults_recordLabelFormatApplied`).
|
||||
|
||||
Missing, by finding:
|
||||
|
||||
1. F01: `QuickSearchPostInsertCustomizerTest` (or a new `TransactionIntegrationTest` against H2 via `qqq-backend-module-rdbms`): insert inside a `QBackendTransaction`, roll back, assert no `publishIndexEvents` call (today it will be called, so this test documents the defect until fixed). Same for update and delete.
|
||||
2. F02: `QuickSearchPostInsertCustomizerTest`: when the publisher throws, assert the returned records carry a warning (after the fix) or that a failure counter/dead-letter row was written; today's `*_publisherThrows_returnsRecordsWithoutThrowing` tests lock in the silent behaviour and should be inverted.
|
||||
3. F03: `FullReindexStepTest` and `BasepullIndexStepTest`: capture every `QueryInput` passed to the source table and assert each filter has an `ORDER BY` on the primary key and uses a keyset criterion (no `skip`). Integration variant: insert a row from another thread between pages (use a `Answer` on the mocked client to trigger the insert during `indexDocuments`) and assert every id was indexed exactly once.
|
||||
4. F04: `BasepullIndexStepTest.testLastBasepullTime_isCapturedBeforeQuerying`: insert a record with `modifyDate` between the step's start and end (same trick: insert from within the mocked `indexDocuments`), run twice, assert it is indexed on the second run. `testBasepullFilter_isGreaterThanOrEquals_withOverlap`: assert criterion operator and that the bound is `lastBasepullTime - overlap`.
|
||||
5. F05: `QuickSearchOpenSearchClientTest` or `OpenSearchIntegrationTest.testStaleWriteRejected`: index v2 of a document, then v1 with an older version, assert v2 survives (requires the external-version fix).
|
||||
6. F06: `OpenSearchIntegrationTest.testSearch_doesNotMatchOnSharedPrefixOnly`: index "stella" and "stone", search "stella", assert `totalHits == 1`. Also `testEnsureIndexExists_upgradesMappingWhenVersionChanges`.
|
||||
7. F07: `OpenSearchIntegrationTest.testSearch_tableWithIntegerAndDateSearchableFields_doesNotError`: add an `INTEGER` and a `DATE_TIME` field to a searchable table, reindex, search a word. `testIndex_dynamicMappingConflictOnFieldValues_reportedPerItem` for the first-value-wins mapping case.
|
||||
8. F08: `OpenSearchIntegrationTest.testSearchResults_annotationTable_haveRecordLabel`: assert `getRecordLabel()` is not null for the annotation-based `testEntity` table (currently it would fail). `IndexingUtilsTest.testBuildDocument_withoutFormat_usesTableRecordLabelFields`.
|
||||
9. F09: `FullReindexStepTest.testBulkItemFailures_runRecordHasErrorMessage` (currently `errorMessage` is null on that path). `QuickSearchOpenSearchClientTest`/integration: `deleteDocumentsForTable` proceeds on version conflicts.
|
||||
10. F10: `QuickSearchOpenSearchClientTest.testSearch_indexMissing_throwsQException` (start a container, delete the index, search); `QuickSearchActionTest.testOpenSearchException_isWrappedAsQException` using a mocked client that throws `OpenSearchException`.
|
||||
11. F11: `QuickSearchActionTest.testLimit_aboveMax_isClamped`, `testOffsetPlusLimit_aboveMaxResultWindow_isRejectedOrClamped`.
|
||||
12. F12: `FullReindexStepTest.testDisabledIndexRow_isSkipped`, `ReconcileIndexStepTest.testDisabledIndexRow_isSkipped`, `BasepullIndexStepTest.testRowTimestampField_isUsedOverAnnotation` (or the inverse, once a source of truth is chosen).
|
||||
13. F13: `BasepullIndexStepTest.testIndexRowWithoutTableConfig_runMarkedFailedAndNoScan`.
|
||||
14. F14: `QuickSearchQBitProducerTest.testIndexTable_hasUniqueKeyOnTableName`; `BasepullIndexStepTest.testEnsureIndexRowExists_concurrentFirstRun_createsOneRow` (two threads).
|
||||
15. F15: `BasepullIndexStepTest.testOneTableFails_stepReportsFailure` (assert the step throws or sets an output value after processing the other tables).
|
||||
16. F16: `QuickSearchQBitProducerTest.testProduce_openSearchUnreachable_failsFastWhenConfigured`.
|
||||
17. F17: `QuickSearchQBitProducerTest.testProduce_twice_closesPreviousClient` (spy on client), `testProduce_sameQInstanceTwice_doesNotLeaveContextHalfReplaced`.
|
||||
18. F18: `IndexingUtilsTest.testBuildDocument_recordLabelFormatWithNumericSpecifierAndNullValue_doesNotThrow`; `SynchronousIndexEventPublisherTest.testOneBadRecord_doesNotDropWholeBatch`.
|
||||
19. F19: `IndexingUtilsTest.testBuildSearchableText_possibleValueField_usesLabel`, `testBuildSearchableText_instantAndBigDecimal_formatting`, `testBuildSearchableText_veryLongValue_truncated`.
|
||||
20. F20: `IndexingUtilsTest.testBuildSearchableText_nullIncludeLabels_doesNotThrow`.
|
||||
21. F22: `ReconcileIndexStepTest.testDocumentIndexedByLaggingClock_duringRun_notDeleted` (requires the run-token design).
|
||||
22. F23: `AbstractIndexingStep` sweeper test: a `RUNNING` run older than the threshold is marked `ABANDONED` on the next step start.
|
||||
|
||||
## Summary of blockers for 1.0
|
||||
|
||||
Must fix or explicitly document before 1.0: F01 (pre-commit indexing), F02 (silent real-time failures), F03 (unordered paging), F04 (basepull window), F06 (search analyzer), F07 (non-text searchable fields break search), F08 (null record labels), F09 (errorMessage and conflicts on full reindex), F10 (unchecked OpenSearchException), F11 (limit/offset bounds), F14 (unique key on tableName), F15 (basepull hides failures). Everything else can ship with a documented limitation.
|
||||
@@ -0,0 +1,108 @@
|
||||
# Deep-dive audit: QBit Quick Search for 1.0 on QQQ 4.1
|
||||
|
||||
Date: 2026-10-04. Code: `develop` `000e5bb` (identical to `feature/GH-6-repin-and-drift-fix`), 0.2.1-SNAPSHOT. References: QQQ `v4.1.0-RC.1`, opensearch-java 2.10.0, OpenSearch and AWS documentation as of this date. Five parallel code-grounded reviews; the raw reports with file:line evidence are the numbered files beside this one. Every claim below was spot-checked against source where it changed a decision; one agent claim (that Voyage does not use the QBit) was wrong and is corrected here.
|
||||
|
||||
## Verdict
|
||||
|
||||
The QBit compiles and passes its 233 unit and 12 integration tests on QQQ 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1 with no source change, so it is **API-compatible with QQQ 4.1 today**. It is **not feature complete or production-safe for a 1.0**: the index can hold uncommitted writes, failures are silent, basepull loses records and is never scheduled, search leaks data across tables, the search analyzer inflates matches, the OpenSearch client line is end-of-life, and only HTTP basic auth over an optional TLS connection is supported. None of these are hard; most are a few days each. The draft PRD covered about half of them. The PRD is updated alongside this audit.
|
||||
|
||||
| Dimension | Score (1 to 10) | One line |
|
||||
|---|---|---|
|
||||
| QQQ 4.1 API compatibility | 9 | Builds and tests green on RC.1 (see section 1 for the contract gaps that remain) |
|
||||
| QQQ 4.1 QBit contract | 3 | Plain producer, no `QBitMetaData`, no schedule, no permission rules, static context, slot collisions |
|
||||
| Index trustworthiness | 3 | Pre-commit indexing, silent failures, lossy basepull window, unordered paging, wipe-first reindex |
|
||||
| Search quality and safety | 3 | No permissions, query terms edge-ngrammed, OR semantics, non-text fields break search, unbounded paging |
|
||||
| OpenSearch and AWS compatibility | 4 | 2.x self-managed and AWS FGAC basic auth work; 3.x needs a client bump; IAM SigV4 and Serverless unsupported |
|
||||
| Authentication and secrets | 4 | Basic only, plaintext credentials allowed, password on serializable metadata, no TLS trust config |
|
||||
| Operability | 4 | No timeouts, no health check, no failed-event capture, admin tables thin, processes without input screens |
|
||||
| Tests and docs | 5 | Good unit coverage of happy paths; JaCoCo gate inert; integration suite tests one word; README over-promises |
|
||||
|
||||
## 1. QQQ 4.1 integration readiness
|
||||
|
||||
Full detail with file:line evidence in `01-qqq-4.1-integration.md`. The QBit compiles and tests green on 4.1.0-RC.1, but it uses none of the 4.1 contract and misses three 4.1 APIs that solve its hardest problems. All three are absent from 4.0.0 (verified by tag), so a 1.0 that adopts them requires qqq 4.1 (decision D10).
|
||||
|
||||
| Topic | Today | 4.1 answer | Refs |
|
||||
|---|---|---|---|
|
||||
| Producer contract | Plain class; `produce(QInstance)` adds tables, processes and app directly; no `QBitMetaData`; qqq-all stamps one by hand with version `0.2.1-SNAPSHOT` and its test asserts artifactId `quick-search` | `QBitMetaDataProducer<QuickSearchQBitConfig>` (3 of 4 sibling QBits use it); `getQBitMetaData()` with the real version from a Maven-filtered resource; component producers or `@QMetaDataProducingEntity`; registration in `postProduceActions`; keep `withConfig()` and a `produce(QInstance)` override that adds to the instance so qqq-all and Voyage keep compiling | 01 F-01, F-12 |
|
||||
| Real-time hook and transactions | Three per-table POST_* customizers; run inside the caller's open transaction before commit; `withCustomizer` throws `IllegalArgumentException` on an occupied slot | `RecordChangeListenerInterface` (instance-level, `appliesTo(table, type)`, covers INSERT/UPDATE/DELETE, cannot fail the write, carries old records and the transaction) registered with `QInstance.withRecordChangeListener`; publish through `QBackendTransaction.addAfterCommitCallback` when a transaction is present (discarded on rollback; `RDBMSTransaction` runs them after JDBC commit); reference implementation `qqq-esb` `EsbRecordChangeListener`. `MultiCustomizer` only if customizers are kept as a 4.0 fallback; instance-level `withTableCustomizer` is unsuitable because `DeleteAction` never consults it | 01 F-02, F-03, F-04 |
|
||||
| Scheduling | No `QScheduleMetaData` or `BasepullConfiguration`; `QApplicationLauncher` starts the scheduler only if some process has a schedule; validator requires repeat-or-cron plus an existing scheduler name | `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression` on the config; attach schedules only when `schedulerName` is set; keep the per-table `quickSearchIndex` watermark (core `BasepullConfiguration` is keyed per process, not per table) but anchor it at run start minus an overlap | 01 F-05 |
|
||||
| Permissions | No `PermissionsHelper` in search; no `QPermissionRules` on produced metadata (enricher default `NOT_PROTECTED`) | Mirror core `RecordSearchAction` (table READ filter, `QueryAction` for locks); `permissionRules` config defaulting to `HAS_ACCESS_PERMISSION` for processes and app, `READ_INSERT_EDIT_DELETE_PERMISSIONS` for the operational tables | 01 F-06 |
|
||||
| Validation timing | Backend lookup works for the qqq-all ordering; package scanning would instantiate the producer with a null config and NPE; `QBitConfig.validate` swallows thrown exceptions (only collected errors fail); `QInstanceValidator` has no qbit checks | `isEnabled()` false when config is null; keep list-based validate null-safe; a `QInstanceValidatorPluginInterface` for late checks (backend, searchable tables, scheduler) | 01 F-07 |
|
||||
| Runtime state and lifecycle | Static `QuickSearchQBitContext`; OpenSearch connect and index creation happen during metadata production; nothing closes the transport | `QRuntimeServiceInterface` (`start` connects and ensures the index, `stop` closes), registered with `qInstance.withRuntimeService`; config resolved from `SourceQBitAware.getSourceQBitConfig()` on produced tables and processes (the webhooks pattern); `startupMode` `DEGRADED` or `FAIL_FAST`. Note: qqq-all's `FullQBitsTest` builds the full instance with OpenSearch at `127.0.0.1:1`, so either `DEGRADED` stays the default or qqq-all's test changes | 01 F-08, F-09, F-10 |
|
||||
| Core search overlap | `RecordSearchExecutor` hard-wires `new RecordSearchAction()`; no SPI | Propose `QInstance.withRecordSearchProvider(QCodeReference)` plus `RecordSearchProviderInterface { appliesTo(table); search(input, table, term, limit) }`, consulted after core's permission check | 01 F-11 |
|
||||
| Build | JaCoCo `check` has no `prepare-agent` and no version; JUnit pinned 5.10.0 while parent manages `junit-bom` 6.0.1; parent 2.1.0 not yet tagged | Bind the agent with an explicit version; drop the JUnit pins; re-pin to parent 2.1.0 when released and retire the snapshot profile's BOM import | 01 F-13 |
|
||||
|
||||
Ordered migration plan (from report 01): 1 contract and compat shims; 2 config surface; 3 record-change listener with after-commit publish; 4 runtime service and `SourceQBitAware` config resolution; 5 schedules, watermark, keyset paging; 6 permission-aware search; 7 build and parent 2.1.0 re-pin; 8 qqq-all follow-up (remove the manual `addQBit`, pass `withSchedulerName("demoScheduler")`) and the qqq SPI proposal.
|
||||
|
||||
## Status (2026-10-04, branch feature/GH-1-0-release-readiness)
|
||||
|
||||
All 25 blockers below are implemented on the release branch (commits 90cd09d through the admin-screen commit). Verification: 307 unit tests and 18 Testcontainers integration tests green against qqq 4.1.0-RC.1 and OpenSearch 2.19.6, JaCoCo 85% instruction / 96% class. Remaining before the 1.0.0 tag: parent 2.1.0 re-pin when published, qqq-all follow-up (remove its manual `addQBit`, set `schedulerName`, `startupMode DEGRADED` for its test), CI image matrix job, and the qqq core search SPI proposal (1.1).
|
||||
|
||||
## 2. Blockers for 1.0, deduplicated across reports
|
||||
|
||||
Severity: Critical = wrong or leaked data; High = breaks a target platform or a core promise; Medium = operational risk. IDs reference the raw reports (A = auth, C = compatibility, F = correctness, G = features).
|
||||
|
||||
| # | Sev | Blocker | Evidence | Fix in brief | Refs |
|
||||
|---|---|---|---|---|---|
|
||||
| B1 | Critical | Real-time indexing runs inside the host's open transaction, before commit; a rollback leaves a ghost document, and post-update re-reads uncommitted values through the same transaction (javalin single-record endpoints pass no transaction and autocommit, so this bites ETL, bulk and transactional host code) | `customizers/*` :97, :114, :151, :109; qqq `InsertAction.java:286`, `RDBMSTransaction.java:55,87`, `StreamedETLExecuteStep.java:204-218` | Replace the customizers with a `RecordChangeListenerInterface` (4.1) and publish through `QBackendTransaction.addAfterCommitCallback` (4.1, discarded on rollback) when a transaction is present; publish immediately otherwise | F01, 01 F-02, F-03 |
|
||||
| B2 | Critical | Real-time indexing failures are fully silent: customizers catch everything and log WARN; nothing reaches the host records or any table | `customizers/*` :99-120; `SynchronousIndexEventPublisher.java:160-171` | The record-change listener cannot fail the write by design; record failed (table, id, action) rows in a failed-event table that the scheduled run replays, and surface counts on `quickSearchIndex` | F02, G06 |
|
||||
| B3 | Critical | Search returns labels and highlight snippets from every indexed table with no session, table-permission or record-lock check; the `tableName` filter is caller-controlled | `actions/QuickSearchAction.java:81-121`; `QuickSearchOpenSearchClient.java:481-487` | `QuickSearchInput extends AbstractActionInput`; filter tables with `PermissionsHelper.hasTablePermission(READ)` as core `RecordSearchAction.java:110` does; `terms` filter on `sourceTable`; re-read hits via `QueryAction` by primary key for record locks | A1, G02 |
|
||||
| B4 | High | Basepull and full reindex page with limit/skip and no order-by; concurrent writes skip or duplicate rows, and full reindex wiped first so skipped rows vanish | `AbstractIndexingStep.java:358-359`; qqq `RDBMSQueryAction.java:340-354` | Reuse the keyset loop already in `ReconcileIndexStep.java:222-238` | F03, G12 |
|
||||
| B5 | High | Basepull watermark is `Instant.now()` at the end of the run with a strict greater-than, so rows modified during the run or in the boundary second are lost forever | `BasepullIndexStep.java:198-209, 268-279` | Capture run start before the first query, persist that minus an overlap, use greater-than-or-equal; reconcile already does this at `:142, :274` | F04, G08 |
|
||||
| B6 | High | Basepull and reconcile are never scheduled; README calls basepull "scheduled" | no `withSchedule` anywhere; `QuickSearchQBitProducer.java:510-518` | `QScheduleMetaData` on both processes with a `schedulerName` config (decision D4) | G03 |
|
||||
| B7 | High | `searchableText` has the edge-ngram analyzer but no `search_analyzer`, so query terms are n-grammed too: "stella" matches anything starting with "st"; `multi_match` defaults to OR with no `minimum_should_match` | `QuickSearchOpenSearchClient.java:177-190, 472-475` | Add a standard+lowercase search analyzer, `operator=AND` or `minimum_should_match`; mapping change, so bundle with the index-versioning work | F06, C5, G04 |
|
||||
| B8 | High | `fieldValues` is one dynamic object shared by all tables with raw typed values; same-named fields of different types cause `mapper_parsing_exception`, and boost clauses without `lenient` make a text term against a numeric field fail the whole search | `QuickSearchOpenSearchClient.java:192, 536-561`; `IndexingUtils.java:150-155` | `dynamic_templates` mapping `fieldValues.*` to text, `date_detection: false`, stringify values, `lenient(true)` | F07, C6, G09 |
|
||||
| B9 | High | `recordLabel` is null for every annotation-configured table because no source query sets `shouldGenerateDisplayValues` and insert output has no label; PVS fields index ids, not labels | `IndexingUtils.java:94, 142`; qqq `QueryAction.java:503-506` | Fall back to `QValueFormatter.formatRecordLabel`; query with display values and PVS translation | F08, F19, G19 |
|
||||
| B10 | High | Full reindex wipes the table's documents first; a mid-run failure leaves a partial index with no recovery; delete-by-query uses `conflicts=abort`; FAILED runs from the bulk path have a null error message | `FullReindexStep.java:152, 160-185, 210`; `QuickSearchOpenSearchClient.java:388-403` | Build into a fresh index and swap an alias (decision D5) or reuse the reconcile algorithm; `conflicts(Proceed)`; pass the first bulk error into the run record | F09, G12 |
|
||||
| B11 | High | No TLS trust configuration (no CA, trust store, or guarded insecure flag); `useSsl` defaults false and basic credentials over plain HTTP pass validation | `QuickSearchOpenSearchClient.java:98-139`; `QuickSearchQBitConfig.java:54, 103-112` | `tls` block (CA path, trust store, hostname verification, guarded `insecureSkipVerify`); reject credentials over http unless `allowPlaintextCredentials`; prefer a single `opensearchUrl` defaulting to https | C1, A2, A5 |
|
||||
| B12 | High | Password is a plain String on a Serializable config that hosts attach to `QBitMetaData`; `QInstanceAdapter.qInstanceToJson` would emit it | `QuickSearchQBitConfig.java:52-53`; qqq-all `DemoQBits.java:162-163` | Interpret `${env.X}` references with `QMetaDataVariableInterpreter` at client construction (the RDBMS/Mongo/S3 backend pattern); `@JsonIgnore` secret getters; redacting `toString()` | A3 |
|
||||
| B13 | High | opensearch-java pinned at 2.10.0, the end-of-life 2.x line documented for OpenSearch 1.x to 2.x; AWS already offers 3.1 to 3.5 | `pom.xml:24`; compatibility matrix | Bump to 3.10.0; the only compile break is `.refresh(true)` to `.refresh(Refresh.True)` at `QuickSearchOpenSearchClient.java:427` (verified: 233 tests pass after) | C2 |
|
||||
| B14 | High | Startup is best-effort: an unreachable cluster leaves `client=null`, the host boots healthy, every write logs and every search throws; connect and index creation are side effects inside metadata production | `QuickSearchQBitProducer.java:159-195` | Move connect, `ensureIndexExists` and close into a `QRuntimeServiceInterface` (4.1) with `startupMode` `FAIL_FAST` or `DEGRADED`; qqq-all's `FullQBitsTest` currently relies on degraded behaviour, so the default is a decision (PRD D11) | F16, G05, 01 F-09, F-10 |
|
||||
| B15 | Medium | No connect/response timeouts or pool sizing exposed; opensearch-java defaults are 1 s connect and 30 s response; a hung node blocks every host write | `QuickSearchOpenSearchClient.java:105-139` | `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `User-Agent`; multi-host list | C9, A7, G01 |
|
||||
| B16 | Medium | Processes, operational tables and the admin app declare no `QPermissionRules`, so they inherit the instance default (`NOT_PROTECTED` unless the host tightened it); anyone can trigger a full reindex | `QuickSearchQBitProducer.java:457-549` | `HAS_ACCESS_PERMISSION` on processes and app with base name `quickSearchAdmin`; `READ_WRITE_PERMISSIONS` on tables; overridable | A6 |
|
||||
| B17 | Medium | `limit` and `offset` unbounded; `from + size > 10000` fails with a raw error; `totalHits` silently capped at 10000; no term length bounds | `QuickSearchAction.java:67-79, 123-124` | Clamp limit (100) and window (10000), bound term length (2 to 100) matching core, `track_total_hits` bounded with a lower-bound flag | F11, C7, A11, G18, G24 |
|
||||
| B18 | Medium | `OpenSearchException` is unchecked and bypasses every `catch(IOException)`, so search callers get a raw RuntimeException | `QuickSearchOpenSearchClient.java` all client methods; `QuickSearchAction.java:91` | Catch `OpenSearchException` and wrap in `QException` | F10 |
|
||||
| B19 | Medium | `quickSearchIndex.tableName` has no unique key, so concurrent first runs on two replicas create duplicate rows and the table is processed twice per cycle | `AbstractIndexingStep.java:143-195`; `QuickSearchQBitProducer.java:457-475` | `withUniqueKey(new UniqueKey("tableName"))` plus DDL note | F14 |
|
||||
| B20 | Medium | Per-table basepull failures are logged at WARN and the process reports success forever | `BasepullIndexStep.java:124-132` | Aggregate failures and fail the step or set a FAILED status | F15 |
|
||||
| B21 | Medium | `quickSearchIndex.enabled` is honoured only by basepull; customizers, full reindex, reconcile and search ignore it | `BasepullIndexStep.java:89-92` only | Consult the row (cached) in all paths; exclude disabled tables from search | F12, G07 |
|
||||
| B22 | Medium | Integration tests pin OpenSearch 2.11.0 (2023) and search one whole word; nothing covers 2.19.x or 3.x, typeahead, multi-word, boosts, basepull, scheduling or permissions; JaCoCo check is skipped (no `prepare-agent`) | `*IntegrationTest.java:76-85`; `pom.xml:188-212` | Parameterize the image, run 2.19.6 and 3.9.0 in CI, bind the JaCoCo agent, add the tests listed in reports 04 and 05 | C10, G32 |
|
||||
| B23 | Medium | No orphan sweep for tables removed from config or renamed; no detection that config or mapping drifted from the live index (`ensureIndexExists` returns early if the index exists) | `ReconcileIndexStep.java:98`; `QuickSearchOpenSearchClient.java:167-171`; `AbstractIndexingStep.java:167-175` | Reconcile deletes `sourceTable` not in the discovered set; compare config to `searchableFieldsJson` and set `NEEDS_REINDEX`; verify analyzer settings on an existing index | G14, G15 |
|
||||
| B24 | Medium | Validation never checks that searchable fields or the basepull timestamp field exist on the table; tables without `modifyDate` fail at run time, caught and logged | `QuickSearchQBitConfig.java:73-157` | Fail fast at produce; allow `basepullTimestampField = null` meaning reconcile-only | G13 |
|
||||
| B25 | Medium | Processes have no input screen, so `tableName` cannot be supplied from the admin UI; `enableScheduledProcesses=false` also removes the manual full reindex and reconcile | `QuickSearchQBitProducer.java:228-232, 510-549` | Frontend step with an optional table PVS; split the flag | G17, G20 |
|
||||
|
||||
Also from report 01: drop the local JUnit 5.10.0 pins in favour of the parent's `junit-bom` 6.0.1 (01 F-13); add `getDefaultBackendNameForTables()` and a settable `tableMetaDataCustomizer` on the config so hosts can set RDBMS backend details the way qqq-all does for the other QBits (01 answer 1); use the `QBitMetaData.namespace` as the default `tableNamePrefix` (01 F-12).
|
||||
|
||||
Non-blocking but recommended before 1.0: external document versioning from `modifyDate` so stale writes are rejected (F05); failed-event replay plus bounded retry on bulk 429/5xx (G06); byte-size cap on bulk batches for the AWS 10 MiB payload limit on small instances (C8); close the client and publisher on shutdown and on re-produce (A8, F17); `IndexEvent` needs `occurredAt` and a version if the publisher SPI is to be called stable (G26); run-history purge (G21); admin table fields `lastRunStatus`, `lastErrorMessage`, `documentCount`, `lastReconcileTime` (G22); health method (G23); per-field `maxLength` (G25); mark `indexEventPublisher` transient on the Serializable config (A10).
|
||||
|
||||
## 3. OpenSearch and AWS compatibility matrix
|
||||
|
||||
| Target | Works today | What breaks | Needed |
|
||||
|---|---|---|---|
|
||||
| OpenSearch 2.x self-managed, security disabled | Yes | Nothing; the only tested configuration (image 2.11.0) | Test 2.19.x too |
|
||||
| OpenSearch 2.x self-managed, security plugin, basic auth over TLS | Partly | Demo or private-CA certificates fail the JVM trust check; basic auth only | B11; documented least-privilege role |
|
||||
| OpenSearch 3.x self-managed | No (client line) | opensearch-java 2.x is documented for 1.x to 2.x; server negotiates HTTP/2 | B13; add 3.x to the IT matrix |
|
||||
| AWS managed domain 2.x or 3.x, FGAC, internal-database master user, basic auth | Yes for 2.x, 3.x after B13 | Needs port 443, `useSsl`, and an access policy that does not require signed requests; 10 MiB payload limit on t3 and `*.large` | B13, byte-capped bulks, README guidance |
|
||||
| AWS managed domain with IAM master user or IAM principals in the access policy (SigV4) | No | Unsigned requests rejected; no `AwsSdk2Transport`, region or service name | `authMode=AWS_SIGV4` with `AwsSdk2Transport`, service `es`, default credential chain, optional assume-role; declare `software.amazon.awssdk:aws-crt-client` and `auth` as optional dependencies (decision D6) |
|
||||
| Amazon OpenSearch Serverless | No | SigV4 service `aoss` plus `x-amz-content-sha256`; `_delete_by_query` and `_refresh` are not in the AOSS supported-operations table (used by full reindex and reconcile); custom `_id` only on Search collections; refresh lag about 10 s; client must be 3.x | Declare unsupported for 1.0; later: SigV4, replace delete-by-query with search plus bulk delete, replace refresh with a bounded wait |
|
||||
| Legacy Elasticsearch 7.10 domains, OpenSearch 1.x | Untested | Client matrix lists only OpenSearch 1.x to 3.x; renamed APIs | Out of scope |
|
||||
|
||||
Recommended 1.0 supported-platforms statement: self-managed OpenSearch 2.19.x and 3.x latest minor over HTTP or HTTPS with the security plugin disabled or HTTP basic auth; Amazon OpenSearch Service managed domains 2.x and 3.x with fine-grained access control, an internal-database master user, basic auth, port 443 and TLS. Not supported in 1.0: IAM SigV4 (unless D6 says otherwise), Serverless, Elasticsearch endpoints, OpenSearch 1.x. Least-privilege role on the one index: `create_index`, `read`, `write`, `delete`, plus `indices:admin/refresh*` and an `indices:admin/*` read for the exists check, and the cluster-level `cluster_composite_ops` group for `_bulk`. Sources are listed in report 02.
|
||||
|
||||
## 4. Authentication options that make sense for 1.0
|
||||
|
||||
Today: none, or HTTP basic. Recommended minimum: an `authMode` enum with `NONE`, `BASIC`, `AWS_SIGV4` (`awsRegion`, `awsServiceName` default `es`, optional `awsAssumeRoleArn`), plus one escape hatch, a `transportCustomizer` `QCodeReference` to an interface with `customizeTransport(ApacheHttpClient5TransportBuilder)` and `customizeHttpClient(HttpAsyncClientBuilder)`, which lets hosts add bearer or JWT tokens, API-key headers and mTLS without a fork. Do not add `BEARER`, `API_KEY` or `MTLS` enum values in 1.0; each needs a token-refresh or key-store story the SPI defers. All secret-bearing strings are resolved with `QMetaDataVariableInterpreter` inside the client factory so hosts write `"${env.OPENSEARCH_PASSWORD}"` and metadata never holds the value. Validation: BASIC needs both username and password; AWS_SIGV4 needs a resolvable region and forbids username/password; credentials over http rejected unless `allowPlaintextCredentials`; `insecureSkipVerify` refused for non-loopback hosts unless a second explicit flag is set, with a WARN at startup. Full field sketch with defaults is in report 03, section "Recommended 1.0 auth configuration". The deprecation path keeps `withOpensearchHost/Port/UseSsl/Username/Password` working and infers `BASIC` when a username is set.
|
||||
|
||||
Supply chain: no concrete CVE in the resolved set (httpclient5 5.6.3, jackson 2.21.5, log4j 2.25.5). qqq-backend-core still pulls AWS SDK for Java v1 1.12.797, which reached end of support 2025-12-31; a qqq concern, and this QBit should use SDK v2 only.
|
||||
|
||||
## 5. Feature completeness versus QQQ 4.1 core search
|
||||
|
||||
QQQ 4.1 added `RecordSearchAction`, `QTableMetaData.searchFields` and `POST /qqq/v1/search`, consumed by the Next UI. Core has: table READ permission and capability checks, field personalisation (hidden and masked fields excluded), record locks via `QueryAction`, `limitPerTable` (5 default, 25 max), a `tableNames` list, a 100-character term cap, a 30 s timeout, `tableLabel`, typed `recordId`, label fallback and display values. Quick Search has: relevance scoring with weights, typeahead (defective, B7), highlights, offset pagination with totals, cross-table ranking, OLTP isolation, run history and the publisher SPI. To back the Next UI (PRD E4) Quick Search must first accept `tableNames` and `limitPerTable` and return `tableLabel` and a typed `recordId` (G16), and it should default its field list from `searchFields` so hosts do not maintain two lists. Core has no provider SPI; see decision D1.
|
||||
|
||||
## 6. Consumers (verified)
|
||||
|
||||
- qqq-all `DemoQBits.java:151-163` (origin/develop): config-driven `customer` with `name` and `email`, hand-registered `QBitMetaData` at `0.2.1-SNAPSHOT`, no `schedulerName`, `OPENSEARCH_SSL` defaults false. Its `FullQBitsTest` casts the config back out of `QInstance.getQBits()`.
|
||||
- Voyage `pom.xml:249-253` (origin/develop): pins `qbit-quick-search:0.2.0` from GitHub Packages and produces the QBit in `OrderAppMetaDataProvider.produceQuickSearchQBit` for `crmContact` and `crmCompany` when `OPENSEARCH_HOST` is set. Report 04's statement that Voyage has no reference was based on a stale local checkout and is wrong.
|
||||
|
||||
## 7. Missing tests (consolidated)
|
||||
|
||||
Reports 04 and 05 list them by class and scenario. The themes: a transaction test against H2 through `qqq-backend-module-rdbms` (insert, roll back, assert no publish); inverted customizer tests (publisher throws, records carry a warning or a failed-event row exists); keyset paging assertions on every source query; watermark captured before querying with an overlap; stale-write rejection once versioning lands; "stella" does not match "stone"; integer and date searchable fields do not break search; annotation tables have a record label; full-reindex FAILED run has an error message; `OpenSearchException` wrapped; limit and window clamps; disabled rows skipped everywhere; unique key on `tableName`; one failing table fails the basepull step; produce twice closes the previous client; recordLabelFormat with a null numeric; PVS label indexing; permission denial; string primary keys through reconcile; OpenSearch down during a host insert; pre-existing index with a different mapping; two tables sharing a field name with different types; image matrix 2.19.6 and 3.9.0.
|
||||
@@ -0,0 +1,174 @@
|
||||
---
|
||||
tracker_key: "QRun-IO/qbit-quick-search#12"
|
||||
project: qbit-quick-search
|
||||
status: released-rc
|
||||
updated: 2026-10-06
|
||||
---
|
||||
|
||||
# Initiative: QBit Quick Search 1.0 on QQQ 4.1
|
||||
|
||||
> Stakeholder-readable PRD. This markdown is canonical; the tracker mirrors it. Scope claims are grounded in the code at `develop` (`000e5bb`) and in QQQ `v4.1.0-RC.1`, and in the deep-dive audit of 2026-10-04 (`docs/audits/2026-10-04/README.md`, blockers B1 to B25).
|
||||
|
||||
## Status
|
||||
|
||||
`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the review follow-ups #13 to #17, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
|
||||
|
||||
## Executive Summary
|
||||
|
||||
QBit Quick Search gives QQQ applications OpenSearch-backed full-text search across tables. It is at 0.2.1-SNAPSHOT, published publicly as 0.2.0, pinned by Voyage and bundled by `qqq-all`. The July 2026 audit found the index could not be trusted to converge with the source database. PR #7 (merged 2026-09-26) fixed update clobbering, silent bulk failures and added a reconcile process. The 2026-10-04 deep-dive confirmed the QBit builds and passes all tests on QQQ 4.1.0-RC.1 unchanged, and found 25 remaining blockers across trust, security, search quality, platform compatibility and operability.
|
||||
|
||||
1.0 means three things. First, a standard QQQ 4.1 QBit: built on the 4.1 build parent, registered through `QBitMetaDataProducer`, scheduled through QQQ's scheduler, composable with other QBits on the same tables, fail-fast at startup. Second, a trustworthy index: no uncommitted or lost writes, no silent failures, deterministic batch jobs, a reindex that keeps serving results, and search that honours QQQ permissions. Third, a credible platform story: current OpenSearch client, self-managed 2.x and 3.x plus Amazon OpenSearch Service managed domains, TLS trust configuration, secrets by reference, and an authentication model that covers basic auth and, if decision D6 says so, AWS IAM SigV4.
|
||||
|
||||
The 4.1 release train does not wait for 1.0. A compatibility-only 0.3.0 rides the proposed `qbit-build-parent` 2.1.0 chain (qqq#921). 1.0 follows on the same parent once the epics below land.
|
||||
|
||||
## Business Objective
|
||||
|
||||
Ship a search QBit that QQQ adopters (qqq-all demo users, Voyage, future Business Platform tenants) can run in production without a reconciliation babysitter, against the OpenSearch deployments they actually have, and that the Next UI can expose directly.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
Current state at `develop` as of 2026-10-04:
|
||||
|
||||
| Area | State | Evidence |
|
||||
|---|---|---|
|
||||
| Build | Parent `qbit-build-parent:2.0.0` (qqq 4.0.0). Opt-in `-Pqqq-snapshot` profile. | `pom.xml` |
|
||||
| 4.1 compatibility | 233 unit tests pass on 4.0.0, 4.1.0-SNAPSHOT and 4.1.0-RC.1; `mvn verify` on RC.1 with Docker passes all 12 integration tests. No source change needed. | surefire and failsafe reports |
|
||||
| Coverage gate | JaCoCo `check` is skipped (no `prepare-agent`), so the 0.70 / 0.90 thresholds never run. | `mvn verify` output |
|
||||
| Open PRs | #8 (orb 0.6.8, tag-only GA publish, Apache checkstyle, AssertJ 3.27.7): CI green, awaiting review. #1 superseded by #8. | GitHub |
|
||||
| Release chain | QQQ 4.1.0-RC.1 published 2026-10-02. Parent 2.1.0-RC.1 and quick-search 0.3.0-RC.1 proposed in qqq#921; qqq-all BOM pins 0.2.1-SNAPSHOT. | qqq#921, qbit-bom#4, qqq-all#10 |
|
||||
| Index trust | Real-time indexing runs inside the host transaction before commit (B1); failures are swallowed with no record (B2); basepull pages without order-by (B4) and loses rows modified during a run (B5); basepull and reconcile are never scheduled (B6); full reindex wipes first (B10); no unique key on `quickSearchIndex.tableName` (B19). | audit B1, B2, B4, B5, B6, B10, B19 |
|
||||
| Search quality | Query terms are edge-ngrammed because no search analyzer is set, and multi-match defaults to OR (B7); shared dynamic `fieldValues` object breaks on mixed field types and non-lenient boosts (B8); record labels null for annotation tables and PVS ids indexed instead of labels (B9); unbounded paging and term length (B17). | audit B7, B8, B9, B17 |
|
||||
| Security | Search has no permission or session checks (B3); no permission rules on processes, tables or app (B16); credentials allowed over plain HTTP and no TLS trust configuration (B11); password stored on serializable metadata (B12). | audit B3, B11, B12, B16 |
|
||||
| Platform | opensearch-java 2.10.0 is the end-of-life 2.x line (B13); IAM SigV4 and Serverless unsupported; no timeouts (B15); integration tests pin OpenSearch 2.11.0 only (B22). | audit section 3 |
|
||||
| QBit contract | Plain producer, no `QBitMetaData` (qqq-all hand-registers with a hard-coded version), no `sourceQBitName`, static context, `withCustomizer` throws on occupied slots, best-effort startup (B14). | `QuickSearchQBitProducer`, qqq-all `DemoQBits.java:161` |
|
||||
| Core overlap | QQQ 4.1 `RecordSearchAction`, `QTableMetaData.searchFields` and `POST /qqq/v1/search` back the Next UI with permissions and no provider SPI; Quick Search has no UI surface and cannot yet satisfy `tableNames` or `limitPerTable`. | qqq commit `7dd36c8ad`; audit section 5 |
|
||||
| Docs | README shows a `0.2.0-SNAPSHOT` dependency, promises a scheduled basepull, admin-UI per-table reindex, "field label" prefixes, enforced coverage and qbit-crm/wms integrations, none of which hold. | `README.md` |
|
||||
|
||||
Doing nothing leaves qqq-all and Voyage shipping a search feature whose basepull never runs, whose index can contain rolled-back and lost writes, whose results leak across tables regardless of permissions, and whose reindex blanks search.
|
||||
|
||||
## Vision / Desired Future State
|
||||
|
||||
A host adds the dependency, configures the QBit like any other 4.1 QBit with secrets by reference, points it at a self-managed or AWS-managed OpenSearch cluster over TLS, and gets: real-time indexing that reflects committed data and survives slot collisions, a scheduled catch-up and reconcile with a replayed failed-event log, a reindex that keeps serving results, a search API that returns only what the session may read, and a Next UI search box backed by OpenSearch when the QBit is present.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
|
||||
- Released 4.1 build parent; CI matrix on the 4.1 GA line plus the next snapshot; Docker-backed integration tests on OpenSearch 2.19.x and 3.x; JaCoCo gate actually enforced.
|
||||
- Standard QBit contract: `QBitMetaDataProducer`, self-registered `QBitMetaData` with the real artifact version, namespace support, `sourceQBitName`, fail-fast validation including field existence, explicit permission rules, instance-scoped runtime state, composable customizers.
|
||||
- Index trust: post-commit indexing or a documented contract, failed-event capture and replay, keyset paging everywhere, start-anchored basepull watermark with overlap, scheduling through `QScheduleMetaData`, blackout-free full reindex, unique key on the index table, orphan and drift detection, external document versioning.
|
||||
- Search quality and safety: search analyzer, AND semantics, typed-safe `fieldValues` mapping, display values and labels, permission-aware results, bounded paging and term length, wrapped OpenSearch exceptions, `tableNames` and `limitPerTable` inputs, `tableLabel` and typed `recordId` outputs.
|
||||
- Connection security: opensearch-java 3.x, `opensearchUrl`, TLS trust block, `authMode` (`NONE`, `BASIC`, and `AWS_SIGV4` per D6), secrets by `${env.}` reference, redaction, timeouts and pool sizing, transport customizer SPI, client lifecycle.
|
||||
- Operability and docs: process input screens, admin table status fields, run-history purge, health method, truthful README with a supported-platforms statement and least-privilege roles, configuration reference, migration notes from 0.x, CHANGELOG.
|
||||
|
||||
### Out of Scope
|
||||
|
||||
- Asynchronous publisher implementations (RabbitMQ, Kafka, ESB). The `IndexEventPublisher` SPI stays and gets `occurredAt` and a version so an async implementation can be correct, but it is marked experimental in 1.0.
|
||||
- Amazon OpenSearch Serverless (needs SigV4 `aoss`, and lacks `_delete_by_query` and `_refresh`). Legacy Elasticsearch endpoints and OpenSearch 1.x.
|
||||
- Bearer, JWT, API-key and mTLS as first-class `authMode` values (available through the transport customizer SPI).
|
||||
- Record-level security beyond the `QueryAction` post-filter (D3). Index-time security keys.
|
||||
- Multi-index layouts, composite primary keys, synonyms, fuzzy matching, phrase queries, language analyzers, sort options.
|
||||
- `com.kingsrook` to `io.qrun` groupId migration (ADR-0007). qbit-crm / qbit-wms companion integrations.
|
||||
- Model-assisted table routing or re-ranking (TypeSafe Jev); candidates in `docs/reference/typesafe-jev-candidates.md` for 1.x.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- `com.kingsrook.qbits:qbit-quick-search:1.0.0` on Maven Central, parent importing the released qqq 4.1 BOM, Apache-2.0 metadata, opensearch-java 3.x.
|
||||
- CI runs the Testcontainers suite on OpenSearch 2.19.x and 3.x latest minor with no silent skips, on 4.1 GA and the next snapshot; JaCoCo thresholds enforced.
|
||||
- qqq-all `full` profile and Voyage pin 1.0.0; qqq-all's smoke finds a seeded customer through the Next UI search box (or through the API if D1's SPI slips).
|
||||
- A session without read permission on a table gets zero results from that table; a session with a record security lock sees only its rows. Both covered by tests.
|
||||
- A rolled-back insert produces no document (or, under the documented alternative, is removed by the next scheduled reconcile). Covered by an H2 transaction test.
|
||||
- Full reindex of a seeded table returns non-empty results at every point during the run. Covered by an integration test.
|
||||
- Basepull and reconcile fire on schedule in a host with a configured scheduler; a row modified during a basepull run is indexed by the next run. Covered by tests.
|
||||
- "stella" does not match "stone"; a table with an integer and a date searchable field searches without error. Covered by integration tests.
|
||||
- Basic auth over TLS against a cluster with a private CA works with the `tls` block; credentials over plain HTTP are rejected unless explicitly allowed. Covered by a secured Testcontainers test.
|
||||
- Every blocker B1 to B25 in the 2026-10-04 audit is closed or explicitly deferred with a reason in this doc.
|
||||
|
||||
## Key Stakeholders
|
||||
|
||||
| Role | Person | Responsibility |
|
||||
|---|---|---|
|
||||
| Sponsor / Product Owner | James Maes | Decisions D1 to D11, release approval |
|
||||
| Engineering Lead | James Maes (with Claude Code) | Build, review, release |
|
||||
| Downstream consumers | qqq-all (QRun-IO/qqq-all#10); Voyage (pins 0.2.0 from GitHub Packages, `OrderAppMetaDataProvider.produceQuickSearchQBit`) | Pin and verify 1.0.0 |
|
||||
|
||||
## Initiative-level Constraints
|
||||
|
||||
- Parent `qbit-build-parent` is the only source of the qqq version (ADR-0007). No child BOM imports outside the opt-in snapshot profile. httpclient5 and Jackson stay BOM-managed.
|
||||
- Apache-2.0 everywhere; Central artifacts are immutable, so metadata fixes ship as new versions.
|
||||
- 3-space indentation, flower-box Javadoc, fluent `withX()` setters, wrapper types (CODE_STYLE.md).
|
||||
- JaCoCo gates stay at 0.70 instruction / 0.90 class, and must actually run.
|
||||
- Source compatibility for the fluent config setters and `QuickSearchAction` input/output used by qqq-all and Voyage (`withOpensearchHost/Port/UseSsl/Username/Password`, `withSearchableTable`, `withSearchableEntityClasses`, `QuickSearchQBitProducer().withConfig().produce(qInstance)`). New fields are additive; deprecated fields keep working through 1.x.
|
||||
- No secrets in metadata objects, logs, or the frontend `environmentValues` (never name an OpenSearch secret `QQQ_ENV_*`).
|
||||
- Any mapping change (search analyzer, `fieldValues` templates) lands before the first 1.0 index is created, with an index-version marker so an existing 0.x index is detected and rebuilt rather than reused.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- QQQ 4.1.0 GA and `qbit-build-parent` 2.1.0 publish before the 1.0 release; work proceeds on `4.1.0-RC.1` via the snapshot profile.
|
||||
- OpenSearch 2.19.x and 3.x are the supported server lines; opensearch-java 3.10.0 is the client (its only compile break here is `.refresh(true)` to `.refresh(Refresh.True)`).
|
||||
- Hosts that want scheduling already configure a QQQ scheduler; the QBit does not provide one.
|
||||
- QQQ 4.1 provides `RecordChangeListenerInterface`, `QBackendTransaction.addAfterCommitCallback` and `QRuntimeServiceInterface`, none of which exist in 4.0.0 (verified by tag). 1.0 uses them and therefore requires qqq 4.1 (D10).
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Mitigation |
|
||||
|---|---|
|
||||
| Core `RecordSearchAction` has no SPI, so Next UI integration needs a qqq core change | Raise the SPI as a qqq issue early (D1); 1.0 can ship API-only, UI integration in 1.1 |
|
||||
| Adopting 4.1-only APIs drops 4.0 hosts | Voyage and qqq-all both move to 4.1; 0.3.0 remains the last 4.0-compatible line (D10) |
|
||||
| After-commit publishing depends on the backend transaction honouring the callback contract | RDBMS does at the 4.1 tag; verify Mongo and other backends in the integration matrix; flush immediately when no transaction is passed; scheduled reconcile is the backstop |
|
||||
| Removing the static context changes how custom publishers and process steps obtain the client | Keep static accessors as deprecated delegates to the instance-scoped holder for one minor version |
|
||||
| Permission post-filtering reduces page sizes unpredictably | Over-fetch (limit times two, capped) and filter down; document that `totalHits` is an upper bound |
|
||||
| Alias-swap reindex needs index-create and alias privileges on the cluster | Fall back to in-place reconcile when the configured user lacks the privilege; document required roles |
|
||||
| opensearch-java 3.x bump changes transport behaviour against 2.x servers (HTTP/2 negotiation, URL encoding) | CI matrix on 2.19.6 and 3.9.0; keep the content-compression workaround until proven unnecessary |
|
||||
| SigV4 pulls AWS SDK v2 artifacts into a QBit | Declare them `optional`; load the AWS transport reflectively or in a separate class only when `authMode=AWS_SIGV4` |
|
||||
|
||||
## Dependencies
|
||||
|
||||
- qqq 4.1.0 GA (qqq#798, #921), `qbit-build-parent` 2.1.0 (qbit-bom#4), qqq-orb 0.6.8 (PR #8).
|
||||
- CircleCI executor with Docker for the integration suite (orb capability to confirm).
|
||||
- qqq core change for a record-search provider SPI (new issue, D1).
|
||||
- Optional: `software.amazon.awssdk:aws-crt-client` and `auth` for SigV4 (D6).
|
||||
|
||||
## Proposed Epics
|
||||
|
||||
Estimates are engineer-days with AI-augmented throughput. E1 is foundational and gets broken into stories first. Blocker IDs reference `docs/audits/2026-10-04/README.md`.
|
||||
|
||||
### Epic 1: 4.1 platform and QBit contract (foundational, ~7 days)
|
||||
Purpose: make the QBit a standard 4.1 citizen, following the ordered migration plan in audit report 01. `QBitMetaDataProducer<QuickSearchQBitConfig>` with `getQBitMetaData()` (groupId `com.kingsrook.qbits`, artifactId `quick-search` which qqq-all asserts, version from a Maven-filtered resource, namespace, config); tables, processes and app as package-scanned component producers or `@QMetaDataProducingEntity`; registration and bootstrap in `postProduceActions`; `withConfig()` and a `produce(QInstance)` override that adds to the instance kept as deprecated shims so qqq-all and Voyage compile unchanged; `isEnabled()` false when the config is null; `getDefaultBackendNameForTables()` and a settable `tableMetaDataCustomizer`; namespace as default `tableNamePrefix`. Explicit `QPermissionRules` with config override (B16). Null-safe list-based validation covering backend, field existence, timestamp field and OpenSearch naming rules, plus a `QInstanceValidatorPluginInterface` for late checks (B24). `QuickSearchRuntimeService implements QRuntimeServiceInterface` owning connect, `ensureIndexExists` and close, with `startupMode` per D11 (B14); config resolved from `SourceQBitAware.getSourceQBitConfig()` in steps and the listener, `QuickSearchQBitContext` kept as a deprecated facade for one release. Unique key on `quickSearchIndex.tableName` (B19); split `enableScheduledProcesses` into basepull and maintenance flags with the old name as a deprecated alias (B25). Build: JaCoCo `prepare-agent` with an explicit version, drop the JUnit 5.10.0 pins for the parent's `junit-bom` 6.0.1, parent 2.1.x re-pin when released; CI matrix (GA plus next snapshot), Docker-backed integration tests on 2.19.x and 3.x (B22).
|
||||
|
||||
### Epic 2: Index trustworthiness (~7 days)
|
||||
Purpose: finish the convergence story. Replace the three customizers with one `QuickSearchRecordChangeListener implements RecordChangeListenerInterface` (`appliesTo` only discovered tables; UPDATE keeps the full-record re-read through `event.getTransaction()`), registered with `qInstance.withRecordChangeListener`, publishing through `QBackendTransaction.addAfterCommitCallback` when a transaction is present and immediately otherwise (B1, D7); the listener cannot fail the write, so record failures in a `quickSearchFailedEvent` table that the scheduled run replays, with bounded retry on bulk 429/5xx and failure counts on `quickSearchIndex` (B2, D8); keyset paging in basepull and full reindex reusing the reconcile loop (B4); start-anchored watermark with configurable overlap and greater-than-or-equal (B5); `QScheduleMetaData` on basepull and reconcile with `schedulerName` and per-table interval (B6); full reindex into a fresh index with alias swap and in-place fallback, `conflicts(Proceed)`, error message on FAILED runs (B10); aggregate per-table failures into the step result (B20); honour `enabled` in every path (B21); orphan sweep for removed or renamed tables, config and mapping drift detection with a `NEEDS_REINDEX` status and an index-version marker (B23); external document versioning from `modifyDate`; run-history purge; `IndexEvent.occurredAt` and version.
|
||||
|
||||
### Epic 3: Search quality and permission-aware search (~5 days)
|
||||
Purpose: make results correct and safe. Standard+lowercase `search_analyzer`, `operator=AND` or `minimum_should_match`, boosts that apply to prefix matches (B7); `dynamic_templates` for `fieldValues.*`, `date_detection: false`, stringified values, `lenient(true)` (B8); display values and PVS labels at index time, `QValueFormatter.formatRecordLabel` fallback, per-field `maxLength` (B9); table-level READ permission filter mirroring core, `terms` filter on `sourceTable`, `QueryAction` post-filter by primary key for record locks per D3, hidden and masked fields excluded at index time (B3); bounded `limit`, window and term length, `track_total_hits` with a lower-bound flag (B17); `OpenSearchException` wrapped in `QException` (B18); disabled tables excluded from search (B21).
|
||||
|
||||
### Epic 4: Search surface and UI integration (~4 days, partly in qqq core)
|
||||
Purpose: make search usable from the UI. Input parity with core: `tableNames`, `limitPerTable`, `inputSource`; output parity: `tableLabel`, typed `recordId`; default the field list from `QTableMetaData.searchFields` when present; propose and implement a provider SPI in qqq core so `RecordSearchAction` delegates to a registered provider (D1); ship `QuickSearchRecordSearchProvider`; keep `QuickSearchAction` as the richer backend API (scores, highlights, pagination).
|
||||
|
||||
### Epic 5: Connection security and authentication (~5 days)
|
||||
Purpose: run against real clusters safely. opensearch-java 3.10.0 bump with the `Refresh.True` fix and content-compression workaround kept (B13); `opensearchUrl` and optional host list, https default; `tls` block (CA path, trust store, key store for mTLS, hostname verification, guarded `insecureSkipVerify`) (B11); `authMode` enum `NONE`, `BASIC`, `AWS_SIGV4` with `awsRegion`, `awsServiceName`, `awsAssumeRoleArn` and optional AWS SDK v2 dependencies per D6; `transportCustomizer` `QCodeReference` SPI for bearer, API key and mTLS; `${env.}` interpretation of secrets via `QMetaDataVariableInterpreter` at client construction, `@JsonIgnore` secret getters, redacting `toString()`, `transient` publisher (B12); `allowPlaintextCredentials` gate; `connectTimeoutMillis`, `responseTimeoutMillis`, `maxConnections`, `User-Agent`, byte-capped bulk batches (B15); validation rules and the deprecation path for the old setters; a secured Testcontainers test (`withSecurityEnabled`).
|
||||
|
||||
### Epic 6: Operations, docs and release (~4 days)
|
||||
Purpose: ship 1.0 cleanly. Process input screens with a table PVS and summary screen (B25); admin table fields `lastRunStatus`, `lastErrorMessage`, `lastReconcileTime`, `documentCount`, read-only tracking fields, PVS from run to index, sections and labels; health method surfaced in the admin table; README rewrite (truthful feature list, dependency snippet, supported-platforms statement, least-privilege roles for the security plugin and AWS FGAC, required cluster privileges for alias swap, `QQQ_ENV_*` warning, SPI stability statement); configuration reference; 0.x to 1.0 migration notes including the forced reindex; CHANGELOG and release notes; 0.3.0 compatibility release on parent 2.1.0 first, then 1.0.0; qqq-all and Voyage pin bumps.
|
||||
|
||||
## Decisions for the sponsor
|
||||
|
||||
Each has a recommendation. Resolve one at a time; answers get folded into tickets as facts.
|
||||
|
||||
| ID | Question | Recommendation |
|
||||
|---|---|---|
|
||||
| D1 | Integrate with core `RecordSearchAction` via a new qqq SPI, or keep Quick Search API-only? | Propose the SPI in qqq now; ship 1.0 API-only if it slips, UI in 1.1 |
|
||||
| D2 | Version plan: does the qqq#921 chain ship 0.3.0 (compat only) and 1.0 follows, or does 4.1 wait for 1.0? | 0.3.0 on the RC chain, 1.0 after E1 to E6; do not gate 4.1 on 1.0 |
|
||||
| D3 | Permission depth for 1.0: table-level read permission only, or record-level via `QueryAction` post-filter? | Table-level plus `QueryAction` post-filter by primary key (reuses locks); no indexed security keys in 1.0 |
|
||||
| D4 | Scheduling default: schedule automatically when a scheduler exists, or require `schedulerName`? | Require `schedulerName` when basepull is enabled; validation error otherwise so the README promise is enforced |
|
||||
| D5 | Full reindex strategy: alias swap (needs cluster privileges) or in-place reconcile semantics only? | Alias swap with documented privileges, in-place fallback when `createIndex` is denied |
|
||||
| D6 | AWS IAM SigV4 in 1.0 (optional AWS SDK v2 dependencies, `authMode=AWS_SIGV4`), or declare unsupported and ship basic-auth FGAC only? | Include it: the transport ships inside opensearch-java, the SDK artifacts are optional, and IAM is the default posture for AWS domains; Serverless stays out |
|
||||
| D7 | Pre-commit indexing (B1): use 4.1's `RecordChangeListenerInterface` plus `QBackendTransaction.addAfterCommitCallback`, or keep customizers and document pre-commit semantics with scheduled reconcile as the corrector? | Use the 4.1 listener and after-commit callback (reference: `qqq-esb` `EsbRecordChangeListener`); flush immediately when no transaction is passed; keep reconcile scheduled as the backstop; this makes 1.0 4.1-only (D10) |
|
||||
| D8 | Real-time failure contract (B2): surface as QQQ record warnings plus a failed-event table (host write succeeds), or fail the host write when OpenSearch is down? | Warnings plus failed-event table; never fail the host write for a search index |
|
||||
| D9 | Supported platforms statement for 1.0: self-managed 2.19.x and 3.x plus AWS managed domains (FGAC basic auth, and IAM if D6); Serverless, Elasticsearch and OpenSearch 1.x unsupported? | Yes, as stated; revisit Serverless in 1.x once `_delete_by_query` is removed from the design |
|
||||
| D10 | Does 1.0 require qqq 4.1 (adopting `RecordChangeListenerInterface`, after-commit callbacks and `QRuntimeServiceInterface`), or keep a 4.0 fallback path with customizers and `MultiCustomizer`? | 4.1-only; 0.3.0 is the last 4.0 line; delete the customizers in 1.0 rather than carrying two real-time paths |
|
||||
| D11 | `startupMode` default: `FAIL_FAST` (host refuses to boot when OpenSearch is unreachable) or `DEGRADED` (current behaviour, which qqq-all's `FullQBitsTest` relies on with OpenSearch at `127.0.0.1:1`)? | `FAIL_FAST` by default for production safety; qqq-all's full profile sets `DEGRADED` explicitly in its follow-up PR, and `DEGRADED` surfaces a health notice |
|
||||
|
||||
## Definition of Done
|
||||
|
||||
All six epics complete; success criteria met; 1.0.0 published on Central and pinned by qqq-all and Voyage; README, configuration reference, migration notes and CHANGELOG accurate; every blocker in `docs/audits/2026-10-04/README.md` closed or deferred with a reason recorded here and in the second-brain project note.
|
||||
@@ -0,0 +1,36 @@
|
||||
# TypeSafe (Jev) candidates for Quick Search
|
||||
|
||||
Captured 2026-10-04. Not in 1.0 scope (see `docs/initiatives/quick-search-1.0.md`); a possible 1.x epic or companion extension. Nothing here is implemented.
|
||||
|
||||
Jev is TypeSafe's System One model: it returns typed answers (Choice, Score, Noul probability) over a `state` payload via `POST https://api.typesafe.ai/v1/systemone`. There is no Java SDK; the HTTP API is a single JSON POST, and the project already has httpclient5 and Jackson. Docs: https://docs.typesafe.ai/llms.txt
|
||||
|
||||
## How the project judges things today
|
||||
|
||||
| Concern | Current mechanism | Code |
|
||||
|---|---|---|
|
||||
| Ranking | BM25 over edge-ngram `searchableText` plus per-field `should` boosts with hand-set weights | `QuickSearchOpenSearchClient.search`, `buildFieldBoostQueries` |
|
||||
| Routing | Optional `tableName` filter only; no logic decides which table a query is about | `QuickSearchInput`, `QuickSearchAction` |
|
||||
| Query classification | Lowercase and whitespace collapse | `IndexingUtils.normalizeSearchText` |
|
||||
| Index event routing | Deterministic INDEX / DELETE; no semantic step, no Jev fit | `IndexEventAction`, customizers, `ReconcileIndexStep` |
|
||||
|
||||
## Candidates, strongest first
|
||||
|
||||
1. **Table routing (one Choice).** State: the query plus a catalog of indexed tables (label, searchable field names). Question: which kind of record is this query looking for, one option per table plus "no specific table". Use the probability distribution as per-table boosts on `sourceTable`, or feed it to core `RecordSearchAction.tableNames` / `limitPerTable` once the Epic 4 provider exists. Confidence-gate: narrow only when confidence is high, otherwise search everything. Tiny state, cheap, fast. New capability, not a simplification.
|
||||
2. **Re-ranking the OpenSearch shortlist (one Noul per candidate).** Over-fetch about three pages, ask "is this record what the query is asking for" with the query and each candidate's `recordLabel` and `fieldValues` in state (all candidates in one request as parallel questions), sort by probability, cut to `limit`. Pattern: https://docs.typesafe.ai/cookbooks/rerank_typesafe.md. Costs: a network hop on the search hot path (not for per-keystroke typeahead), and record data leaving the cluster (needs enterprise zero-data-retention terms for CRM or health-adjacent data). Only as an opt-in reranker behind a flag, explicit searches, first page.
|
||||
3. **Highlight selection.** Choice over highlight fragment ids ("which snippet best shows why this matched") instead of joining all fragments with "...". Small win, same egress concern as item 2.
|
||||
4. **Offline weight suggestion.** Dev-time scoring of sample records per field for how identifying each value is, to suggest `weight`. Not runtime.
|
||||
|
||||
## Rejected
|
||||
|
||||
- Query-shape classification (email, phone, order number, name, free text) to switch exact keyword vs ngram matching. Regexes do this deterministically; Jev's own jaggedness notes say to keep literal pattern work in code.
|
||||
|
||||
## Integration notes
|
||||
|
||||
- Put any client behind an interface (`SearchRouter`, `ResultReranker`) so the core QBit stays OpenSearch-only and the integration is optional. Dependency or build changes need owner approval.
|
||||
- Jev 1.13 limits: 80 requests per second, 100K tokens per second, 32k tokens of state per question. Fine for routing; tight for reranking under load.
|
||||
- Pin a versioned model id (`jev-1.13.0`) once confidence thresholds are tuned; aliases move.
|
||||
- API key stays server-side in host configuration, never in the QBit's metadata.
|
||||
|
||||
## Recommendation
|
||||
|
||||
Prototype item 1 only, as a confidence-gated boost, and measure against the seeded qqq-all customer data before considering item 2.
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/bin/sh
|
||||
# ----------------------------------------------------------------------------
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Apache Maven Wrapper startup batch script, version 3.3.4
|
||||
#
|
||||
# Optional ENV vars
|
||||
# -----------------
|
||||
# JAVA_HOME - location of a JDK home dir, required when download maven via java source
|
||||
# MVNW_REPOURL - repo url base for downloading maven distribution
|
||||
# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
|
||||
# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
set -euf
|
||||
[ "${MVNW_VERBOSE-}" != debug ] || set -x
|
||||
|
||||
# OS specific support.
|
||||
native_path() { printf %s\\n "$1"; }
|
||||
case "$(uname)" in
|
||||
CYGWIN* | MINGW*)
|
||||
[ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
|
||||
native_path() { cygpath --path --windows "$1"; }
|
||||
;;
|
||||
esac
|
||||
|
||||
# set JAVACMD and JAVACCMD
|
||||
set_java_home() {
|
||||
# For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
|
||||
if [ -n "${JAVA_HOME-}" ]; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
JAVACCMD="$JAVA_HOME/jre/sh/javac"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
JAVACCMD="$JAVA_HOME/bin/javac"
|
||||
|
||||
if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
|
||||
echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
JAVACMD="$(
|
||||
'set' +e
|
||||
'unset' -f command 2>/dev/null
|
||||
'command' -v java
|
||||
)" || :
|
||||
JAVACCMD="$(
|
||||
'set' +e
|
||||
'unset' -f command 2>/dev/null
|
||||
'command' -v javac
|
||||
)" || :
|
||||
|
||||
if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
|
||||
echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# hash string like Java String::hashCode
|
||||
hash_string() {
|
||||
str="${1:-}" h=0
|
||||
while [ -n "$str" ]; do
|
||||
char="${str%"${str#?}"}"
|
||||
h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
|
||||
str="${str#?}"
|
||||
done
|
||||
printf %x\\n $h
|
||||
}
|
||||
|
||||
verbose() { :; }
|
||||
[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
|
||||
|
||||
die() {
|
||||
printf %s\\n "$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
trim() {
|
||||
# MWRAPPER-139:
|
||||
# Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
|
||||
# Needed for removing poorly interpreted newline sequences when running in more
|
||||
# exotic environments such as mingw bash on Windows.
|
||||
printf "%s" "${1}" | tr -d '[:space:]'
|
||||
}
|
||||
|
||||
scriptDir="$(dirname "$0")"
|
||||
scriptName="$(basename "$0")"
|
||||
|
||||
# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
|
||||
while IFS="=" read -r key value; do
|
||||
case "${key-}" in
|
||||
distributionUrl) distributionUrl=$(trim "${value-}") ;;
|
||||
distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
|
||||
esac
|
||||
done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
|
||||
case "${distributionUrl##*/}" in
|
||||
maven-mvnd-*bin.*)
|
||||
MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
|
||||
case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
|
||||
*AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
|
||||
:Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
|
||||
:Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
|
||||
:Linux*x86_64*) distributionPlatform=linux-amd64 ;;
|
||||
*)
|
||||
echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
|
||||
distributionPlatform=linux-amd64
|
||||
;;
|
||||
esac
|
||||
distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
|
||||
;;
|
||||
maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
|
||||
*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
|
||||
esac
|
||||
|
||||
# apply MVNW_REPOURL and calculate MAVEN_HOME
|
||||
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
|
||||
[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
|
||||
distributionUrlName="${distributionUrl##*/}"
|
||||
distributionUrlNameMain="${distributionUrlName%.*}"
|
||||
distributionUrlNameMain="${distributionUrlNameMain%-bin}"
|
||||
MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
|
||||
MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
|
||||
|
||||
exec_maven() {
|
||||
unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
|
||||
exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
|
||||
}
|
||||
|
||||
if [ -d "$MAVEN_HOME" ]; then
|
||||
verbose "found existing MAVEN_HOME at $MAVEN_HOME"
|
||||
exec_maven "$@"
|
||||
fi
|
||||
|
||||
case "${distributionUrl-}" in
|
||||
*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
|
||||
*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
|
||||
esac
|
||||
|
||||
# prepare tmp dir
|
||||
if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
|
||||
clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
|
||||
trap clean HUP INT TERM EXIT
|
||||
else
|
||||
die "cannot create temp dir"
|
||||
fi
|
||||
|
||||
mkdir -p -- "${MAVEN_HOME%/*}"
|
||||
|
||||
# Download and Install Apache Maven
|
||||
verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
|
||||
verbose "Downloading from: $distributionUrl"
|
||||
verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
|
||||
# select .zip or .tar.gz
|
||||
if ! command -v unzip >/dev/null; then
|
||||
distributionUrl="${distributionUrl%.zip}.tar.gz"
|
||||
distributionUrlName="${distributionUrl##*/}"
|
||||
fi
|
||||
|
||||
# verbose opt
|
||||
__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
|
||||
[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
|
||||
|
||||
# normalize http auth
|
||||
case "${MVNW_PASSWORD:+has-password}" in
|
||||
'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
|
||||
has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
|
||||
esac
|
||||
|
||||
if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
|
||||
verbose "Found wget ... using wget"
|
||||
wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
|
||||
elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
|
||||
verbose "Found curl ... using curl"
|
||||
curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
|
||||
elif set_java_home; then
|
||||
verbose "Falling back to use Java to download"
|
||||
javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
|
||||
targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
cat >"$javaSource" <<-END
|
||||
public class Downloader extends java.net.Authenticator
|
||||
{
|
||||
protected java.net.PasswordAuthentication getPasswordAuthentication()
|
||||
{
|
||||
return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
|
||||
}
|
||||
public static void main( String[] args ) throws Exception
|
||||
{
|
||||
setDefault( new Downloader() );
|
||||
java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
|
||||
}
|
||||
}
|
||||
END
|
||||
# For Cygwin/MinGW, switch paths to Windows format before running javac and java
|
||||
verbose " - Compiling Downloader.java ..."
|
||||
"$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
|
||||
verbose " - Running Downloader.java ..."
|
||||
"$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
|
||||
fi
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven distribution zip file
|
||||
if [ -n "${distributionSha256Sum-}" ]; then
|
||||
distributionSha256Result=false
|
||||
if [ "$MVN_CMD" = mvnd.sh ]; then
|
||||
echo "Checksum validation is not supported for maven-mvnd." >&2
|
||||
echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
|
||||
exit 1
|
||||
elif command -v sha256sum >/dev/null; then
|
||||
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
|
||||
distributionSha256Result=true
|
||||
fi
|
||||
elif command -v shasum >/dev/null; then
|
||||
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
|
||||
distributionSha256Result=true
|
||||
fi
|
||||
else
|
||||
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
|
||||
echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ $distributionSha256Result = false ]; then
|
||||
echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
|
||||
echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# unzip and move
|
||||
if command -v unzip >/dev/null; then
|
||||
unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
|
||||
else
|
||||
tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
|
||||
fi
|
||||
|
||||
# Find the actual extracted directory name (handles snapshots where filename != directory name)
|
||||
actualDistributionDir=""
|
||||
|
||||
# First try the expected directory name (for regular distributions)
|
||||
if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
|
||||
if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
|
||||
actualDistributionDir="$distributionUrlNameMain"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If not found, search for any directory with the Maven executable (for snapshots)
|
||||
if [ -z "$actualDistributionDir" ]; then
|
||||
# enable globbing to iterate over items
|
||||
set +f
|
||||
for dir in "$TMP_DOWNLOAD_DIR"/*; do
|
||||
if [ -d "$dir" ]; then
|
||||
if [ -f "$dir/bin/$MVN_CMD" ]; then
|
||||
actualDistributionDir="$(basename "$dir")"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
set -f
|
||||
fi
|
||||
|
||||
if [ -z "$actualDistributionDir" ]; then
|
||||
verbose "Contents of $TMP_DOWNLOAD_DIR:"
|
||||
verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
|
||||
die "Could not find Maven distribution directory in extracted archive"
|
||||
fi
|
||||
|
||||
verbose "Found extracted Maven distribution directory: $actualDistributionDir"
|
||||
printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
|
||||
mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
|
||||
|
||||
clean || :
|
||||
exec_maven "$@"
|
||||
@@ -0,0 +1,189 @@
|
||||
<# : batch portion
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one
|
||||
@REM or more contributor license agreements. See the NOTICE file
|
||||
@REM distributed with this work for additional information
|
||||
@REM regarding copyright ownership. The ASF licenses this file
|
||||
@REM to you under the Apache License, Version 2.0 (the
|
||||
@REM "License"); you may not use this file except in compliance
|
||||
@REM with the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing,
|
||||
@REM software distributed under the License is distributed on an
|
||||
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
@REM KIND, either express or implied. See the License for the
|
||||
@REM specific language governing permissions and limitations
|
||||
@REM under the License.
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Apache Maven Wrapper startup batch script, version 3.3.4
|
||||
@REM
|
||||
@REM Optional ENV vars
|
||||
@REM MVNW_REPOURL - repo url base for downloading maven distribution
|
||||
@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
|
||||
@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
|
||||
@SET __MVNW_CMD__=
|
||||
@SET __MVNW_ERROR__=
|
||||
@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
|
||||
@SET PSModulePath=
|
||||
@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
|
||||
IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
|
||||
)
|
||||
@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
|
||||
@SET __MVNW_PSMODULEP_SAVE=
|
||||
@SET __MVNW_ARG0_NAME__=
|
||||
@SET MVNW_USERNAME=
|
||||
@SET MVNW_PASSWORD=
|
||||
@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
|
||||
@echo Cannot start maven from wrapper >&2 && exit /b 1
|
||||
@GOTO :EOF
|
||||
: end batch / begin powershell #>
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ($env:MVNW_VERBOSE -eq "true") {
|
||||
$VerbosePreference = "Continue"
|
||||
}
|
||||
|
||||
# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
|
||||
$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
|
||||
if (!$distributionUrl) {
|
||||
Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
}
|
||||
|
||||
switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
|
||||
"maven-mvnd-*" {
|
||||
$USE_MVND = $true
|
||||
$distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
|
||||
$MVN_CMD = "mvnd.cmd"
|
||||
break
|
||||
}
|
||||
default {
|
||||
$USE_MVND = $false
|
||||
$MVN_CMD = $script -replace '^mvnw','mvn'
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
# apply MVNW_REPOURL and calculate MAVEN_HOME
|
||||
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
|
||||
if ($env:MVNW_REPOURL) {
|
||||
$MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
|
||||
$distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
|
||||
}
|
||||
$distributionUrlName = $distributionUrl -replace '^.*/',''
|
||||
$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
|
||||
|
||||
$MAVEN_M2_PATH = "$HOME/.m2"
|
||||
if ($env:MAVEN_USER_HOME) {
|
||||
$MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
|
||||
New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
|
||||
}
|
||||
|
||||
$MAVEN_WRAPPER_DISTS = $null
|
||||
if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
|
||||
$MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
|
||||
} else {
|
||||
$MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
|
||||
}
|
||||
|
||||
$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
|
||||
$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
|
||||
$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
|
||||
|
||||
if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
|
||||
Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
|
||||
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
|
||||
exit $?
|
||||
}
|
||||
|
||||
if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
|
||||
Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
|
||||
}
|
||||
|
||||
# prepare tmp dir
|
||||
$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
|
||||
$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
|
||||
$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
|
||||
trap {
|
||||
if ($TMP_DOWNLOAD_DIR.Exists) {
|
||||
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
|
||||
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
|
||||
|
||||
# Download and Install Apache Maven
|
||||
Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
|
||||
Write-Verbose "Downloading from: $distributionUrl"
|
||||
Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
|
||||
$webclient = New-Object System.Net.WebClient
|
||||
if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
|
||||
$webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
|
||||
}
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven distribution zip file
|
||||
$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
|
||||
if ($distributionSha256Sum) {
|
||||
if ($USE_MVND) {
|
||||
Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
|
||||
}
|
||||
Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
|
||||
if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
|
||||
Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
|
||||
}
|
||||
}
|
||||
|
||||
# unzip and move
|
||||
Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
|
||||
|
||||
# Find the actual extracted directory name (handles snapshots where filename != directory name)
|
||||
$actualDistributionDir = ""
|
||||
|
||||
# First try the expected directory name (for regular distributions)
|
||||
$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
|
||||
$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
|
||||
if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
|
||||
$actualDistributionDir = $distributionUrlNameMain
|
||||
}
|
||||
|
||||
# If not found, search for any directory with the Maven executable (for snapshots)
|
||||
if (!$actualDistributionDir) {
|
||||
Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
|
||||
$testPath = Join-Path $_.FullName "bin/$MVN_CMD"
|
||||
if (Test-Path -Path $testPath -PathType Leaf) {
|
||||
$actualDistributionDir = $_.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!$actualDistributionDir) {
|
||||
Write-Error "Could not find Maven distribution directory in extracted archive"
|
||||
}
|
||||
|
||||
Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
|
||||
Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
|
||||
try {
|
||||
Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
|
||||
} catch {
|
||||
if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
|
||||
Write-Error "fail to move MAVEN_HOME"
|
||||
}
|
||||
} finally {
|
||||
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
|
||||
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
|
||||
}
|
||||
|
||||
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
|
||||
@@ -18,16 +18,19 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.2.0-RC.1</revision>
|
||||
<revision>1.0.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<opensearch.version>2.10.0</opensearch.version>
|
||||
<opensearch.version>3.10.0</opensearch.version>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<opensearch.test.image>opensearchproject/opensearch:2.19.6</opensearch.test.image>
|
||||
<aws.sdk.version>2.41.10</aws.sdk.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>1.6.0</version>
|
||||
<version>2.0.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -66,25 +69,6 @@
|
||||
<tag>HEAD</tag>
|
||||
</scm>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.junit</groupId>
|
||||
<artifactId>junit-bom</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-bom-pom</artifactId>
|
||||
<version>0.40.0</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
@@ -97,16 +81,32 @@
|
||||
<version>${opensearch.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- httpclient5 and jackson versions are managed by the qqq BOM, so they -->
|
||||
<!-- stay binary-compatible with the httpcore5 and jackson it brings. -->
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents.client5</groupId>
|
||||
<artifactId>httpclient5</artifactId>
|
||||
<version>5.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||
<version>2.21.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Only needed for authMode AWS_SIGV4 (Amazon OpenSearch Service with IAM). -->
|
||||
<!-- Optional: hosts that use it declare them; the core modules already come -->
|
||||
<!-- through qqq-backend-core at the same version. -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>apache-client</artifactId>
|
||||
<version>${aws.sdk.version}</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>sts</artifactId>
|
||||
<version>${aws.sdk.version}</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -125,14 +125,13 @@
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.27.3</version>
|
||||
<version>3.27.7</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
@@ -147,18 +146,34 @@
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers</artifactId>
|
||||
<version>1.21.0</version>
|
||||
<version>1.21.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>1.21.0</version>
|
||||
<version>1.21.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>true</filtering>
|
||||
<includes>
|
||||
<include>quick-search.properties</include>
|
||||
</includes>
|
||||
</resource>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>false</filtering>
|
||||
<excludes>
|
||||
<exclude>quick-search.properties</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
@@ -177,6 +192,9 @@
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
<version>3.1.2</version>
|
||||
<configuration>
|
||||
<systemPropertyVariables>
|
||||
<opensearch.test.image>${opensearch.test.image}</opensearch.test.image>
|
||||
</systemPropertyVariables>
|
||||
<includes>
|
||||
<include>**/*IntegrationTest.java</include>
|
||||
<include>**/*IT.java</include>
|
||||
@@ -194,7 +212,21 @@
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>${jacoco.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>check</id>
|
||||
<goals>
|
||||
@@ -224,4 +256,45 @@
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<!-- 1.0 requires qqq 4.1 APIs (record-change listeners, after-commit -->
|
||||
<!-- callbacks, runtime services). Until qbit-build-parent 2.1.0 imports the -->
|
||||
<!-- released 4.1 BOM, this profile is active by default and imports -->
|
||||
<!-- qqq-bom-pom 4.1.0-RC.1 ahead of the parent's 4.0.0 BOM. Override with -->
|
||||
<!-- -Dqqq.snapshot.version=... ; deactivate with -P!qqq-snapshot. Remove the -->
|
||||
<!-- activation when the parent moves to 4.1 (ADR-0007). -->
|
||||
<profile>
|
||||
<id>qqq-snapshot</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.1</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-bom-pom</artifactId>
|
||||
<version>${qqq.snapshot.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>sonatype-snapshots</id>
|
||||
<url>https://central.sonatype.com/repository/maven-snapshots/</url>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** How the QBit authenticates to OpenSearch.
|
||||
**
|
||||
** NONE - no credentials (local development, security plugin disabled).
|
||||
** BASIC - HTTP basic authentication (security plugin internal users, or
|
||||
** Amazon OpenSearch Service fine-grained access control with an
|
||||
** internal-database master user).
|
||||
** AWS_SIGV4 - AWS Signature Version 4 request signing with the default AWS
|
||||
** credential chain (Amazon OpenSearch Service domains that use
|
||||
** IAM). Requires the optional AWS SDK v2 dependencies.
|
||||
**
|
||||
** Bearer tokens, API keys and client certificates are added through an
|
||||
** {@link com.kingsrook.qbits.quicksearch.opensearch.OpenSearchTransportCustomizer}.
|
||||
*******************************************************************************/
|
||||
public enum QuickSearchAuthMode
|
||||
{
|
||||
NONE,
|
||||
BASIC,
|
||||
AWS_SIGV4
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -13,44 +13,58 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Static holder for Quick Search QBit runtime state.
|
||||
** Deprecated static facade over {@link QuickSearchRuntime}.
|
||||
**
|
||||
** Provides access to the active configuration, the OpenSearch client, the
|
||||
** index-event publisher, and the list of tables discovered at startup.
|
||||
** Until 1.0 the QBit kept its live state in these static fields. The state now
|
||||
** lives on the produced QBit's runtime, resolved from QContext. The getters
|
||||
** here return an explicitly set static value when one exists (tests and
|
||||
** legacy hosts), otherwise they delegate to {@link QuickSearchRuntime#get()}.
|
||||
**
|
||||
** Call clear() in test @AfterEach / @BeforeEach to reset all state.
|
||||
** @deprecated resolve state with {@link QuickSearchRuntime#get()}; this
|
||||
** facade will be removed in 1.1.
|
||||
*******************************************************************************/
|
||||
@Deprecated
|
||||
public class QuickSearchQBitContext
|
||||
{
|
||||
private static volatile QuickSearchQBitConfig config;
|
||||
private static volatile List<QuickSearchableTableConfig> discoveredTables;
|
||||
private static volatile QuickSearchOpenSearchClient client;
|
||||
private static volatile IndexEventPublisher publisher;
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchQBitContext.class);
|
||||
|
||||
private static volatile QuickSearchQBitConfig config;
|
||||
private static volatile List<QuickSearchableTableConfig> discoveredTables;
|
||||
private static volatile QuickSearchOpenSearchClient client;
|
||||
private static volatile IndexEventPublisher publisher;
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Getter for config
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Get the config: the static override, else the resolved runtime's config.
|
||||
*******************************************************************************/
|
||||
public static QuickSearchQBitConfig getConfig()
|
||||
{
|
||||
return config;
|
||||
if(config != null)
|
||||
{
|
||||
return (config);
|
||||
}
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
return (runtime == null ? null : runtime.getConfig());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Setter for config
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Set a static config override.
|
||||
*******************************************************************************/
|
||||
public static void setConfig(QuickSearchQBitConfig config)
|
||||
{
|
||||
QuickSearchQBitContext.config = config;
|
||||
@@ -58,19 +72,24 @@ public class QuickSearchQBitContext
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Getter for discoveredTables
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Get the discovered tables: the static override, else the runtime's.
|
||||
*******************************************************************************/
|
||||
public static List<QuickSearchableTableConfig> getDiscoveredTables()
|
||||
{
|
||||
return discoveredTables;
|
||||
if(discoveredTables != null)
|
||||
{
|
||||
return (discoveredTables);
|
||||
}
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
return (runtime == null ? null : runtime.getDiscoveredTables());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Setter for discoveredTables
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Set a static discovered-tables override.
|
||||
*******************************************************************************/
|
||||
public static void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
|
||||
{
|
||||
QuickSearchQBitContext.discoveredTables = discoveredTables;
|
||||
@@ -78,44 +97,62 @@ public class QuickSearchQBitContext
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Find a table config by table name from the discovered tables list.
|
||||
**
|
||||
** Returns null if discoveredTables is null or no match is found.
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Find a table config by name, or null.
|
||||
*******************************************************************************/
|
||||
public static QuickSearchableTableConfig getTableConfig(String tableName)
|
||||
{
|
||||
if(discoveredTables == null)
|
||||
List<QuickSearchableTableConfig> tables = getDiscoveredTables();
|
||||
if(tables == null || tableName == null)
|
||||
{
|
||||
return null;
|
||||
return (null);
|
||||
}
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : discoveredTables)
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
if(tableName.equals(tableConfig.getTableName()))
|
||||
{
|
||||
return tableConfig;
|
||||
return (tableConfig);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Getter for client
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Get the client: the static override, else the runtime's (built lazily).
|
||||
** Returns null when no runtime is registered or the client cannot be built.
|
||||
*******************************************************************************/
|
||||
public static QuickSearchOpenSearchClient getClient()
|
||||
{
|
||||
return client;
|
||||
if(client != null)
|
||||
{
|
||||
return (client);
|
||||
}
|
||||
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return (runtime.getClient());
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
LOG.warn("Could not build OpenSearch client", e);
|
||||
return (null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Setter for client
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Set a static client override.
|
||||
*******************************************************************************/
|
||||
public static void setClient(QuickSearchOpenSearchClient client)
|
||||
{
|
||||
QuickSearchQBitContext.client = client;
|
||||
@@ -123,19 +160,38 @@ public class QuickSearchQBitContext
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Getter for publisher
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Get the publisher: the static override, else the runtime's.
|
||||
*******************************************************************************/
|
||||
public static IndexEventPublisher getPublisher()
|
||||
{
|
||||
return publisher;
|
||||
if(publisher != null)
|
||||
{
|
||||
return (publisher);
|
||||
}
|
||||
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return (runtime.getPublisher());
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
LOG.warn("Could not build index event publisher", e);
|
||||
return (null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Setter for publisher
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Set a static publisher override.
|
||||
*******************************************************************************/
|
||||
public static void setPublisher(IndexEventPublisher publisher)
|
||||
{
|
||||
QuickSearchQBitContext.publisher = publisher;
|
||||
@@ -143,18 +199,15 @@ public class QuickSearchQBitContext
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Reset all static state to null.
|
||||
**
|
||||
** Call in test @BeforeEach or @AfterEach to prevent state leakage
|
||||
** between tests.
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Clear the static overrides (tests).
|
||||
*******************************************************************************/
|
||||
public static void clear()
|
||||
{
|
||||
config = null;
|
||||
config = null;
|
||||
discoveredTables = null;
|
||||
client = null;
|
||||
publisher = null;
|
||||
client = null;
|
||||
publisher = null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -13,265 +13,271 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
|
||||
import java.util.Properties;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostDeleteCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostInsertCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostUpdateCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
|
||||
import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Central orchestrator for the Quick Search QBit.
|
||||
** Entry point of the Quick Search QBit, a QQQ 4.1 QBitMetaDataProducer.
|
||||
**
|
||||
** Validates config, discovers annotated tables, produces QQQ metadata
|
||||
** (tables, processes, app), initializes the OpenSearch client, and registers
|
||||
** real-time customizers on source tables.
|
||||
** The framework's default produce() registers QBitMetaData, validates the
|
||||
** config, runs the component producers in this package (operational tables,
|
||||
** processes, admin app), applies the config's table customizer and default
|
||||
** backend, stamps sourceQBitName, then calls {@link #postProduceActions}, which
|
||||
** discovers searchable tables, builds the runtime, registers the record-change
|
||||
** listener and runtime service, and (in FAIL_FAST mode) prepares the index.
|
||||
**
|
||||
** Entry point: {@link #produce(QInstance)}.
|
||||
** Host usage, unchanged from 0.x:
|
||||
** new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
** That overload adds everything to the instance itself; hosts that discover
|
||||
** this producer through MetaDataProducerHelper get the standard contract.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchQBitProducer
|
||||
public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearchQBitConfig>
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchQBitProducer.class);
|
||||
|
||||
public static final String BASEPULL_PROCESS_NAME = "quickSearchBasepullIndex";
|
||||
public static final String FULL_REINDEX_PROCESS_NAME = "quickSearchFullReindex";
|
||||
public static final String APP_NAME = "quickSearchAdmin";
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "quick-search";
|
||||
|
||||
private QuickSearchQBitConfig config;
|
||||
public static final String BASEPULL_PROCESS_NAME = QuickSearchProcessMetaDataHelper.BASEPULL_PROCESS_NAME;
|
||||
public static final String FULL_REINDEX_PROCESS_NAME = QuickSearchProcessMetaDataHelper.FULL_REINDEX_PROCESS_NAME;
|
||||
public static final String RECONCILE_PROCESS_NAME = QuickSearchProcessMetaDataHelper.RECONCILE_PROCESS_NAME;
|
||||
public static final String APP_NAME = QuickSearchAdminAppMetaDataProducer.APP_NAME;
|
||||
|
||||
private static final String VERSION = loadVersion();
|
||||
|
||||
private QuickSearchQBitConfig qBitConfig;
|
||||
private String namespace;
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Fluent setter for config.
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the config (0.x name).
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitProducer withConfig(QuickSearchQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
this.qBitConfig = config;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Getter for config.
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Getter for the config (0.x name).
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig getConfig()
|
||||
{
|
||||
return (this.config);
|
||||
return (this.qBitConfig);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Produce all QQQ metadata for the Quick Search QBit and register it
|
||||
** into the given QInstance.
|
||||
**
|
||||
** Steps:
|
||||
** 1. Validate configuration
|
||||
** 2. Discover @QuickSearchable annotated entity classes
|
||||
** 3. Create and initialize the OpenSearch client
|
||||
** 4. Create the index event publisher
|
||||
** 5. Store everything in QuickSearchQBitContext
|
||||
** 6. Produce operational tables
|
||||
** 7. Produce processes
|
||||
** 8. Register customizers on source tables (if real-time indexing enabled)
|
||||
** 9. Produce an admin app
|
||||
**
|
||||
** @param qInstance the QQQ instance to register metadata into
|
||||
** @throws QException if validation fails or an unrecoverable error occurs
|
||||
***************************************************************************/
|
||||
public void produce(QInstance qInstance) throws QException
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the config.
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitProducer withQBitConfig(QuickSearchQBitConfig config)
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// 1. Validate config //
|
||||
////////////////////////////////////////////////////
|
||||
List<String> errors = new ArrayList<>();
|
||||
config.validate(qInstance, errors);
|
||||
if(!errors.isEmpty())
|
||||
this.qBitConfig = config;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for the config.
|
||||
*******************************************************************************/
|
||||
public void setQBitConfig(QuickSearchQBitConfig config)
|
||||
{
|
||||
this.qBitConfig = config;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QuickSearchQBitConfig getQBitConfig()
|
||||
{
|
||||
return (this.qBitConfig);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Namespace for this QBit instance; also the default tableNamePrefix.
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitProducer withNamespace(String namespace)
|
||||
{
|
||||
this.namespace = namespace;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getNamespace()
|
||||
{
|
||||
return (this.namespace);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QBitMetaData getQBitMetaData()
|
||||
{
|
||||
return (new QBitMetaData()
|
||||
.withGroupId(GROUP_ID)
|
||||
.withArtifactId(ARTIFACT_ID)
|
||||
.withVersion(VERSION)
|
||||
.withNamespace(getNamespace())
|
||||
.withConfig(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A producer found by package scanning has no config; skip it.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (qBitConfig != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produce and add everything to the instance. Returns an empty output so a
|
||||
** host that also calls addSelfToInstance on the result adds nothing twice.
|
||||
**
|
||||
** Validation errors are reported as one QException listing every error.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
|
||||
{
|
||||
if(qBitConfig == null)
|
||||
{
|
||||
throw new QException("QuickSearchQBit configuration validation failed: " + String.join("; ", errors));
|
||||
throw (new QException("QuickSearchQBit configuration validation failed: no config was supplied (call withConfig)"));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2. Discover searchable tables from annotations //
|
||||
////////////////////////////////////////////////////
|
||||
if(StringUtils.hasContent(namespace) && !StringUtils.hasContent(qBitConfig.getTableNamePrefix()))
|
||||
{
|
||||
qBitConfig.setTableNamePrefix(namespace);
|
||||
}
|
||||
|
||||
List<String> errors = new ArrayList<>();
|
||||
qBitConfig.validate(qInstance, errors);
|
||||
if(!errors.isEmpty())
|
||||
{
|
||||
throw (new QException("QuickSearchQBit configuration validation failed: " + String.join("; ", errors)));
|
||||
}
|
||||
|
||||
MetaDataProducerMultiOutput output = QBitMetaDataProducer.super.produce(qInstance);
|
||||
output.addSelfToInstance(qInstance);
|
||||
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Discover tables, build the runtime, register the listener and runtime
|
||||
** service, and prepare the index in FAIL_FAST mode.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postProduceActions(MetaDataProducerMultiOutput output, QInstance qInstance) throws QException
|
||||
{
|
||||
List<QuickSearchableTableConfig> annotationTables = discoverSearchableTables(qInstance);
|
||||
List<QuickSearchableTableConfig> configTables = convertConfigDrivenTables(qInstance);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2b. Convert config-driven searchable tables //
|
||||
////////////////////////////////////////////////////
|
||||
List<QuickSearchableTableConfig> configTables = convertConfigDrivenTables(qInstance);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2c. Detect duplicate table names across both //
|
||||
// annotation-based and config-driven sources //
|
||||
////////////////////////////////////////////////////
|
||||
for(QuickSearchableTableConfig annotationTable : annotationTables)
|
||||
{
|
||||
for(QuickSearchableTableConfig configTable : configTables)
|
||||
{
|
||||
if(annotationTable.getTableName().equals(configTable.getTableName()))
|
||||
{
|
||||
throw new QException("Duplicate table name found in both annotation-based and config-driven searchable tables: " + annotationTable.getTableName());
|
||||
throw (new QException("Duplicate table name found in both annotation-based and config-driven searchable tables: " + annotationTable.getTableName()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2d. Merge both lists into discoveredTables //
|
||||
////////////////////////////////////////////////////
|
||||
List<QuickSearchableTableConfig> discoveredTables = new ArrayList<>();
|
||||
discoveredTables.addAll(annotationTables);
|
||||
discoveredTables.addAll(configTables);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 3. Create OpenSearch client //
|
||||
////////////////////////////////////////////////////
|
||||
QuickSearchOpenSearchClient client = null;
|
||||
try
|
||||
QuickSearchRuntime previous = qBitConfig.getRuntime();
|
||||
if(previous != null)
|
||||
{
|
||||
client = new QuickSearchOpenSearchClient(config);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to create OpenSearch client during produce; continuing without client", e);
|
||||
previous.close();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 4. Initialize index (best-effort) //
|
||||
////////////////////////////////////////////////////
|
||||
if(client != null)
|
||||
QuickSearchRuntime runtime = new QuickSearchRuntime(qBitConfig);
|
||||
runtime.setDiscoveredTables(discoveredTables);
|
||||
qBitConfig.setRuntime(runtime);
|
||||
|
||||
if(Boolean.TRUE.equals(qBitConfig.getEnableRealTimeIndexing()))
|
||||
{
|
||||
try
|
||||
{
|
||||
client.ensureIndexExists();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to ensure OpenSearch index exists during produce; continuing", e);
|
||||
}
|
||||
qInstance.withRecordChangeListener(new QCodeReference(QuickSearchRecordChangeListener.class));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 5. Create publisher //
|
||||
////////////////////////////////////////////////////
|
||||
IndexEventPublisher publisher = null;
|
||||
if(config.getIndexEventPublisher() != null)
|
||||
qInstance.withRuntimeService(new QCodeReference(QuickSearchRuntimeService.class));
|
||||
|
||||
if(!StringUtils.hasContent(qBitConfig.getSchedulerName()) && Boolean.TRUE.equals(qBitConfig.getEnableBasepullProcess()))
|
||||
{
|
||||
publisher = config.getIndexEventPublisher();
|
||||
}
|
||||
else if(client != null)
|
||||
{
|
||||
publisher = new SynchronousIndexEventPublisher(client, discoveredTables, config.getBulkBatchSize());
|
||||
LOG.warn("Quick Search has no schedulerName; the basepull and reconcile processes are registered but will not run on a schedule");
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 6. Store in context //
|
||||
////////////////////////////////////////////////////
|
||||
QuickSearchQBitContext.setConfig(config);
|
||||
QuickSearchQBitContext.setClient(client);
|
||||
QuickSearchQBitContext.setPublisher(publisher);
|
||||
QuickSearchQBitContext.setDiscoveredTables(discoveredTables);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 7. Produce operational tables //
|
||||
////////////////////////////////////////////////////
|
||||
String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME);
|
||||
String indexRunTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME);
|
||||
|
||||
QTableMetaData indexTable = buildQuickSearchIndexTable(indexTableName, config.getBackendName());
|
||||
QTableMetaData indexRunTable = buildQuickSearchIndexRunTable(indexRunTableName, config.getBackendName());
|
||||
|
||||
qInstance.addTable(indexTable);
|
||||
qInstance.addTable(indexRunTable);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 8. Produce processes //
|
||||
////////////////////////////////////////////////////
|
||||
String basepullProcessName = config.applyPrefix(BASEPULL_PROCESS_NAME);
|
||||
String fullReindexProcessName = config.applyPrefix(FULL_REINDEX_PROCESS_NAME);
|
||||
|
||||
QProcessMetaData basepullProcess = buildBasepullProcess(basepullProcessName);
|
||||
QProcessMetaData fullReindexProcess = buildFullReindexProcess(fullReindexProcessName);
|
||||
|
||||
if(Boolean.TRUE.equals(config.getEnableScheduledProcesses()))
|
||||
if(qBitConfig.getStartupMode() == QuickSearchStartupMode.FAIL_FAST)
|
||||
{
|
||||
qInstance.addProcess(basepullProcess);
|
||||
qInstance.addProcess(fullReindexProcess);
|
||||
runtime.start();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 9. Register customizers on source tables //
|
||||
////////////////////////////////////////////////////
|
||||
if(Boolean.TRUE.equals(config.getEnableRealTimeIndexing()))
|
||||
{
|
||||
registerCustomizers(qInstance, discoveredTables);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 10. Produce admin app //
|
||||
////////////////////////////////////////////////////
|
||||
String appName = config.applyPrefix(APP_NAME);
|
||||
QAppMetaData app = new QAppMetaData()
|
||||
.withName(appName)
|
||||
.withLabel("Quick Search Admin")
|
||||
.withChild(indexTable)
|
||||
.withChild(indexRunTable);
|
||||
|
||||
if(Boolean.TRUE.equals(config.getEnableScheduledProcesses()))
|
||||
{
|
||||
app.withChild(basepullProcess);
|
||||
app.withChild(fullReindexProcess);
|
||||
}
|
||||
|
||||
qInstance.addApp(app);
|
||||
LOG.info("Quick Search QBit produced", logPair("version", VERSION), logPair("tables", discoveredTables.size()),
|
||||
logPair("indexName", qBitConfig.getOpensearchIndexName()), logPair("startupMode", qBitConfig.getStartupMode()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Discover searchable table configurations by scanning annotated entity
|
||||
** classes from the config.
|
||||
**
|
||||
** For each class, extracts @QuickSearchable annotation attributes and
|
||||
** scans fields for @QuickSearchField annotations. Falls back to the
|
||||
** annotation's fields() array if no @QuickSearchField-annotated fields
|
||||
** are found.
|
||||
**
|
||||
** @param qInstance the QInstance to look up table primary keys
|
||||
** @return list of discovered table configurations
|
||||
***************************************************************************/
|
||||
List<QuickSearchableTableConfig> discoverSearchableTables(QInstance qInstance)
|
||||
/*******************************************************************************
|
||||
** Discover searchable tables from @QuickSearchable entity classes. Fails when
|
||||
** a configured field does not exist on a table that is in the instance.
|
||||
*******************************************************************************/
|
||||
List<QuickSearchableTableConfig> discoverSearchableTables(QInstance qInstance) throws QException
|
||||
{
|
||||
List<QuickSearchableTableConfig> tables = new ArrayList<>();
|
||||
List<Class<?>> entityClasses = config.getSearchableEntityClasses();
|
||||
List<QuickSearchableTableConfig> tables = new ArrayList<>();
|
||||
List<Class<?>> entityClasses = qBitConfig.getSearchableEntityClasses();
|
||||
|
||||
if(entityClasses == null || entityClasses.isEmpty())
|
||||
{
|
||||
@@ -283,81 +289,65 @@ public class QuickSearchQBitProducer
|
||||
QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class);
|
||||
if(annotation == null)
|
||||
{
|
||||
LOG.warn("Entity class does not have @QuickSearchable annotation; skipping",
|
||||
"className", entityClass.getName());
|
||||
LOG.warn("Entity class does not have @QuickSearchable annotation; skipping", logPair("className", entityClass.getName()));
|
||||
continue;
|
||||
}
|
||||
|
||||
String tableName = annotation.tableName();
|
||||
Integer basepullIntervalMinutes = annotation.basepullIntervalMinutes();
|
||||
String basepullTimestampField = annotation.basepullTimestampField();
|
||||
Boolean enabledByDefault = annotation.enabledByDefault();
|
||||
String tableName = annotation.tableName();
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Scan fields for @QuickSearchField annotations //
|
||||
//////////////////////////////////////////////////////////
|
||||
List<String> searchableFields = new ArrayList<>();
|
||||
Map<String, Integer> fieldWeights = new LinkedHashMap<>();
|
||||
List<String> searchableFields = new ArrayList<>();
|
||||
Map<String, Integer> fieldWeights = new LinkedHashMap<>();
|
||||
Map<String, Boolean> fieldIncludeLabels = new LinkedHashMap<>();
|
||||
|
||||
List<Field> allFields = new ArrayList<>();
|
||||
Class<?> current = entityClass;
|
||||
Class<?> current = entityClass;
|
||||
while(current != null && current != Object.class)
|
||||
{
|
||||
allFields.addAll(java.util.Arrays.asList(current.getDeclaredFields()));
|
||||
allFields.addAll(Arrays.asList(current.getDeclaredFields()));
|
||||
current = current.getSuperclass();
|
||||
}
|
||||
|
||||
for(Field field : allFields)
|
||||
{
|
||||
QuickSearchField fieldAnnotation = field.getAnnotation(QuickSearchField.class);
|
||||
if(fieldAnnotation != null)
|
||||
{
|
||||
String fieldName = field.getName();
|
||||
searchableFields.add(fieldName);
|
||||
fieldWeights.put(fieldName, fieldAnnotation.weight());
|
||||
fieldIncludeLabels.put(fieldName, fieldAnnotation.includeLabel());
|
||||
searchableFields.add(field.getName());
|
||||
fieldWeights.put(field.getName(), fieldAnnotation.weight());
|
||||
fieldIncludeLabels.put(field.getName(), fieldAnnotation.includeLabel());
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Fall back to @QuickSearchable.fields() if no //
|
||||
// @QuickSearchField-annotated fields were found //
|
||||
//////////////////////////////////////////////////////////
|
||||
if(searchableFields.isEmpty())
|
||||
if(searchableFields.isEmpty() && annotation.fields() != null)
|
||||
{
|
||||
String[] annotationFields = annotation.fields();
|
||||
if(annotationFields != null)
|
||||
for(String fieldName : annotation.fields())
|
||||
{
|
||||
for(String fieldName : annotationFields)
|
||||
{
|
||||
searchableFields.add(fieldName);
|
||||
fieldWeights.put(fieldName, 1);
|
||||
fieldIncludeLabels.put(fieldName, false);
|
||||
}
|
||||
searchableFields.add(fieldName);
|
||||
fieldWeights.put(fieldName, 1);
|
||||
fieldIncludeLabels.put(fieldName, false);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Get primary key field from QInstance table metadata //
|
||||
//////////////////////////////////////////////////////////
|
||||
String primaryKeyField = "id";
|
||||
QTableMetaData table = qInstance.getTable(tableName);
|
||||
String primaryKeyField = "id";
|
||||
String basepullTimestampField = StringUtils.hasContent(annotation.basepullTimestampField()) ? annotation.basepullTimestampField() : null;
|
||||
QTableMetaData table = qInstance.getTable(tableName);
|
||||
if(table != null)
|
||||
{
|
||||
primaryKeyField = table.getPrimaryKeyField();
|
||||
primaryKeyField = table.getPrimaryKeyField();
|
||||
basepullTimestampField = resolveTimestampField(table, searchableFields, basepullTimestampField);
|
||||
removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
|
||||
}
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
tables.add(new QuickSearchableTableConfig()
|
||||
.withTableName(tableName)
|
||||
.withPrimaryKeyField(primaryKeyField)
|
||||
.withSearchableFields(searchableFields)
|
||||
.withFieldWeights(fieldWeights)
|
||||
.withFieldIncludeLabels(fieldIncludeLabels)
|
||||
.withBasepullIntervalMinutes(basepullIntervalMinutes)
|
||||
.withBasepullIntervalMinutes(annotation.basepullIntervalMinutes())
|
||||
.withBasepullTimestampField(basepullTimestampField)
|
||||
.withEnabledByDefault(enabledByDefault);
|
||||
|
||||
tables.add(tableConfig);
|
||||
.withEnabledByDefault(annotation.enabledByDefault())
|
||||
.withMaxFieldLength(qBitConfig.getMaxFieldLength()));
|
||||
}
|
||||
|
||||
return (tables);
|
||||
@@ -365,24 +355,13 @@ public class QuickSearchQBitProducer
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Convert config-driven SearchableTableConfig entries into
|
||||
** QuickSearchableTableConfig objects.
|
||||
**
|
||||
** Maps each SearchableFieldConfig into the searchableFields list,
|
||||
** fieldWeights map, and fieldIncludeLabels map. Resolves the primary
|
||||
** key field from the QInstance table metadata (falls back to "id").
|
||||
** Applies the default basepullIntervalMinutes from QBit config when
|
||||
** the table-level value is null. Transfers recordLabelFormat and
|
||||
** recordLabelFields.
|
||||
**
|
||||
** @param qInstance the QInstance to look up table primary keys
|
||||
** @return list of converted table configurations, empty if none configured
|
||||
***************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Convert config-driven tables, resolving primary keys and defaults.
|
||||
*******************************************************************************/
|
||||
private List<QuickSearchableTableConfig> convertConfigDrivenTables(QInstance qInstance)
|
||||
{
|
||||
List<QuickSearchableTableConfig> tables = new ArrayList<>();
|
||||
List<SearchableTableConfig> searchableTables = config.getSearchableTables();
|
||||
List<QuickSearchableTableConfig> tables = new ArrayList<>();
|
||||
List<SearchableTableConfig> searchableTables = qBitConfig.getSearchableTables();
|
||||
|
||||
if(searchableTables == null || searchableTables.isEmpty())
|
||||
{
|
||||
@@ -391,53 +370,45 @@ public class QuickSearchQBitProducer
|
||||
|
||||
for(SearchableTableConfig stc : searchableTables)
|
||||
{
|
||||
List<String> searchableFields = new ArrayList<>();
|
||||
Map<String, Integer> fieldWeights = new LinkedHashMap<>();
|
||||
Map<String, Boolean> fieldIncludeLabels = new LinkedHashMap<>();
|
||||
List<String> searchableFields = new ArrayList<>();
|
||||
Map<String, Integer> fieldWeights = new LinkedHashMap<>();
|
||||
Map<String, Boolean> fieldIncludeLabels = new LinkedHashMap<>();
|
||||
|
||||
if(stc.getFields() != null)
|
||||
for(SearchableFieldConfig fieldConfig : stc.getFields() == null ? List.<SearchableFieldConfig>of() : stc.getFields())
|
||||
{
|
||||
for(SearchableFieldConfig fieldConfig : stc.getFields())
|
||||
{
|
||||
String fieldName = fieldConfig.getFieldName();
|
||||
searchableFields.add(fieldName);
|
||||
fieldWeights.put(fieldName, fieldConfig.getWeight() != null ? fieldConfig.getWeight() : 1);
|
||||
fieldIncludeLabels.put(fieldName, fieldConfig.getIncludeLabel() != null ? fieldConfig.getIncludeLabel() : false);
|
||||
}
|
||||
searchableFields.add(fieldConfig.getFieldName());
|
||||
fieldWeights.put(fieldConfig.getFieldName(), fieldConfig.getWeight() != null ? fieldConfig.getWeight() : 1);
|
||||
fieldIncludeLabels.put(fieldConfig.getFieldName(), fieldConfig.getIncludeLabel() != null ? fieldConfig.getIncludeLabel() : false);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Resolve primary key from QInstance table metadata //
|
||||
//////////////////////////////////////////////////////////
|
||||
String primaryKeyField = "id";
|
||||
QTableMetaData table = qInstance.getTable(stc.getTableName());
|
||||
String primaryKeyField = "id";
|
||||
String basepullTimestampField = stc.getBasepullTimestampField();
|
||||
QTableMetaData table = qInstance.getTable(stc.getTableName());
|
||||
if(table != null)
|
||||
{
|
||||
primaryKeyField = table.getPrimaryKeyField();
|
||||
primaryKeyField = table.getPrimaryKeyField();
|
||||
basepullTimestampField = (basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField)) ? null : basepullTimestampField;
|
||||
removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// Apply default basepullIntervalMinutes from config //
|
||||
//////////////////////////////////////////////////////////
|
||||
Integer basepullInterval = stc.getBasepullIntervalMinutes();
|
||||
if(basepullInterval == null)
|
||||
{
|
||||
basepullInterval = config.getDefaultBasepullIntervalMinutes();
|
||||
basepullInterval = qBitConfig.getDefaultBasepullIntervalMinutes();
|
||||
}
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
tables.add(new QuickSearchableTableConfig()
|
||||
.withTableName(stc.getTableName())
|
||||
.withPrimaryKeyField(primaryKeyField)
|
||||
.withSearchableFields(searchableFields)
|
||||
.withFieldWeights(fieldWeights)
|
||||
.withFieldIncludeLabels(fieldIncludeLabels)
|
||||
.withBasepullIntervalMinutes(basepullInterval)
|
||||
.withBasepullTimestampField(stc.getBasepullTimestampField())
|
||||
.withBasepullTimestampField(basepullTimestampField)
|
||||
.withEnabledByDefault(stc.getEnabledByDefault())
|
||||
.withRecordLabelFormat(stc.getRecordLabelFormat())
|
||||
.withRecordLabelFields(stc.getRecordLabelFields());
|
||||
|
||||
tables.add(tableConfig);
|
||||
.withRecordLabelFields(stc.getRecordLabelFields())
|
||||
.withMaxFieldLength(qBitConfig.getMaxFieldLength()));
|
||||
}
|
||||
|
||||
return (tables);
|
||||
@@ -445,115 +416,102 @@ public class QuickSearchQBitProducer
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Build QTableMetaData for the quickSearchIndex operational table.
|
||||
***************************************************************************/
|
||||
private QTableMetaData buildQuickSearchIndexTable(String tableName, String backendName)
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
/*******************************************************************************
|
||||
** Fail on missing searchable fields. A missing timestamp field disables
|
||||
** incremental basepull for the table (reconcile still covers it): the
|
||||
** default "modifyDate" is downgraded with a warning, an explicit other name
|
||||
** is an error.
|
||||
*******************************************************************************/
|
||||
private static String resolveTimestampField(QTableMetaData table, List<String> searchableFields, String basepullTimestampField) throws QException
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName(tableName)
|
||||
.withLabel("Quick Search Index")
|
||||
.withBackendName(backendName)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tableName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
|
||||
.withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("searchableFieldsJson", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("recordCount", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Build QTableMetaData for the quickSearchIndexRun operational table.
|
||||
***************************************************************************/
|
||||
private QTableMetaData buildQuickSearchIndexRunTable(String tableName, String backendName)
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName(tableName)
|
||||
.withLabel("Quick Search Index Run")
|
||||
.withBackendName(backendName)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("runType", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("errorCount", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("errorMessage", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Build the basepull indexing process metadata.
|
||||
***************************************************************************/
|
||||
private QProcessMetaData buildBasepullProcess(String processName)
|
||||
{
|
||||
return new QProcessMetaData()
|
||||
.withName(processName)
|
||||
.withLabel("Quick Search Basepull Index")
|
||||
.withStep(new QBackendStepMetaData()
|
||||
.withName("basepull")
|
||||
.withCode(new QCodeReference(BasepullIndexStep.class)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Build the full reindex process metadata with optional tableName input.
|
||||
***************************************************************************/
|
||||
private QProcessMetaData buildFullReindexProcess(String processName)
|
||||
{
|
||||
return new QProcessMetaData()
|
||||
.withName(processName)
|
||||
.withLabel("Quick Search Full Reindex")
|
||||
.withStep(new QBackendStepMetaData()
|
||||
.withName("fullReindex")
|
||||
.withCode(new QCodeReference(FullReindexStep.class)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Register post-insert, post-update, and post-delete customizers on each
|
||||
** discovered source table.
|
||||
***************************************************************************/
|
||||
private void registerCustomizers(QInstance qInstance, List<QuickSearchableTableConfig> discoveredTables)
|
||||
{
|
||||
for(QuickSearchableTableConfig tableConfig : discoveredTables)
|
||||
List<String> missing = new ArrayList<>();
|
||||
for(String fieldName : searchableFields)
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
QTableMetaData table = qInstance.getTable(tableName);
|
||||
|
||||
if(table == null)
|
||||
if(!table.getFields().containsKey(fieldName))
|
||||
{
|
||||
LOG.warn("Source table not found in QInstance; skipping customizer registration",
|
||||
"tableName", tableName);
|
||||
continue;
|
||||
missing.add(fieldName);
|
||||
}
|
||||
}
|
||||
if(!missing.isEmpty())
|
||||
{
|
||||
throw (new QException("Searchable field(s) " + missing + " do not exist on table [" + table.getName() + "]"));
|
||||
}
|
||||
|
||||
table.withCustomizer(TableCustomizers.POST_INSERT_RECORD,
|
||||
new QCodeReference(QuickSearchPostInsertCustomizer.class));
|
||||
if(basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField))
|
||||
{
|
||||
if(QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD.equals(basepullTimestampField))
|
||||
{
|
||||
LOG.warn("Table has no modifyDate field; incremental basepull is disabled for it (reconcile still covers it)", logPair("tableName", table.getName()));
|
||||
return (null);
|
||||
}
|
||||
throw (new QException("basepullTimestampField [" + basepullTimestampField + "] does not exist on table [" + table.getName() + "]"));
|
||||
}
|
||||
return (basepullTimestampField);
|
||||
}
|
||||
|
||||
table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD,
|
||||
new QCodeReference(QuickSearchPostUpdateCustomizer.class));
|
||||
|
||||
table.withCustomizer(TableCustomizers.POST_DELETE_RECORD,
|
||||
new QCodeReference(QuickSearchPostDeleteCustomizer.class));
|
||||
|
||||
/*******************************************************************************
|
||||
** Hidden fields are never indexed: their values would leak through
|
||||
** searchable text and highlights.
|
||||
*******************************************************************************/
|
||||
private static void removeHiddenFields(QTableMetaData table, List<String> searchableFields, Map<String, Integer> fieldWeights, Map<String, Boolean> fieldIncludeLabels)
|
||||
{
|
||||
List<String> hidden = new ArrayList<>();
|
||||
for(String fieldName : searchableFields)
|
||||
{
|
||||
if(table.getFields().containsKey(fieldName) && Boolean.TRUE.equals(table.getField(fieldName).getIsHidden()))
|
||||
{
|
||||
hidden.add(fieldName);
|
||||
}
|
||||
}
|
||||
if(!hidden.isEmpty())
|
||||
{
|
||||
LOG.warn("Hidden fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
|
||||
searchableFields.removeAll(hidden);
|
||||
hidden.forEach(fieldWeights::remove);
|
||||
hidden.forEach(fieldIncludeLabels::remove);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The artifact version, from the Maven-filtered quick-search.properties.
|
||||
*******************************************************************************/
|
||||
static String loadVersion()
|
||||
{
|
||||
try(InputStream inputStream = QuickSearchQBitProducer.class.getClassLoader().getResourceAsStream("quick-search.properties"))
|
||||
{
|
||||
if(inputStream != null)
|
||||
{
|
||||
Properties properties = new Properties();
|
||||
properties.load(inputStream);
|
||||
String version = properties.getProperty("version");
|
||||
if(StringUtils.hasContent(version) && !version.contains("${"))
|
||||
{
|
||||
return (version);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.debug("Could not read quick-search.properties", e);
|
||||
}
|
||||
return ("unknown");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the artifact version.
|
||||
*******************************************************************************/
|
||||
public static String getVersion()
|
||||
{
|
||||
return (VERSION);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Live state of one produced Quick Search QBit: its discovered tables, the
|
||||
** lazily built OpenSearch client and publisher, and a short-lived cache of
|
||||
** which tables are enabled.
|
||||
**
|
||||
** One runtime exists per produced QBit. It hangs off the QBit's config
|
||||
** (QuickSearchQBitConfig.getRuntime()), which the host's QInstance holds in
|
||||
** QBitMetaData, so steps, listeners and actions resolve it from QContext with
|
||||
** {@link #get()} rather than from JVM-global statics. Two QInstances in one
|
||||
** JVM therefore no longer share state.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchRuntime
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchRuntime.class);
|
||||
|
||||
private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
|
||||
|
||||
private final QuickSearchQBitConfig config;
|
||||
|
||||
private List<QuickSearchableTableConfig> discoveredTables = new ArrayList<>();
|
||||
private Map<String, QuickSearchableTableConfig> tablesByName = new LinkedHashMap<>();
|
||||
|
||||
private volatile QuickSearchOpenSearchClient client;
|
||||
private volatile IndexEventPublisher publisher;
|
||||
private volatile boolean indexReady = false;
|
||||
private volatile boolean closed = false;
|
||||
|
||||
private final Map<String, CachedFlag> enabledCache = new ConcurrentHashMap<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A cached enabled flag with the time it was read.
|
||||
*******************************************************************************/
|
||||
private record CachedFlag(boolean enabled, long readAtMillis)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
*******************************************************************************/
|
||||
public QuickSearchRuntime(QuickSearchQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve the runtime of the Quick Search QBit registered in the current
|
||||
** QContext's QInstance. Returns null when there is no QContext, no QInstance,
|
||||
** no Quick Search QBit, or the QBit has not been produced yet.
|
||||
*******************************************************************************/
|
||||
public static QuickSearchRuntime get()
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
if(qInstance == null || qInstance.getQBits() == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
for(QBitMetaData qBitMetaData : qInstance.getQBits().values())
|
||||
{
|
||||
if(qBitMetaData.getConfig() instanceof QuickSearchQBitConfig quickSearchConfig && quickSearchConfig.getRuntime() != null)
|
||||
{
|
||||
return (quickSearchConfig.getRuntime());
|
||||
}
|
||||
}
|
||||
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Like {@link #get()}, but throws when no runtime can be resolved.
|
||||
*******************************************************************************/
|
||||
public static QuickSearchRuntime getOrThrow() throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = get();
|
||||
if(runtime == null)
|
||||
{
|
||||
throw (new QException("Quick Search QBit is not registered in the current QInstance (or has not been produced)"));
|
||||
}
|
||||
return (runtime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for config
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig getConfig()
|
||||
{
|
||||
return (this.config);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for discoveredTables (never null)
|
||||
*******************************************************************************/
|
||||
public List<QuickSearchableTableConfig> getDiscoveredTables()
|
||||
{
|
||||
return (Collections.unmodifiableList(this.discoveredTables));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Replace the discovered tables.
|
||||
*******************************************************************************/
|
||||
public void setDiscoveredTables(List<QuickSearchableTableConfig> discoveredTables)
|
||||
{
|
||||
this.discoveredTables = new ArrayList<>(discoveredTables == null ? List.of() : discoveredTables);
|
||||
this.tablesByName = new LinkedHashMap<>();
|
||||
for(QuickSearchableTableConfig tableConfig : this.discoveredTables)
|
||||
{
|
||||
this.tablesByName.put(tableConfig.getTableName(), tableConfig);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Find the config for one table, or null.
|
||||
*******************************************************************************/
|
||||
public QuickSearchableTableConfig getTableConfig(String tableName)
|
||||
{
|
||||
return (tableName == null ? null : tablesByName.get(tableName));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return the OpenSearch client, building it on first use. Building the
|
||||
** client does not contact the cluster; {@link #ensureIndexReady()} does.
|
||||
*******************************************************************************/
|
||||
public QuickSearchOpenSearchClient getClient() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient existing = client;
|
||||
if(existing != null)
|
||||
{
|
||||
return (existing);
|
||||
}
|
||||
|
||||
synchronized(this)
|
||||
{
|
||||
if(client == null)
|
||||
{
|
||||
client = new QuickSearchOpenSearchClient(config);
|
||||
}
|
||||
return (client);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Make sure the index (or alias) exists with the current mapping. Runs the
|
||||
** check once per runtime; a failure leaves it unready so the next call
|
||||
** retries.
|
||||
*******************************************************************************/
|
||||
public void ensureIndexReady() throws QException
|
||||
{
|
||||
if(indexReady)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
synchronized(this)
|
||||
{
|
||||
if(!indexReady)
|
||||
{
|
||||
getClient().ensureIndexExists();
|
||||
indexReady = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether {@link #ensureIndexReady()} has succeeded.
|
||||
*******************************************************************************/
|
||||
public boolean isIndexReady()
|
||||
{
|
||||
return (indexReady);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return the publisher: the host-supplied one from config, else a
|
||||
** SynchronousIndexEventPublisher over this runtime's client.
|
||||
*******************************************************************************/
|
||||
public IndexEventPublisher getPublisher() throws QException
|
||||
{
|
||||
IndexEventPublisher existing = publisher;
|
||||
if(existing != null)
|
||||
{
|
||||
return (existing);
|
||||
}
|
||||
|
||||
synchronized(this)
|
||||
{
|
||||
if(publisher == null)
|
||||
{
|
||||
if(config.getIndexEventPublisher() != null)
|
||||
{
|
||||
publisher = config.getIndexEventPublisher();
|
||||
}
|
||||
else
|
||||
{
|
||||
publisher = new SynchronousIndexEventPublisher(getClient(), discoveredTables, config.getBulkBatchSize());
|
||||
}
|
||||
}
|
||||
return (publisher);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether a table is enabled for indexing and search, per its
|
||||
** quickSearchIndex row. A table with no row yet is enabled. Results are
|
||||
** cached for one minute; a failed lookup is treated as enabled so indexing
|
||||
** does not stop because the operational table is unreachable.
|
||||
*******************************************************************************/
|
||||
public boolean isTableEnabled(String tableName)
|
||||
{
|
||||
if(tableName == null || !tablesByName.containsKey(tableName))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
long now = System.currentTimeMillis();
|
||||
CachedFlag cached = enabledCache.get(tableName);
|
||||
if(cached != null && now - cached.readAtMillis() < ENABLED_CACHE_TTL_MILLIS)
|
||||
{
|
||||
return (cached.enabled());
|
||||
}
|
||||
|
||||
boolean enabled = readEnabledFlag(tableName);
|
||||
enabledCache.put(tableName, new CachedFlag(enabled, now));
|
||||
return (enabled);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forget cached enabled flags (after an operator toggles a table).
|
||||
*******************************************************************************/
|
||||
public void invalidateEnabledCache()
|
||||
{
|
||||
enabledCache.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the enabled flag from the quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
private boolean readEnabledFlag(String tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(QContext.getQInstance() == null || QContext.getQInstance().getTable(config.getQuickSearchIndexTableName()) == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(config.getQuickSearchIndexTableName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
|
||||
if(rows == null || rows.isEmpty())
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
Boolean enabled = rows.get(0).getValueBoolean("enabled");
|
||||
return (enabled == null || enabled);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not read enabled flag for table; treating as enabled", e, logPair("tableName", tableName));
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start the runtime per the configured startup mode: FAIL_FAST throws when
|
||||
** the index cannot be prepared; DEGRADED logs and continues. A LinkageError
|
||||
** (an optional library such as the AWS SDK missing while the client is
|
||||
** built) follows the same modes, with a message naming the missing class,
|
||||
** so a DEGRADED host still boots.
|
||||
*******************************************************************************/
|
||||
public void start() throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
ensureIndexReady();
|
||||
}
|
||||
catch(LinkageError e)
|
||||
{
|
||||
String message = "Quick Search could not load a class to build its OpenSearch client [" + e.getMessage()
|
||||
+ "]; authMode AWS_SIGV4 needs the optional AWS SDK v2 dependencies (apache-client, sts) on the classpath";
|
||||
if(config.getStartupMode() == QuickSearchStartupMode.DEGRADED)
|
||||
{
|
||||
LOG.warn(message + "; Quick Search is running degraded", e, logPair("indexName", config.getOpensearchIndexName()));
|
||||
return;
|
||||
}
|
||||
|
||||
throw (new QException(message, e));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
if(config.getStartupMode() == QuickSearchStartupMode.DEGRADED)
|
||||
{
|
||||
LOG.warn("OpenSearch is not reachable; Quick Search is running degraded and will retry on first use", e,
|
||||
logPair("indexName", config.getOpensearchIndexName()));
|
||||
return;
|
||||
}
|
||||
|
||||
throw (new QException("Quick Search could not prepare its OpenSearch index [" + config.getOpensearchIndexName()
|
||||
+ "]; set startupMode=DEGRADED to boot without it: " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Close the publisher and client. Safe to call more than once.
|
||||
*******************************************************************************/
|
||||
public synchronized void close()
|
||||
{
|
||||
if(closed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
|
||||
if(publisher != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
publisher.close();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Error closing Quick Search publisher", e);
|
||||
}
|
||||
}
|
||||
|
||||
if(client != null)
|
||||
{
|
||||
client.close();
|
||||
}
|
||||
|
||||
indexReady = false;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Lifecycle hook for hosts that start through QApplicationLauncher: prepares
|
||||
** the OpenSearch index on start (per startupMode) and closes the transport on
|
||||
** stop. Hosts without a launcher get the same start behaviour from produce()
|
||||
** in FAIL_FAST mode and lazily on first use in DEGRADED mode.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchRuntimeService implements QRuntimeServiceInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchRuntimeService.class);
|
||||
|
||||
private QuickSearchRuntime runtime;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getName()
|
||||
{
|
||||
return ("quickSearch");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void start(QInstance qInstance) throws QException
|
||||
{
|
||||
runtime = findRuntime(qInstance);
|
||||
if(runtime == null)
|
||||
{
|
||||
LOG.warn("Quick Search runtime service started, but no produced Quick Search QBit was found in the QInstance");
|
||||
return;
|
||||
}
|
||||
runtime.start();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void stop()
|
||||
{
|
||||
if(runtime != null)
|
||||
{
|
||||
runtime.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Find the runtime on the instance's Quick Search QBit (not through
|
||||
** QContext, which may not be set when the launcher starts services).
|
||||
*******************************************************************************/
|
||||
static QuickSearchRuntime findRuntime(QInstance qInstance)
|
||||
{
|
||||
if(qInstance == null || qInstance.getQBits() == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
for(QBitMetaData qBitMetaData : qInstance.getQBits().values())
|
||||
{
|
||||
if(qBitMetaData.getConfig() instanceof QuickSearchQBitConfig config && config.getRuntime() != null)
|
||||
{
|
||||
return (config.getRuntime());
|
||||
}
|
||||
}
|
||||
return (null);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What happens when OpenSearch cannot be reached while the QBit starts.
|
||||
**
|
||||
** FAIL_FAST - produce() and the runtime service throw, so the host does not
|
||||
** boot with search silently broken. The default.
|
||||
** DEGRADED - the failure is logged; the host boots; indexing and search
|
||||
** retry the connection on first use and report through the
|
||||
** health check. Intended for demo profiles that may run without
|
||||
** an OpenSearch cluster.
|
||||
*******************************************************************************/
|
||||
public enum QuickSearchStartupMode
|
||||
{
|
||||
FAIL_FAST,
|
||||
DEGRADED
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** TLS trust and client-certificate settings for the OpenSearch connection.
|
||||
**
|
||||
** By default the JVM trust store and hostname verification are used. Supply
|
||||
** caCertificatePath (a PEM bundle) or trustStorePath to trust a private CA;
|
||||
** supply keyStorePath for client-certificate (mTLS) authentication.
|
||||
** insecureSkipVerify disables certificate and hostname checks and is refused
|
||||
** for non-loopback hosts unless allowInsecureInProduction is also set.
|
||||
** hostnameVerification=false keeps certificate checks but skips the host name
|
||||
** check; it is allowed for any host and logs a warning.
|
||||
**
|
||||
** Path and password values may be ${env.X} / ${prop.X} references; they are
|
||||
** interpreted when the transport is built, never stored resolved. A path that
|
||||
** resolves to an empty value is treated as missing.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchTlsConfig implements Serializable
|
||||
{
|
||||
private String caCertificatePath;
|
||||
private String trustStorePath;
|
||||
private String trustStorePassword;
|
||||
private String trustStoreType;
|
||||
private String keyStorePath;
|
||||
private String keyStorePassword;
|
||||
private String keyStoreType;
|
||||
private Boolean hostnameVerification = true;
|
||||
private Boolean insecureSkipVerify = false;
|
||||
private Boolean allowInsecureInProduction = false;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether any custom trust or key material is configured.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public boolean hasCustomTrustOrKeyMaterial()
|
||||
{
|
||||
return (caCertificatePath != null || trustStorePath != null || keyStorePath != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for caCertificatePath
|
||||
*******************************************************************************/
|
||||
public String getCaCertificatePath()
|
||||
{
|
||||
return (this.caCertificatePath);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for caCertificatePath
|
||||
*******************************************************************************/
|
||||
public void setCaCertificatePath(String caCertificatePath)
|
||||
{
|
||||
this.caCertificatePath = caCertificatePath;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for caCertificatePath
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withCaCertificatePath(String caCertificatePath)
|
||||
{
|
||||
this.caCertificatePath = caCertificatePath;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for trustStorePath
|
||||
*******************************************************************************/
|
||||
public String getTrustStorePath()
|
||||
{
|
||||
return (this.trustStorePath);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for trustStorePath
|
||||
*******************************************************************************/
|
||||
public void setTrustStorePath(String trustStorePath)
|
||||
{
|
||||
this.trustStorePath = trustStorePath;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for trustStorePath
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withTrustStorePath(String trustStorePath)
|
||||
{
|
||||
this.trustStorePath = trustStorePath;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for trustStorePassword. Hidden from JSON so the secret (or its
|
||||
** reference) never leaves the JVM.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public String getTrustStorePassword()
|
||||
{
|
||||
return (this.trustStorePassword);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for trustStorePassword
|
||||
*******************************************************************************/
|
||||
public void setTrustStorePassword(String trustStorePassword)
|
||||
{
|
||||
this.trustStorePassword = trustStorePassword;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for trustStorePassword
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withTrustStorePassword(String trustStorePassword)
|
||||
{
|
||||
this.trustStorePassword = trustStorePassword;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for trustStoreType (JKS or PKCS12; defaults to the JVM default)
|
||||
*******************************************************************************/
|
||||
public String getTrustStoreType()
|
||||
{
|
||||
return (this.trustStoreType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for trustStoreType
|
||||
*******************************************************************************/
|
||||
public void setTrustStoreType(String trustStoreType)
|
||||
{
|
||||
this.trustStoreType = trustStoreType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for trustStoreType
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withTrustStoreType(String trustStoreType)
|
||||
{
|
||||
this.trustStoreType = trustStoreType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for keyStorePath
|
||||
*******************************************************************************/
|
||||
public String getKeyStorePath()
|
||||
{
|
||||
return (this.keyStorePath);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for keyStorePath
|
||||
*******************************************************************************/
|
||||
public void setKeyStorePath(String keyStorePath)
|
||||
{
|
||||
this.keyStorePath = keyStorePath;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for keyStorePath
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withKeyStorePath(String keyStorePath)
|
||||
{
|
||||
this.keyStorePath = keyStorePath;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for keyStorePassword. Hidden from JSON.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public String getKeyStorePassword()
|
||||
{
|
||||
return (this.keyStorePassword);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for keyStorePassword
|
||||
*******************************************************************************/
|
||||
public void setKeyStorePassword(String keyStorePassword)
|
||||
{
|
||||
this.keyStorePassword = keyStorePassword;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for keyStorePassword
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withKeyStorePassword(String keyStorePassword)
|
||||
{
|
||||
this.keyStorePassword = keyStorePassword;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for keyStoreType
|
||||
*******************************************************************************/
|
||||
public String getKeyStoreType()
|
||||
{
|
||||
return (this.keyStoreType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for keyStoreType
|
||||
*******************************************************************************/
|
||||
public void setKeyStoreType(String keyStoreType)
|
||||
{
|
||||
this.keyStoreType = keyStoreType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for keyStoreType
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withKeyStoreType(String keyStoreType)
|
||||
{
|
||||
this.keyStoreType = keyStoreType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for hostnameVerification
|
||||
*******************************************************************************/
|
||||
public Boolean getHostnameVerification()
|
||||
{
|
||||
return (this.hostnameVerification);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for hostnameVerification
|
||||
*******************************************************************************/
|
||||
public void setHostnameVerification(Boolean hostnameVerification)
|
||||
{
|
||||
this.hostnameVerification = hostnameVerification;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for hostnameVerification
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withHostnameVerification(Boolean hostnameVerification)
|
||||
{
|
||||
this.hostnameVerification = hostnameVerification;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for insecureSkipVerify
|
||||
*******************************************************************************/
|
||||
public Boolean getInsecureSkipVerify()
|
||||
{
|
||||
return (this.insecureSkipVerify);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for insecureSkipVerify
|
||||
*******************************************************************************/
|
||||
public void setInsecureSkipVerify(Boolean insecureSkipVerify)
|
||||
{
|
||||
this.insecureSkipVerify = insecureSkipVerify;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for insecureSkipVerify
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withInsecureSkipVerify(Boolean insecureSkipVerify)
|
||||
{
|
||||
this.insecureSkipVerify = insecureSkipVerify;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for allowInsecureInProduction
|
||||
*******************************************************************************/
|
||||
public Boolean getAllowInsecureInProduction()
|
||||
{
|
||||
return (this.allowInsecureInProduction);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for allowInsecureInProduction
|
||||
*******************************************************************************/
|
||||
public void setAllowInsecureInProduction(Boolean allowInsecureInProduction)
|
||||
{
|
||||
this.allowInsecureInProduction = allowInsecureInProduction;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for allowInsecureInProduction
|
||||
*******************************************************************************/
|
||||
public QuickSearchTlsConfig withAllowInsecureInProduction(Boolean allowInsecureInProduction)
|
||||
{
|
||||
this.allowInsecureInProduction = allowInsecureInProduction;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -39,6 +39,7 @@ public class QuickSearchableTableConfig
|
||||
private Boolean enabledByDefault;
|
||||
private String recordLabelFormat;
|
||||
private List<String> recordLabelFields;
|
||||
private Integer maxFieldLength;
|
||||
|
||||
|
||||
|
||||
@@ -350,4 +351,35 @@ public class QuickSearchableTableConfig
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for maxFieldLength
|
||||
*******************************************************************************/
|
||||
public Integer getMaxFieldLength()
|
||||
{
|
||||
return (this.maxFieldLength);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for maxFieldLength
|
||||
*******************************************************************************/
|
||||
public void setMaxFieldLength(Integer maxFieldLength)
|
||||
{
|
||||
this.maxFieldLength = maxFieldLength;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for maxFieldLength
|
||||
*******************************************************************************/
|
||||
public QuickSearchableTableConfig withMaxFieldLength(Integer maxFieldLength)
|
||||
{
|
||||
this.maxFieldLength = maxFieldLength;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,116 +17,520 @@
|
||||
package com.kingsrook.qbits.quicksearch.actions;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
import org.opensearch.client.opensearch._types.OpenSearchException;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.opensearch.client.opensearch.core.search.Hit;
|
||||
import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Action that executes a quick-search query against OpenSearch and returns
|
||||
** normalized, paginated results.
|
||||
** Permission-aware, bounded search over the Quick Search index.
|
||||
**
|
||||
** Validates and normalizes the input, delegates to the
|
||||
** QuickSearchOpenSearchClient, maps each hit to a QuickSearchResult, and
|
||||
** computes pagination metadata (totalHits, hasMore).
|
||||
** Only tables the session may read (table READ permission), that are
|
||||
** configured and enabled, are searched. With applyRecordSecurityLocks (the
|
||||
** default) hits are re-read through QueryAction by primary key, so record
|
||||
** security locks apply and no label or snippet is returned for a record the
|
||||
** session cannot query.
|
||||
**
|
||||
** limit is capped at maxSearchLimit and offset + limit at OpenSearch's result
|
||||
** window. tableNames restricts the search; limitPerTable searches each table
|
||||
** separately and returns up to that many hits per table.
|
||||
**
|
||||
** offset counts results the caller can see. Without locks it is passed
|
||||
** straight to OpenSearch. With locks, hits are read from position 0 in
|
||||
** batches that double in size, filtered, the first offset accessible results
|
||||
** are skipped and limit are kept, so consecutive pages neither overlap nor
|
||||
** skip an accessible hit. No request reads past MAX_RESULT_WINDOW (10,000
|
||||
** raw hits), so accessible results beyond it are not reachable.
|
||||
**
|
||||
** With locks, each scan has a hard budget of raw hits, read in batches of
|
||||
** at most MAX_LOCKED_BATCH_SIZE, and of OpenSearch requests, each followed
|
||||
** by one QueryAction per table in the batch. A single-list search gets
|
||||
** MAX_RESULT_WINDOW hits and MAX_LOCKED_ROUND_TRIPS requests. In per-table
|
||||
** mode each of the T tables gets its own equal share,
|
||||
** max(MAX_RESULT_WINDOW / T, min(MAX_RESULT_WINDOW, 2 x limitPerTable)) hits
|
||||
** and max(2, MAX_LOCKED_ROUND_TRIPS / T) requests, so a fully locked table
|
||||
** cannot starve another and one call reads at most
|
||||
** max(MAX_RESULT_WINDOW, T x 2 x limitPerTable) raw hits over at most
|
||||
** max(MAX_LOCKED_ROUND_TRIPS, 2 x T) requests; offset never raises it.
|
||||
** A budget ends a scan like the result window does: hasMore is false and
|
||||
** the total is a lower bound, so in per-table mode paging within a table
|
||||
** stops past max(MAX_RESULT_WINDOW / T, 2 x limitPerTable) raw hits. Hits
|
||||
** are deduplicated by table and record id within a call; batches are
|
||||
** separate from/size requests without a tiebreak sort, so a hit that moves
|
||||
** between batches during a refresh can still be missed.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchAction
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchAction.class);
|
||||
private static final int DEFAULT_LIMIT = 25;
|
||||
private static final int DEFAULT_OFFSET = 0;
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchAction.class);
|
||||
|
||||
public static final int DEFAULT_LIMIT = 25;
|
||||
public static final int MIN_TERM_LENGTH = 2;
|
||||
public static final int MAX_TERM_LENGTH = 100;
|
||||
public static final int MAX_LIMIT_PER_TABLE = 100;
|
||||
|
||||
public static final int MAX_LOCKED_ROUND_TRIPS = 32;
|
||||
public static final int MAX_LOCKED_BATCH_SIZE = 1_000;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute a quick-search for the given input.
|
||||
**
|
||||
** Returns an empty output (totalHits=0, hasMore=false, empty results list)
|
||||
** when the search term is null or blank. Otherwise normalizes the term,
|
||||
** applies default pagination values, calls the OpenSearch client, and maps
|
||||
** hits to QuickSearchResult objects.
|
||||
**
|
||||
** @param input the search parameters; must not be null
|
||||
** @return populated QuickSearchOutput
|
||||
** @throws QException if the OpenSearch call fails
|
||||
*******************************************************************************/
|
||||
public QuickSearchOutput execute(QuickSearchInput input) throws QException
|
||||
{
|
||||
String searchTerm = input.getSearchTerm();
|
||||
|
||||
if(searchTerm == null || searchTerm.isBlank())
|
||||
{
|
||||
LOG.debug("QuickSearch called with blank/null search term - returning empty output");
|
||||
return (new QuickSearchOutput()
|
||||
.withResults(Collections.emptyList())
|
||||
.withTotalHits(0L)
|
||||
.withHasMore(false));
|
||||
return (emptyOutput());
|
||||
}
|
||||
|
||||
String normalizedTerm = IndexingUtils.normalizeSearchText(searchTerm);
|
||||
if(normalizedTerm.length() < MIN_TERM_LENGTH)
|
||||
{
|
||||
return (emptyOutput());
|
||||
}
|
||||
if(normalizedTerm.length() > MAX_TERM_LENGTH)
|
||||
{
|
||||
throw (new QException("Search term is too long (maximum " + MAX_TERM_LENGTH + " characters)"));
|
||||
}
|
||||
|
||||
int limit = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : input.getLimit();
|
||||
int offset = (input.getOffset() == null || input.getOffset() < 0) ? DEFAULT_OFFSET : input.getOffset();
|
||||
|
||||
QuickSearchOpenSearchClient client = QuickSearchQBitContext.getClient();
|
||||
List<QuickSearchableTableConfig> tableConfigs = QuickSearchQBitContext.getDiscoveredTables();
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
QuickSearchQBitConfig config = runtime != null ? runtime.getConfig() : QuickSearchQBitContext.getConfig();
|
||||
QuickSearchOpenSearchClient client = runtime != null ? runtime.getClient() : QuickSearchQBitContext.getClient();
|
||||
List<QuickSearchableTableConfig> tableConfigs = runtime != null ? runtime.getDiscoveredTables() : QuickSearchQBitContext.getDiscoveredTables();
|
||||
|
||||
if(client == null)
|
||||
{
|
||||
throw new QException("QuickSearch client is not initialized. The QBit may not have connected to OpenSearch during startup.");
|
||||
}
|
||||
if(tableConfigs == null)
|
||||
{
|
||||
tableConfigs = Collections.emptyList();
|
||||
}
|
||||
|
||||
LOG.debug("Executing quick search", "term", normalizedTerm, "tableName", input.getTableName(), "limit", limit, "offset", offset);
|
||||
int maxLimit = (config == null || config.getMaxSearchLimit() == null) ? MAX_LIMIT_PER_TABLE : config.getMaxSearchLimit();
|
||||
int limit = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : Math.min(input.getLimit(), maxLimit);
|
||||
int offset = (input.getOffset() == null || input.getOffset() < 0) ? 0 : input.getOffset();
|
||||
|
||||
SearchResponse<OpenSearchDocument> response = client.search(normalizedTerm, input.getTableName(), limit, offset, tableConfigs);
|
||||
if(offset >= QuickSearchOpenSearchClient.MAX_RESULT_WINDOW)
|
||||
{
|
||||
return (emptyOutput());
|
||||
}
|
||||
|
||||
List<QuickSearchResult> results = new ArrayList<>();
|
||||
List<String> allowedTables = resolveAllowedTables(input, runtime, tableConfigs);
|
||||
if(allowedTables.isEmpty())
|
||||
{
|
||||
return (emptyOutput());
|
||||
}
|
||||
|
||||
boolean applyLocks = config == null || Boolean.TRUE.equals(config.getApplyRecordSecurityLocks());
|
||||
|
||||
LOG.debug("Executing quick search", logPair("termLength", normalizedTerm.length()), logPair("tables", allowedTables.size()), logPair("limit", limit), logPair("offset", offset));
|
||||
|
||||
if(input.getLimitPerTable() != null && input.getLimitPerTable() > 0)
|
||||
{
|
||||
return (searchPerTable(client, normalizedTerm, allowedTables, offset, Math.min(input.getLimitPerTable(), maxLimit), tableConfigs, applyLocks));
|
||||
}
|
||||
|
||||
Page page = searchPage(client, normalizedTerm, allowedTables, offset, limit, tableConfigs, applyLocks, new ScanBudget(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW, MAX_LOCKED_ROUND_TRIPS));
|
||||
return (page.toOutput());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Search each allowed table separately, skipping offset results and
|
||||
** returning up to limitPerTable hits in each table.
|
||||
*******************************************************************************/
|
||||
private QuickSearchOutput searchPerTable(QuickSearchOpenSearchClient client, String term, List<String> allowedTables, int offset, int limitPerTable, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks) throws QException
|
||||
{
|
||||
List<QuickSearchResult> results = new ArrayList<>();
|
||||
long totalHits = 0;
|
||||
boolean hasMore = false;
|
||||
boolean lowerBound = false;
|
||||
|
||||
int window = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
|
||||
int tableHits = Math.max(window / allowedTables.size(), Math.min(window, 2 * limitPerTable));
|
||||
int tableTrips = Math.max(2, MAX_LOCKED_ROUND_TRIPS / allowedTables.size());
|
||||
|
||||
for(String tableName : allowedTables)
|
||||
{
|
||||
Page page = searchPage(client, term, List.of(tableName), offset, limitPerTable, tableConfigs, applyLocks, new ScanBudget(tableHits, tableTrips));
|
||||
results.addAll(page.results());
|
||||
totalHits += page.totalHits();
|
||||
hasMore |= page.hasMore();
|
||||
lowerBound |= page.lowerBound();
|
||||
}
|
||||
|
||||
return (new QuickSearchOutput().withResults(results).withTotalHits(totalHits).withTotalHitsIsLowerBound(lowerBound).withHasMore(hasMore));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One page over the given tables: skip offset results, keep up to limit.
|
||||
** Without locks offset goes straight to OpenSearch; with locks offset and
|
||||
** limit count accessible results within one scan budget (see the class
|
||||
** comment). Every request keeps from + size within MAX_RESULT_WINDOW.
|
||||
*******************************************************************************/
|
||||
private Page searchPage(QuickSearchOpenSearchClient client, String term, List<String> tables, int offset, int limit, List<QuickSearchableTableConfig> tableConfigs, boolean applyLocks, ScanBudget budget) throws QException
|
||||
{
|
||||
int window = QuickSearchOpenSearchClient.MAX_RESULT_WINDOW;
|
||||
int pageSize = Math.min(limit, window - offset);
|
||||
if(pageSize <= 0)
|
||||
{
|
||||
return (new Page(Collections.emptyList(), 0L, false, false));
|
||||
}
|
||||
|
||||
if(!applyLocks)
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, pageSize, offset, tableConfigs, null);
|
||||
List<QuickSearchResult> results = toResults(response);
|
||||
|
||||
long totalHits = response.hits().total() != null ? response.hits().total().value() : results.size();
|
||||
boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
|
||||
return (new Page(results, totalHits, lowerBound, (offset + results.size()) < totalHits));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// read candidates from position 0 until offset + pageSize accessible //
|
||||
// results are known, the hits run out, the result window is reached, or //
|
||||
// the scan budget is spent. each batch over-reads 2x what is still //
|
||||
// needed and at least doubles the last one, up to MAX_LOCKED_BATCH_SIZE, //
|
||||
// and the last round trip takes up to MAX_LOCKED_BATCH_SIZE of the rest. //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
int target = offset + pageSize;
|
||||
List<QuickSearchResult> accessible = new ArrayList<>();
|
||||
int position = 0;
|
||||
int batchSize = 0;
|
||||
boolean exhausted = false;
|
||||
boolean rawFloor = false;
|
||||
Set<String> seen = new HashSet<>();
|
||||
String preference = "quick-search-" + UUID.randomUUID();
|
||||
|
||||
while(accessible.size() < target && !exhausted && position < window && budget.hasRemaining())
|
||||
{
|
||||
int ceiling = Math.min(MAX_LOCKED_BATCH_SIZE, Math.min(window - position, budget.remainingHits));
|
||||
batchSize = budget.remainingRoundTrips == 1 ? ceiling : Math.min(ceiling, Math.max(2 * (target - accessible.size()), 2 * batchSize));
|
||||
|
||||
SearchResponse<OpenSearchDocument> response = runSearch(client, term, tables, batchSize, position, tableConfigs, preference);
|
||||
int rawHits = response.hits().hits().size();
|
||||
long rawTotal = response.hits().total() != null ? response.hits().total().value() : position + rawHits;
|
||||
rawFloor = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
|
||||
|
||||
List<QuickSearchResult> candidates = new ArrayList<>();
|
||||
for(QuickSearchResult result : toResults(response))
|
||||
{
|
||||
if(seen.add(result.getTableName() + ":" + result.getRecordId()))
|
||||
{
|
||||
candidates.add(result);
|
||||
}
|
||||
}
|
||||
accessible.addAll(filterByRecordAccess(candidates, tableConfigs));
|
||||
position += rawHits;
|
||||
budget.spend(rawHits);
|
||||
exhausted = rawHits < batchSize || position >= rawTotal;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// the raw total would reveal how many matching rows the session may not //
|
||||
// see; report only what was verified accessible. hasMore says only that //
|
||||
// more candidates exist, so a caller can page on to accessible rows. //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
boolean truncated = accessible.size() > target;
|
||||
boolean hasMore = truncated || (!exhausted && position < window && budget.hasRemaining());
|
||||
|
||||
List<QuickSearchResult> results = accessible.size() > offset
|
||||
? new ArrayList<>(accessible.subList(offset, Math.min(target, accessible.size())))
|
||||
: new ArrayList<>();
|
||||
|
||||
return (new Page(results, (long) accessible.size(), !exhausted || rawFloor, hasMore));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Requested tables (tableNames, else tableName, else all configured) that
|
||||
** are configured, enabled, and readable by the session.
|
||||
*******************************************************************************/
|
||||
private List<String> resolveAllowedTables(QuickSearchInput input, QuickSearchRuntime runtime, List<QuickSearchableTableConfig> tableConfigs)
|
||||
{
|
||||
Set<String> configured = new HashSet<>();
|
||||
for(QuickSearchableTableConfig tableConfig : tableConfigs)
|
||||
{
|
||||
configured.add(tableConfig.getTableName());
|
||||
}
|
||||
|
||||
List<String> requested = new ArrayList<>();
|
||||
if(!CollectionUtils.nullSafeIsEmpty(input.getTableNames()))
|
||||
{
|
||||
requested.addAll(input.getTableNames());
|
||||
}
|
||||
else if(StringUtils.hasContent(input.getTableName()))
|
||||
{
|
||||
requested.add(input.getTableName());
|
||||
}
|
||||
else
|
||||
{
|
||||
requested.addAll(configured);
|
||||
}
|
||||
|
||||
boolean checkPermissions = QContext.getQInstance() != null && QContext.getQSession() != null;
|
||||
List<String> allowed = new ArrayList<>();
|
||||
for(String tableName : new LinkedHashSet<>(requested))
|
||||
{
|
||||
if(!configured.contains(tableName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(runtime != null && !runtime.isTableEnabled(tableName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(checkPermissions && !PermissionsHelper.hasTablePermission(input, tableName, TablePermissionSubType.READ))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
allowed.add(tableName);
|
||||
}
|
||||
|
||||
return (allowed);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep only hits whose record the session can query (record security
|
||||
** locks). Tables not in the QInstance are left as they are.
|
||||
*******************************************************************************/
|
||||
private List<QuickSearchResult> filterByRecordAccess(List<QuickSearchResult> results, List<QuickSearchableTableConfig> tableConfigs) throws QException
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
if(qInstance == null || QContext.getQSession() == null || results.isEmpty())
|
||||
{
|
||||
return (results);
|
||||
}
|
||||
|
||||
Map<String, List<QuickSearchResult>> byTable = new LinkedHashMap<>();
|
||||
for(QuickSearchResult result : results)
|
||||
{
|
||||
byTable.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
|
||||
}
|
||||
|
||||
Set<QuickSearchResult> accessible = new HashSet<>();
|
||||
for(Map.Entry<String, List<QuickSearchResult>> entry : byTable.entrySet())
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(entry.getKey());
|
||||
if(table == null)
|
||||
{
|
||||
accessible.addAll(entry.getValue());
|
||||
continue;
|
||||
}
|
||||
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
for(QuickSearchResult result : entry.getValue())
|
||||
{
|
||||
ids.add(result.getRecordId());
|
||||
}
|
||||
|
||||
Set<String> found = new HashSet<>();
|
||||
for(int start = 0; start < ids.size(); start += MAX_LOCKED_BATCH_SIZE)
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(table.getName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_LOCKED_BATCH_SIZE)))));
|
||||
|
||||
for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
found.add(record.getValueString(table.getPrimaryKeyField()));
|
||||
}
|
||||
}
|
||||
|
||||
for(QuickSearchResult result : entry.getValue())
|
||||
{
|
||||
if(found.contains(result.getRecordId()))
|
||||
{
|
||||
accessible.add(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<QuickSearchResult> filtered = new ArrayList<>();
|
||||
for(QuickSearchResult result : results)
|
||||
{
|
||||
if(accessible.contains(result))
|
||||
{
|
||||
filtered.add(result);
|
||||
}
|
||||
}
|
||||
return (filtered);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QuickSearchResult> toResults(SearchResponse<OpenSearchDocument> response)
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
List<QuickSearchResult> results = new ArrayList<>();
|
||||
|
||||
for(Hit<OpenSearchDocument> hit : response.hits().hits())
|
||||
{
|
||||
OpenSearchDocument source = hit.source();
|
||||
if(source == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
String highlightSnippet = null;
|
||||
Map<String, List<String>> highlight = hit.highlight();
|
||||
|
||||
String highlightSnippet = null;
|
||||
Map<String, List<String>> highlight = hit.highlight();
|
||||
if(highlight != null)
|
||||
{
|
||||
List<String> fragments = highlight.get("searchableText");
|
||||
|
||||
if(fragments != null && !fragments.isEmpty())
|
||||
{
|
||||
highlightSnippet = String.join("...", fragments);
|
||||
}
|
||||
}
|
||||
|
||||
Double rawScore = hit.score();
|
||||
Float score = rawScore != null ? rawScore.floatValue() : null;
|
||||
QTableMetaData table = qInstance == null ? null : qInstance.getTable(source.getSourceTable());
|
||||
String tableLabel = table != null && StringUtils.hasContent(table.getLabel()) ? table.getLabel() : source.getSourceTable();
|
||||
Double rawScore = hit.score();
|
||||
|
||||
results.add(new QuickSearchResult()
|
||||
.withTableName(source.getSourceTable())
|
||||
.withTableLabel(tableLabel)
|
||||
.withRecordId(source.getRecordId())
|
||||
.withRecordLabel(source.getRecordLabel())
|
||||
.withScore(score)
|
||||
.withScore(rawScore != null ? rawScore.floatValue() : null)
|
||||
.withHighlightSnippet(highlightSnippet));
|
||||
}
|
||||
|
||||
long totalHits = response.hits().total() != null ? response.hits().total().value() : results.size();
|
||||
boolean hasMore = (offset + results.size()) < totalHits;
|
||||
return (results);
|
||||
}
|
||||
|
||||
return (new QuickSearchOutput()
|
||||
.withResults(results)
|
||||
.withTotalHits(totalHits)
|
||||
.withHasMore(hasMore));
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run one search, surfacing an OpenSearch client error as a QException so
|
||||
** callers never see an unchecked OpenSearchException.
|
||||
*******************************************************************************/
|
||||
private static SearchResponse<OpenSearchDocument> runSearch(QuickSearchOpenSearchClient client, String term, List<String> tables, int fetchSize, int offset, List<QuickSearchableTableConfig> tableConfigs, String preference) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (client.search(term, tables, fetchSize, offset, tableConfigs, preference));
|
||||
}
|
||||
catch(OpenSearchException e)
|
||||
{
|
||||
throw (new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Work budget for one locked scan: raw hits fetched and OpenSearch round
|
||||
** trips.
|
||||
*******************************************************************************/
|
||||
private static final class ScanBudget
|
||||
{
|
||||
private Integer remainingHits;
|
||||
private Integer remainingRoundTrips;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
*******************************************************************************/
|
||||
ScanBudget(Integer hits, Integer roundTrips)
|
||||
{
|
||||
this.remainingHits = hits;
|
||||
this.remainingRoundTrips = roundTrips;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
boolean hasRemaining()
|
||||
{
|
||||
return (remainingHits > 0 && remainingRoundTrips > 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record one round trip that fetched the given number of raw hits.
|
||||
*******************************************************************************/
|
||||
void spend(int rawHits)
|
||||
{
|
||||
remainingHits -= rawHits;
|
||||
remainingRoundTrips--;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One page of results with its total and paging flags.
|
||||
*******************************************************************************/
|
||||
private record Page(List<QuickSearchResult> results, Long totalHits, Boolean lowerBound, Boolean hasMore)
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
QuickSearchOutput toOutput()
|
||||
{
|
||||
return (new QuickSearchOutput().withResults(results).withTotalHits(totalHits).withTotalHitsIsLowerBound(lowerBound).withHasMore(hasMore));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static QuickSearchOutput emptyOutput()
|
||||
{
|
||||
return (new QuickSearchOutput().withResults(Collections.emptyList()).withTotalHits(0L).withTotalHitsIsLowerBound(false).withHasMore(false));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,18 +17,24 @@
|
||||
package com.kingsrook.qbits.quicksearch.actions;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Input DTO for the QuickSearch action.
|
||||
**
|
||||
** Carries the search term plus optional filtering and pagination parameters.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchInput
|
||||
public class QuickSearchInput extends AbstractActionInput
|
||||
{
|
||||
|
||||
private String searchTerm;
|
||||
private String tableName;
|
||||
private Integer limit;
|
||||
private Integer offset;
|
||||
private List<String> tableNames;
|
||||
private Integer limitPerTable;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -116,7 +122,14 @@ public class QuickSearchInput
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for offset.
|
||||
** Getter for offset: the number of results to skip, counting only results
|
||||
** the session can see (with record security locks on, hits filtered out
|
||||
** by a lock are not counted). With limitPerTable it applies within each
|
||||
** table. Results are reachable only within the first 10,000 raw hits; in
|
||||
** per-table mode with locks each table gets an equal share of that scan
|
||||
** budget, so paging within a table stops past
|
||||
** max(10,000 / T, 2 x limitPerTable) raw hits (hasMore turns false and
|
||||
** totalHitsIsLowerBound stays true).
|
||||
*******************************************************************************/
|
||||
public Integer getOffset()
|
||||
{
|
||||
@@ -142,4 +155,65 @@ public class QuickSearchInput
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tableNames (restrict the search to these tables)
|
||||
*******************************************************************************/
|
||||
public List<String> getTableNames()
|
||||
{
|
||||
return tableNames;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for tableNames
|
||||
*******************************************************************************/
|
||||
public void setTableNames(List<String> tableNames)
|
||||
{
|
||||
this.tableNames = tableNames;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tableNames
|
||||
*******************************************************************************/
|
||||
public QuickSearchInput withTableNames(List<String> tableNames)
|
||||
{
|
||||
this.tableNames = tableNames;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for limitPerTable (search each table separately, up to this many)
|
||||
*******************************************************************************/
|
||||
public Integer getLimitPerTable()
|
||||
{
|
||||
return limitPerTable;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for limitPerTable
|
||||
*******************************************************************************/
|
||||
public void setLimitPerTable(Integer limitPerTable)
|
||||
{
|
||||
this.limitPerTable = limitPerTable;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for limitPerTable
|
||||
*******************************************************************************/
|
||||
public QuickSearchInput withLimitPerTable(Integer limitPerTable)
|
||||
{
|
||||
this.limitPerTable = limitPerTable;
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -31,6 +31,7 @@ public class QuickSearchOutput
|
||||
|
||||
private List<QuickSearchResult> results;
|
||||
private Long totalHits;
|
||||
private Boolean totalHitsIsLowerBound = false;
|
||||
private Boolean hasMore;
|
||||
|
||||
|
||||
@@ -117,4 +118,36 @@ public class QuickSearchOutput
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for totalHitsIsLowerBound: true when totalHits is a floor (hits
|
||||
** beyond the result window, or hits removed by permission filtering)
|
||||
*******************************************************************************/
|
||||
public Boolean getTotalHitsIsLowerBound()
|
||||
{
|
||||
return totalHitsIsLowerBound;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for totalHitsIsLowerBound
|
||||
*******************************************************************************/
|
||||
public void setTotalHitsIsLowerBound(Boolean totalHitsIsLowerBound)
|
||||
{
|
||||
this.totalHitsIsLowerBound = totalHitsIsLowerBound;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for totalHitsIsLowerBound
|
||||
*******************************************************************************/
|
||||
public QuickSearchOutput withTotalHitsIsLowerBound(Boolean totalHitsIsLowerBound)
|
||||
{
|
||||
this.totalHitsIsLowerBound = totalHitsIsLowerBound;
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -25,6 +25,7 @@ public class QuickSearchResult
|
||||
{
|
||||
|
||||
private String tableName;
|
||||
private String tableLabel;
|
||||
private String recordId;
|
||||
private String recordLabel;
|
||||
private Float score;
|
||||
@@ -170,4 +171,35 @@ public class QuickSearchResult
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tableLabel
|
||||
*******************************************************************************/
|
||||
public String getTableLabel()
|
||||
{
|
||||
return tableLabel;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for tableLabel
|
||||
*******************************************************************************/
|
||||
public void setTableLabel(String tableLabel)
|
||||
{
|
||||
this.tableLabel = tableLabel;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tableLabel
|
||||
*******************************************************************************/
|
||||
public QuickSearchResult withTableLabel(String tableLabel)
|
||||
{
|
||||
this.tableLabel = tableLabel;
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.customizers;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Post-delete table customizer that removes deleted records from the Quick
|
||||
** Search OpenSearch index.
|
||||
**
|
||||
** QQQ instantiates this class via reflection using the no-arg constructor.
|
||||
** The publisher and table configuration are obtained from the static
|
||||
** QuickSearchQBitContext at call time.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchPostDeleteCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchPostDeleteCustomizer.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
|
||||
*******************************************************************************/
|
||||
public QuickSearchPostDeleteCustomizer()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Post-delete hook: builds DELETE events for each deleted record and publishes
|
||||
** them via the configured IndexEventPublisher.
|
||||
**
|
||||
** If the publisher is null (QBit not yet initialized) or if publishing fails,
|
||||
** a warning is logged and the original records are returned without throwing,
|
||||
** so as never to block the underlying QQQ delete operation.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
|
||||
{
|
||||
if(records == null || records.isEmpty())
|
||||
{
|
||||
return records;
|
||||
}
|
||||
|
||||
IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
|
||||
if(publisher == null)
|
||||
{
|
||||
LOG.warn("QuickSearch publisher is null; skipping post-delete indexing", logPair("tableName", deleteInput.getTableName()));
|
||||
return records;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
String tableName = deleteInput.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
|
||||
String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
|
||||
|
||||
List<IndexEvent> events = new ArrayList<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
events.add(new IndexEvent()
|
||||
.withAction(IndexEventAction.DELETE)
|
||||
.withTableName(tableName)
|
||||
.withRecordId(record.getValueString(primaryKeyField)));
|
||||
}
|
||||
|
||||
publisher.publishDeleteEvents(events);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to publish post-delete index events", e, logPair("tableName", deleteInput.getTableName()));
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
}
|
||||
-107
@@ -1,107 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.customizers;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Post-insert table customizer that indexes newly inserted records in the
|
||||
** Quick Search OpenSearch index.
|
||||
**
|
||||
** QQQ instantiates this class via reflection using the no-arg constructor.
|
||||
** The publisher and table configuration are obtained from the static
|
||||
** QuickSearchQBitContext at call time.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchPostInsertCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchPostInsertCustomizer.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
|
||||
*******************************************************************************/
|
||||
public QuickSearchPostInsertCustomizer()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Post-insert hook: builds INDEX events for each inserted record and publishes
|
||||
** them via the configured IndexEventPublisher.
|
||||
**
|
||||
** If the publisher is null (QBit not yet initialized) or if publishing fails,
|
||||
** a warning is logged and the original records are returned without throwing,
|
||||
** so as never to block the underlying QQQ insert operation.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
|
||||
{
|
||||
if(records == null || records.isEmpty())
|
||||
{
|
||||
return records;
|
||||
}
|
||||
|
||||
IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
|
||||
if(publisher == null)
|
||||
{
|
||||
LOG.warn("QuickSearch publisher is null; skipping post-insert indexing", logPair("tableName", insertInput.getTableName()));
|
||||
return records;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
String tableName = insertInput.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
|
||||
String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
|
||||
|
||||
List<IndexEvent> events = new ArrayList<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
events.add(new IndexEvent()
|
||||
.withAction(IndexEventAction.INDEX)
|
||||
.withTableName(tableName)
|
||||
.withRecordId(record.getValueString(primaryKeyField))
|
||||
.withRecord(record));
|
||||
}
|
||||
|
||||
publisher.publishIndexEvents(events);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to publish post-insert index events", e, logPair("tableName", insertInput.getTableName()));
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
}
|
||||
-108
@@ -1,108 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.customizers;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Post-update table customizer that re-indexes updated records in the Quick
|
||||
** Search OpenSearch index.
|
||||
**
|
||||
** QQQ instantiates this class via reflection using the no-arg constructor.
|
||||
** The publisher and table configuration are obtained from the static
|
||||
** QuickSearchQBitContext at call time.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchPostUpdateCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchPostUpdateCustomizer.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No-arg constructor required by QQQ's QCodeReference instantiation mechanism.
|
||||
*******************************************************************************/
|
||||
public QuickSearchPostUpdateCustomizer()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Post-update hook: builds INDEX events for each updated record and publishes
|
||||
** them via the configured IndexEventPublisher.
|
||||
**
|
||||
** If the publisher is null (QBit not yet initialized) or if publishing fails,
|
||||
** a warning is logged and the original records are returned without throwing,
|
||||
** so as never to block the underlying QQQ update operation.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
|
||||
{
|
||||
if(records == null || records.isEmpty())
|
||||
{
|
||||
return records;
|
||||
}
|
||||
|
||||
IndexEventPublisher publisher = QuickSearchQBitContext.getPublisher();
|
||||
if(publisher == null)
|
||||
{
|
||||
LOG.warn("QuickSearch publisher is null; skipping post-update indexing", logPair("tableName", updateInput.getTableName()));
|
||||
return records;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
String tableName = updateInput.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = QuickSearchQBitContext.getTableConfig(tableName);
|
||||
String primaryKeyField = (tableConfig != null && tableConfig.getPrimaryKeyField() != null) ? tableConfig.getPrimaryKeyField() : "id";
|
||||
|
||||
List<IndexEvent> events = new ArrayList<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
events.add(new IndexEvent()
|
||||
.withAction(IndexEventAction.INDEX)
|
||||
.withTableName(tableName)
|
||||
.withRecordId(record.getValueString(primaryKeyField))
|
||||
.withRecord(record));
|
||||
}
|
||||
|
||||
publisher.publishIndexEvents(events);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to publish post-update index events", e, logPair("tableName", updateInput.getTableName()));
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
}
|
||||
+496
@@ -0,0 +1,496 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.listeners;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Real-time indexing hook. Registered once per QInstance; applies to the
|
||||
** tables this QBit indexes.
|
||||
**
|
||||
** Events are built while the write's transaction is open, but published only
|
||||
** after that transaction commits, via
|
||||
** QBackendTransaction.addAfterCommitCallback. A write without a transaction
|
||||
** is already committed, so it publishes immediately. A rolled-back write never
|
||||
** reaches the index.
|
||||
**
|
||||
** Inserted and updated records are re-read inside that transaction, with
|
||||
** display values and possible-value labels, so a real-time document matches
|
||||
** the one basepull, reconcile and full reindex build. The event's own records
|
||||
** are not enough: for an update they are the sparse input, and for an insert
|
||||
** they carry only raw values.
|
||||
**
|
||||
** This listener never fails the host's write. When the re-read of inserted or
|
||||
** updated records or publishing fails (including a LinkageError from a missing
|
||||
** optional library while the client is built), each affected (table, record,
|
||||
** action) is written to quickSearchFailedEvent for the scheduled basepull to
|
||||
** replay. A failed re-read is recorded after the commit too, so a rolled-back
|
||||
** write leaves no row.
|
||||
**
|
||||
** While a full reindex rebuilds into a new physical index, the alias still
|
||||
** points at the old one, so a published delete never reaches the new index.
|
||||
** Such deletes are also written to quickSearchFailedEvent with status
|
||||
** AWAITING_REINDEX; the reindex applies them after the alias swap.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchRecordChangeListener implements RecordChangeListenerInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordChangeListener.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean appliesTo(String tableName, RecordChangeType type)
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
return (runtime != null
|
||||
&& Boolean.TRUE.equals(runtime.getConfig().getEnableRealTimeIndexing())
|
||||
&& runtime.getTableConfig(tableName) != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void onRecordsChanged(RecordChangeEvent event) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null || event == null || CollectionUtils.nullSafeIsEmpty(event.getRecords()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String tableName = event.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
|
||||
if(tableConfig == null || !runtime.isTableEnabled(tableName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String primaryKeyField = tableConfig.getPrimaryKeyField() != null ? tableConfig.getPrimaryKeyField() : "id";
|
||||
|
||||
List<IndexEvent> indexEvents = new ArrayList<>();
|
||||
List<IndexEvent> deleteEvents = new ArrayList<>();
|
||||
|
||||
switch(event.getType())
|
||||
{
|
||||
case INSERT, UPDATE ->
|
||||
{
|
||||
try
|
||||
{
|
||||
indexEvents.addAll(toIndexEvents(tableName, primaryKeyField, fetchCurrentRecords(event, primaryKeyField)));
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// log now: with a transaction, nothing else is logged until commit, //
|
||||
// and a rollback (a transaction the failed re-read aborted, too) //
|
||||
// would leave no trace. Then record the written keys after commit, //
|
||||
// so basepull re-reads and indexes them. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
LOG.warn("Could not re-read " + (event.getType() == RecordChangeType.INSERT ? "inserted" : "updated")
|
||||
+ " records for indexing; recording them for replay once the write commits", e,
|
||||
logPair("tableName", tableName), logPair("recordCount", event.getRecords().size()));
|
||||
List<IndexEvent> failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords());
|
||||
runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e));
|
||||
return;
|
||||
}
|
||||
}
|
||||
case DELETE -> deleteEvents.addAll(toDeleteEvents(tableName, primaryKeyField, event.getRecords()));
|
||||
}
|
||||
|
||||
if(indexEvents.isEmpty() && deleteEvents.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
runAfterCommit(event, () -> publish(runtime, indexEvents, deleteEvents));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run work once the event's transaction commits, or now when the write had
|
||||
** no transaction (it is already committed).
|
||||
*******************************************************************************/
|
||||
private static void runAfterCommit(RecordChangeEvent event, Runnable work)
|
||||
{
|
||||
if(event.getTransaction() != null)
|
||||
{
|
||||
event.getTransaction().addAfterCommitCallback(work);
|
||||
}
|
||||
else
|
||||
{
|
||||
work.run();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish, recording any failure durably, and capture the deletes if a full
|
||||
** reindex is rebuilding the table. Never throws.
|
||||
**
|
||||
** The status is read before the deletes are published, never after: a
|
||||
** reindex can swap the alias and clear REBUILDING while the publish is in
|
||||
** flight, and a read after that would miss a delete that reached only the
|
||||
** old index. A read before the publish that sees no REBUILDING means either
|
||||
** the reindex has not started reading the table (the record, deleted and
|
||||
** committed, cannot be read into the new index) or the alias already points
|
||||
** at the new index (the publish reaches it).
|
||||
*******************************************************************************/
|
||||
static void publish(QuickSearchRuntime runtime, List<IndexEvent> indexEvents, List<IndexEvent> deleteEvents)
|
||||
{
|
||||
boolean rebuilding = isRebuildingBeforePublish(runtime, deleteEvents);
|
||||
publishOrRecordFailures(runtime, indexEvents, deleteEvents);
|
||||
if(rebuilding)
|
||||
{
|
||||
captureDeletesDuringRebuild(runtime, deleteEvents);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A LinkageError is caught along with Exception: building the lazy client,
|
||||
** or a host publisher, can hit an optional library that is not on the
|
||||
** classpath (the AWS SDK for AWS_SIGV4), which a DEGRADED start leaves for
|
||||
** the first write to find.
|
||||
*******************************************************************************/
|
||||
private static void publishOrRecordFailures(QuickSearchRuntime runtime, List<IndexEvent> indexEvents, List<IndexEvent> deleteEvents)
|
||||
{
|
||||
IndexEventPublisher publisher;
|
||||
try
|
||||
{
|
||||
publisher = runtime.getPublisher();
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
recordFailures(runtime, indexEvents, e);
|
||||
recordFailures(runtime, deleteEvents, e);
|
||||
return;
|
||||
}
|
||||
|
||||
if(!indexEvents.isEmpty())
|
||||
{
|
||||
try
|
||||
{
|
||||
publisher.publishIndexEvents(indexEvents);
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
recordFailures(runtime, indexEvents, e);
|
||||
}
|
||||
}
|
||||
|
||||
if(!deleteEvents.isEmpty())
|
||||
{
|
||||
try
|
||||
{
|
||||
publisher.publishDeleteEvents(deleteEvents);
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
recordFailures(runtime, deleteEvents, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write one quickSearchFailedEvent row per event. Never throws.
|
||||
*******************************************************************************/
|
||||
static void recordFailures(QuickSearchRuntime runtime, List<IndexEvent> events, Throwable cause)
|
||||
{
|
||||
if(events.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String message = cause.getMessage() == null ? cause.getClass().getSimpleName() : cause.getMessage();
|
||||
LOG.warn("Real-time indexing failed; recording failed events for replay", cause,
|
||||
logPair("tableName", events.get(0).getTableName()), logPair("eventCount", events.size()));
|
||||
|
||||
try
|
||||
{
|
||||
if(!hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)))
|
||||
{
|
||||
LOG.error("quickSearchFailedEvent table is not in the QInstance; failed events are lost", logPair("eventCount", events.size()));
|
||||
return;
|
||||
}
|
||||
|
||||
insertFailedEventRows(runtime, events, message, QuickSearchFailedEvent.STATUS_PENDING);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.error("Could not record failed index events", e, logPair("eventCount", events.size()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether a full reindex is rebuilding the deletes' table. The status is
|
||||
** read from the database (not cached) so a reindex on another node is seen.
|
||||
** Runs after the delete committed. Never throws.
|
||||
*******************************************************************************/
|
||||
static boolean isRebuildingBeforePublish(QuickSearchRuntime runtime, List<IndexEvent> deleteEvents)
|
||||
{
|
||||
if(deleteEvents.isEmpty())
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
String tableName = deleteEvents.get(0).getTableName();
|
||||
try
|
||||
{
|
||||
return (hasTable(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME)) && isRebuilding(runtime, tableName));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.error("Could not read whether a full reindex is running; deletes made during one are not captured", e, logPair("tableName", tableName));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record deletes made while a full reindex is rebuilding the table, so the
|
||||
** reindex can apply them to the new physical index after the alias swap.
|
||||
** basepull does not replay AWAITING_REINDEX rows, so it cannot consume one
|
||||
** against the old index before the swap. Never throws.
|
||||
*******************************************************************************/
|
||||
static void captureDeletesDuringRebuild(QuickSearchRuntime runtime, List<IndexEvent> deleteEvents)
|
||||
{
|
||||
String tableName = deleteEvents.get(0).getTableName();
|
||||
try
|
||||
{
|
||||
List<QRecord> inserted = insertFailedEventRows(runtime, deleteEvents, "deleted during a full reindex", QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
handOverIfRebuildFinished(runtime, tableName, inserted);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.error("Could not capture deletes made during a full reindex; the next full reindex or reconcile removes them", e,
|
||||
logPair("tableName", tableName), logPair("eventCount", deleteEvents.size()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The reindex may have finished, and drained its captured deletes, between
|
||||
** the status read and the insert. Read the status again: if the table is no
|
||||
** longer REBUILDING, make the rows PENDING so basepull replays them against
|
||||
** the current index (deletes are idempotent). If it still is, the reindex
|
||||
** has not drained yet and will see these rows.
|
||||
*******************************************************************************/
|
||||
static void handOverIfRebuildFinished(QuickSearchRuntime runtime, String tableName, List<QRecord> inserted) throws QException
|
||||
{
|
||||
if(inserted.isEmpty() || isRebuilding(runtime, tableName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
for(QRecord row : inserted)
|
||||
{
|
||||
updates.add(new QRecord().withValue("id", row.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING));
|
||||
}
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
|
||||
updateInput.setRecords(updates);
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the table's quickSearchIndex row is REBUILDING.
|
||||
*******************************************************************************/
|
||||
private static boolean isRebuilding(QuickSearchRuntime runtime, String tableName) throws QException
|
||||
{
|
||||
String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
|
||||
if(!hasTable(indexTable))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(indexTable);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)).withLimit(1));
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
return (CollectionUtils.nullSafeHasContents(rows) && AbstractIndexingStep.STATUS_REBUILDING.equals(rows.get(0).getValueString("status")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert one quickSearchFailedEvent row per event; returns the inserted rows.
|
||||
*******************************************************************************/
|
||||
static List<QRecord> insertFailedEventRows(QuickSearchRuntime runtime, List<IndexEvent> events, String message, String status) throws QException
|
||||
{
|
||||
List<QRecord> rows = new ArrayList<>();
|
||||
for(IndexEvent event : events)
|
||||
{
|
||||
rows.add(new QRecord()
|
||||
.withValue("tableName", event.getTableName())
|
||||
.withValue("recordId", event.getRecordId())
|
||||
.withValue("action", event.getAction() == null ? null : event.getAction().name())
|
||||
.withValue("errorMessage", truncate(message, 4000))
|
||||
.withValue("attempts", 0)
|
||||
.withValue("status", status));
|
||||
}
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
|
||||
insertInput.setRecords(rows);
|
||||
return (CollectionUtils.nonNullList(new InsertAction().execute(insertInput).getRecords()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean hasTable(String tableName)
|
||||
{
|
||||
return (QContext.getQInstance() != null && QContext.getQInstance().getTable(tableName) != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<IndexEvent> toIndexEvents(String tableName, String primaryKeyField, List<QRecord> records)
|
||||
{
|
||||
List<IndexEvent> events = new ArrayList<>();
|
||||
for(QRecord record : CollectionUtils.nonNullList(records))
|
||||
{
|
||||
String recordId = record.getValueString(primaryKeyField);
|
||||
if(recordId == null || CollectionUtils.nullSafeHasContents(record.getErrors()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
events.add(new IndexEvent().withAction(IndexEventAction.INDEX).withTableName(tableName).withRecordId(recordId).withRecord(record));
|
||||
}
|
||||
return (events);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<IndexEvent> toDeleteEvents(String tableName, String primaryKeyField, List<QRecord> records)
|
||||
{
|
||||
List<IndexEvent> events = new ArrayList<>();
|
||||
for(QRecord record : CollectionUtils.nonNullList(records))
|
||||
{
|
||||
String recordId = record.getValueString(primaryKeyField);
|
||||
if(recordId == null || CollectionUtils.nullSafeHasContents(record.getErrors()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
events.add(new IndexEvent().withAction(IndexEventAction.DELETE).withTableName(tableName).withRecordId(recordId));
|
||||
}
|
||||
return (events);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The event's records are "as the backend returned them": for an update on
|
||||
** RDBMS that is the sparse input, and for an insert the raw values with no
|
||||
** display values. Re-read the full rows inside the same transaction, with
|
||||
** display values so possible-value labels are indexed (which can run host
|
||||
** possible-value providers).
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> fetchCurrentRecords(RecordChangeEvent event, String primaryKeyField) throws QException
|
||||
{
|
||||
List<Serializable> primaryKeys = new ArrayList<>();
|
||||
for(QRecord record : event.getRecords())
|
||||
{
|
||||
Serializable primaryKey = record.getValue(primaryKeyField);
|
||||
if(primaryKey != null && CollectionUtils.nullSafeIsEmpty(record.getErrors()))
|
||||
{
|
||||
primaryKeys.add(primaryKey);
|
||||
}
|
||||
}
|
||||
|
||||
if(primaryKeys.isEmpty())
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(event.getTableName());
|
||||
queryInput.setTransaction(event.getTransaction());
|
||||
queryInput.setShouldGenerateDisplayValues(true);
|
||||
queryInput.setShouldTranslatePossibleValues(true);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, primaryKeys)));
|
||||
|
||||
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String truncate(String s, int max)
|
||||
{
|
||||
return (s == null || s.length() <= max ? s : s.substring(0, max));
|
||||
}
|
||||
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the quickSearchAdmin app from the tables and processes the other
|
||||
** component producers added to the production output. Runs last.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataProducer<QAppMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
public static final String APP_NAME = "quickSearchAdmin";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QAppMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getQBitConfig();
|
||||
|
||||
QAppMetaData app = new QAppMetaData()
|
||||
.withName(config.applyPrefix(APP_NAME))
|
||||
.withLabel("Quick Search Admin")
|
||||
.withIcon(new QIcon("search"))
|
||||
.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone());
|
||||
|
||||
MetaDataProducerMultiOutput produced = QBitProductionContext.peekMetaDataProducerMultiOutput();
|
||||
if(produced == null)
|
||||
{
|
||||
return (app);
|
||||
}
|
||||
|
||||
for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME, QuickSearchFailedEvent.TABLE_NAME })
|
||||
{
|
||||
QTableMetaData table = produced.get(QTableMetaData.class, config.applyPrefix(tableName));
|
||||
if(table != null)
|
||||
{
|
||||
app.withChild(table);
|
||||
}
|
||||
}
|
||||
|
||||
for(String processName : new String[] { QuickSearchProcessMetaDataHelper.BASEPULL_PROCESS_NAME, QuickSearchProcessMetaDataHelper.FULL_REINDEX_PROCESS_NAME, QuickSearchProcessMetaDataHelper.RECONCILE_PROCESS_NAME })
|
||||
{
|
||||
QProcessMetaData process = produced.get(QProcessMetaData.class, config.applyPrefix(processName));
|
||||
if(process != null)
|
||||
{
|
||||
app.withChild(process);
|
||||
}
|
||||
}
|
||||
|
||||
return (app);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** last, so the tables and processes exist in the output
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (900);
|
||||
}
|
||||
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the basepull process when enableBasepullProcess is true.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchBasepullProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QProcessMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchProcessMetaDataHelper.buildBasepullProcess(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableBasepullProcess()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** after tables, before the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (200);
|
||||
}
|
||||
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the quickSearchFailedEvent table.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchFailedEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchTableMetaDataHelper.buildFailedEventTable(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tables before processes and the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the full reindex process when enableMaintenanceProcesses is true.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchFullReindexProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QProcessMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchProcessMetaDataHelper.buildFullReindexProcess(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableMaintenanceProcesses()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** after tables, before the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (200);
|
||||
}
|
||||
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the possible value source over the quickSearchIndex table.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchIndexPossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QPossibleValueSource produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchTableMetaDataHelper.buildIndexPossibleValueSource(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the quickSearchIndexRun table.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchIndexRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchTableMetaDataHelper.buildIndexRunTable(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tables before processes and the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the quickSearchIndex table.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchIndexTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchTableMetaDataHelper.buildIndexTable(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tables before processes and the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.ReconcileIndexStep;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Builds the three process definitions, with permission rules from config and
|
||||
** schedules when a scheduler is configured.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchProcessMetaDataHelper
|
||||
{
|
||||
public static final String BASEPULL_PROCESS_NAME = "quickSearchBasepullIndex";
|
||||
public static final String FULL_REINDEX_PROCESS_NAME = "quickSearchFullReindex";
|
||||
public static final String RECONCILE_PROCESS_NAME = "quickSearchReconcileIndex";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Basepull catch-up; scheduled every basepullRepeatSeconds when a
|
||||
** schedulerName is set.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildBasepullProcess(QuickSearchQBitConfig config)
|
||||
{
|
||||
QProcessMetaData process = buildProcess(config, BASEPULL_PROCESS_NAME, "Quick Search Basepull Index", "basepull", BasepullIndexStep.class, "update");
|
||||
process.withStep(resultStep("Basepull complete", "tablesIndexed", "failedEventsReplayed"));
|
||||
|
||||
if(StringUtils.hasContent(config.getSchedulerName()))
|
||||
{
|
||||
process.withSchedule(new QScheduleMetaData()
|
||||
.withSchedulerName(config.getSchedulerName())
|
||||
.withRepeatSeconds(config.getBasepullRepeatSeconds() == null ? 300 : config.getBasepullRepeatSeconds())
|
||||
.withDescription("Quick Search basepull catch-up"));
|
||||
}
|
||||
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full reindex; on demand only.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildFullReindexProcess(QuickSearchQBitConfig config)
|
||||
{
|
||||
QProcessMetaData process = buildProcess(config, FULL_REINDEX_PROCESS_NAME, "Quick Search Full Reindex", "fullReindex", FullReindexStep.class, "refresh");
|
||||
process.withStep(0, inputStep(config, "Leave the table blank to rebuild every table into a fresh index and swap the alias; pick a table to rebuild just that table in place."));
|
||||
process.withStep(resultStep("Full reindex complete", "recordsIndexed", "aliasSwapped"));
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reconcile; scheduled by cron when reconcileCronExpression and a
|
||||
** schedulerName are set.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildReconcileProcess(QuickSearchQBitConfig config)
|
||||
{
|
||||
QProcessMetaData process = buildProcess(config, RECONCILE_PROCESS_NAME, "Quick Search Reconcile Index", "reconcile", ReconcileIndexStep.class, "sync");
|
||||
process.withStep(0, inputStep(config, "Leave the table blank to reconcile every table and remove documents of unconfigured tables; pick a table to reconcile just that one."));
|
||||
process.withStep(resultStep("Reconcile complete", "recordsIndexed", "documentsRemoved", "orphanDocumentsRemoved", "tablesSkipped"));
|
||||
|
||||
if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getReconcileCronExpression()))
|
||||
{
|
||||
process.withSchedule(new QScheduleMetaData()
|
||||
.withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(config.getReconcileCronExpression())
|
||||
.withCronTimeZoneId(config.getReconcileCronTimeZoneId())
|
||||
.withDescription("Quick Search reconcile"));
|
||||
}
|
||||
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optional table picker shown before the backend step.
|
||||
*******************************************************************************/
|
||||
private static QFrontendStepMetaData inputStep(QuickSearchQBitConfig config, String help)
|
||||
{
|
||||
return (new QFrontendStepMetaData()
|
||||
.withName("input")
|
||||
.withLabel("Choose table")
|
||||
.withFormField(new QFieldMetaData("tableName", QFieldType.STRING)
|
||||
.withLabel("Table (optional)")
|
||||
.withPossibleValueSourceName(config.applyPrefix(QuickSearchSearchableTablePossibleValueSourceProducer.NAME))
|
||||
.withIsRequired(false))
|
||||
.withViewField(new QFieldMetaData("help", QFieldType.STRING).withLabel(" ").withDefaultValue(help)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary screen showing the step's output values.
|
||||
*******************************************************************************/
|
||||
private static QFrontendStepMetaData resultStep(String label, String... outputFields)
|
||||
{
|
||||
QFrontendStepMetaData step = new QFrontendStepMetaData().withName("result").withLabel(label);
|
||||
for(String field : outputFields)
|
||||
{
|
||||
step.withViewField(new QFieldMetaData(field, QFieldType.STRING));
|
||||
}
|
||||
return (step);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static QProcessMetaData buildProcess(QuickSearchQBitConfig config, String baseName, String label, String stepName, Class<?> stepClass, String icon)
|
||||
{
|
||||
return (new QProcessMetaData()
|
||||
.withName(config.applyPrefix(baseName))
|
||||
.withLabel(label)
|
||||
.withIcon(new QIcon(icon))
|
||||
.withPermissionRules(config.getAdminPermissionRules() == null ? null : config.getAdminPermissionRules().clone())
|
||||
.withStep(new QBackendStepMetaData()
|
||||
.withName(stepName)
|
||||
.withCode(new QCodeReference(stepClass))));
|
||||
}
|
||||
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the reconcile process when enableMaintenanceProcesses is true.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchReconcileProcessMetaDataProducer extends QBitComponentMetaDataProducer<QProcessMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QProcessMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchProcessMetaDataHelper.buildReconcileProcess(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableMaintenanceProcesses()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** after tables, before the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (200);
|
||||
}
|
||||
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Enum possible value source listing the searchable tables (from annotated
|
||||
** entity classes and config-driven tables), used by the process input screens
|
||||
** so an operator can pick one table for a full reindex or reconcile.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchSearchableTablePossibleValueSourceProducer extends QBitComponentMetaDataProducer<QPossibleValueSource, QuickSearchQBitConfig>
|
||||
{
|
||||
public static final String NAME = "quickSearchSearchableTable";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QPossibleValueSource produce(QInstance qInstance) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getQBitConfig();
|
||||
|
||||
Set<String> tableNames = new LinkedHashSet<>();
|
||||
for(Class<?> entityClass : config.getSearchableEntityClasses() == null ? List.<Class<?>>of() : config.getSearchableEntityClasses())
|
||||
{
|
||||
QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class);
|
||||
if(annotation != null)
|
||||
{
|
||||
tableNames.add(annotation.tableName());
|
||||
}
|
||||
}
|
||||
for(SearchableTableConfig tableConfig : config.getSearchableTables() == null ? List.<SearchableTableConfig>of() : config.getSearchableTables())
|
||||
{
|
||||
if(tableConfig.getTableName() != null)
|
||||
{
|
||||
tableNames.add(tableConfig.getTableName());
|
||||
}
|
||||
}
|
||||
|
||||
List<QPossibleValue<?>> values = new ArrayList<>();
|
||||
for(String tableName : tableNames)
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(tableName);
|
||||
String label = table != null && table.getLabel() != null ? table.getLabel() : tableName;
|
||||
values.add(new QPossibleValue<>(tableName, label));
|
||||
}
|
||||
|
||||
return (new QPossibleValueSource()
|
||||
.withName(config.applyPrefix(NAME))
|
||||
.withType(QPossibleValueSourceType.ENUM)
|
||||
.withEnumValues(values));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Builds the QTableMetaData for the three operational tables. Shared by the
|
||||
** component producers and by tests that need the same shapes.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchTableMetaDataHelper
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** quickSearchIndex: one row per indexed table.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData buildIndexTable(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(config.applyPrefix(QuickSearchIndex.TABLE_NAME))
|
||||
.withLabel("Quick Search Index")
|
||||
.withIcon(new QIcon("manage_search"))
|
||||
.withBackendName(config.getBackendName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFields("tableName")
|
||||
.withUniqueKey(new UniqueKey("tableName"))
|
||||
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
|
||||
.withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("searchableFieldsJson", QFieldType.TEXT).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastReconcileTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastRunStatus", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastErrorMessage", QFieldType.TEXT).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("recordCount", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("documentCount", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("realTimeErrorCount", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** quickSearchIndexRun: history of basepull, reindex and reconcile runs.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData buildIndexRunTable(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(config.applyPrefix(QuickSearchIndexRun.TABLE_NAME))
|
||||
.withLabel("Quick Search Index Run")
|
||||
.withIcon(new QIcon("history"))
|
||||
.withBackendName(config.getBackendName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withIsEditable(false)
|
||||
.withPossibleValueSourceName(config.applyPrefix(QuickSearchIndex.TABLE_NAME)))
|
||||
.withField(new QFieldMetaData("runType", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("errorCount", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** quickSearchFailedEvent: real-time index or delete events that could not
|
||||
** reach OpenSearch, replayed by the scheduled basepull.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData buildFailedEventTable(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME))
|
||||
.withLabel("Quick Search Failed Event")
|
||||
.withIcon(new QIcon("error_outline"))
|
||||
.withBackendName(config.getBackendName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("recordId", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("action", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("attempts", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Possible value source over quickSearchIndex, so run rows show the table
|
||||
** name instead of an id.
|
||||
*******************************************************************************/
|
||||
public static QPossibleValueSource buildIndexPossibleValueSource(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (QPossibleValueSource.newForTable(config.applyPrefix(QuickSearchIndex.TABLE_NAME)));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.model;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QField;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A real-time index or delete event that could not be applied to OpenSearch.
|
||||
** Replayed by the scheduled basepull; status PENDING until it succeeds, or
|
||||
** EXHAUSTED after the retry limit.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchFailedEvent extends QRecordEntity
|
||||
{
|
||||
public static final String TABLE_NAME = "quickSearchFailedEvent";
|
||||
|
||||
public static final String STATUS_PENDING = "PENDING";
|
||||
public static final String STATUS_EXHAUSTED = "EXHAUSTED";
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// a delete made while a full reindex rebuilds into a new physical index; //
|
||||
// the reindex applies it after the swap, and basepull does not replay it //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
public static final String STATUS_AWAITING_REINDEX = "AWAITING_REINDEX";
|
||||
|
||||
@QField()
|
||||
private Integer id;
|
||||
|
||||
@QField()
|
||||
private String tableName;
|
||||
|
||||
@QField()
|
||||
private String recordId;
|
||||
|
||||
@QField()
|
||||
private String action;
|
||||
|
||||
@QField()
|
||||
private String errorMessage;
|
||||
|
||||
@QField()
|
||||
private Integer attempts;
|
||||
|
||||
@QField()
|
||||
private String status;
|
||||
|
||||
@QField()
|
||||
private Instant createDate;
|
||||
|
||||
@QField()
|
||||
private Instant modifyDate;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for id
|
||||
*******************************************************************************/
|
||||
public Integer getId()
|
||||
{
|
||||
return (this.id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tableName
|
||||
*******************************************************************************/
|
||||
public String getTableName()
|
||||
{
|
||||
return (this.tableName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for recordId
|
||||
*******************************************************************************/
|
||||
public String getRecordId()
|
||||
{
|
||||
return (this.recordId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for action
|
||||
*******************************************************************************/
|
||||
public String getAction()
|
||||
{
|
||||
return (this.action);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for errorMessage
|
||||
*******************************************************************************/
|
||||
public String getErrorMessage()
|
||||
{
|
||||
return (this.errorMessage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for attempts
|
||||
*******************************************************************************/
|
||||
public Integer getAttempts()
|
||||
{
|
||||
return (this.attempts);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for status
|
||||
*******************************************************************************/
|
||||
public String getStatus()
|
||||
{
|
||||
return (this.status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for createDate
|
||||
*******************************************************************************/
|
||||
public Instant getCreateDate()
|
||||
{
|
||||
return (this.createDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for modifyDate
|
||||
*******************************************************************************/
|
||||
public Instant getModifyDate()
|
||||
{
|
||||
return (this.modifyDate);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -29,6 +29,7 @@ public class BulkIndexResult
|
||||
|
||||
private Integer successCount = 0;
|
||||
private Integer failureCount = 0;
|
||||
private Integer skippedCount = 0;
|
||||
private List<String> errors = new ArrayList<>();
|
||||
|
||||
|
||||
@@ -125,4 +126,24 @@ public class BulkIndexResult
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count a document skipped because a newer version is already indexed.
|
||||
*******************************************************************************/
|
||||
public void addSkipped()
|
||||
{
|
||||
skippedCount++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for skippedCount
|
||||
*******************************************************************************/
|
||||
public Integer getSkippedCount()
|
||||
{
|
||||
return (skippedCount);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -38,6 +38,8 @@ public class OpenSearchDocument
|
||||
private Instant indexedAt;
|
||||
private Map<String, Object> fieldValues;
|
||||
|
||||
private transient Long version;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -170,8 +172,42 @@ public class OpenSearchDocument
|
||||
** Returns the composite document ID used as the OpenSearch _id field.
|
||||
** Excluded from JSON serialization to avoid including it in the document body.
|
||||
*******************************************************************************/
|
||||
/*******************************************************************************
|
||||
** External version for optimistic indexing (epoch millis of the source
|
||||
** row's timestamp), or null to index unconditionally. Not stored in the
|
||||
** document body.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Long getVersion()
|
||||
{
|
||||
return (version);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for version
|
||||
*******************************************************************************/
|
||||
public OpenSearchDocument withVersion(Long version)
|
||||
{
|
||||
this.version = version;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public String getDocumentId()
|
||||
{
|
||||
return (buildDocumentId(sourceTable, recordId));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the composite OpenSearch _id for a source table and record ID.
|
||||
*******************************************************************************/
|
||||
public static String buildDocumentId(String sourceTable, String recordId)
|
||||
{
|
||||
return (sourceTable + ":" + recordId);
|
||||
}
|
||||
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.opensearch;
|
||||
|
||||
|
||||
import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
|
||||
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape hatch for hosts that need something the built-in auth modes do not
|
||||
** cover: bearer or JWT tokens, API-key headers, request interceptors, proxy
|
||||
** settings, or client certificates loaded from a custom source.
|
||||
**
|
||||
** Register an implementation on QuickSearchQBitConfig.transportCustomizer as a
|
||||
** QCodeReference. Both hooks run after the QBit has applied its own settings
|
||||
** (auth, TLS, timeouts) and before the transport is built. Not used for the
|
||||
** AWS_SIGV4 transport, which is built by the AWS SDK.
|
||||
*******************************************************************************/
|
||||
public interface OpenSearchTransportCustomizer
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Adjust the transport builder (default headers, request config callback).
|
||||
*******************************************************************************/
|
||||
default ApacheHttpClient5TransportBuilder customizeTransport(ApacheHttpClient5TransportBuilder builder)
|
||||
{
|
||||
return (builder);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Adjust the Apache HttpAsyncClient builder (interceptors, connection
|
||||
** manager, credentials).
|
||||
*******************************************************************************/
|
||||
default HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder builder)
|
||||
{
|
||||
return (builder);
|
||||
}
|
||||
|
||||
}
|
||||
+394
@@ -0,0 +1,394 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.opensearch;
|
||||
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.security.KeyStore;
|
||||
import java.security.cert.Certificate;
|
||||
import java.security.cert.CertificateFactory;
|
||||
import java.util.Collection;
|
||||
import java.util.regex.Pattern;
|
||||
import javax.net.ssl.HostnameVerifier;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchTlsConfig;
|
||||
import org.apache.hc.client5.http.auth.AuthScope;
|
||||
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
|
||||
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
|
||||
import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
|
||||
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
|
||||
import org.apache.hc.core5.http.HttpHost;
|
||||
import org.apache.hc.core5.http.message.BasicHeader;
|
||||
import org.apache.hc.core5.ssl.SSLContextBuilder;
|
||||
import org.apache.hc.core5.util.Timeout;
|
||||
import org.opensearch.client.json.JsonpMapper;
|
||||
import org.opensearch.client.json.jackson.JacksonJsonpMapper;
|
||||
import org.opensearch.client.transport.OpenSearchTransport;
|
||||
import org.opensearch.client.transport.aws.AwsSdk2Transport;
|
||||
import org.opensearch.client.transport.aws.AwsSdk2TransportOptions;
|
||||
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
|
||||
import software.amazon.awssdk.http.apache.ApacheHttpClient;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.sts.StsClient;
|
||||
import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
|
||||
import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Builds the OpenSearch transport for a config: NONE or BASIC auth over the
|
||||
** Apache HttpClient 5 transport with TLS, timeouts, pool size, user agent and
|
||||
** the optional OpenSearchTransportCustomizer; or AWS_SIGV4 over AwsSdk2Transport.
|
||||
**
|
||||
** Secret-bearing values (username, password, store passwords, paths) are
|
||||
** interpreted here with QMetaDataVariableInterpreter, so ${env.X} references
|
||||
** resolve only inside this factory.
|
||||
*******************************************************************************/
|
||||
public class OpenSearchTransportFactory
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(OpenSearchTransportFactory.class);
|
||||
|
||||
private static final String USER_AGENT = "qbit-quick-search/" + QuickSearchQBitProducer.getVersion();
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// Only a lone ${env.X} or ${prop.X} is echoed in errors: a ?? chain may //
|
||||
// hold a literal fallback secret between its references. //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
private static final Pattern SINGLE_REFERENCE = Pattern.compile("^\\$\\{(env|prop)\\.[A-Za-z0-9_.-]+\\}$");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static OpenSearchTransport build(QuickSearchQBitConfig config) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
JsonpMapper mapper = buildMapper();
|
||||
|
||||
return switch(config.getEffectiveAuthMode())
|
||||
{
|
||||
case AWS_SIGV4 -> buildAwsTransport(config, mapper);
|
||||
case NONE, BASIC -> buildApacheTransport(config, mapper);
|
||||
};
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
throw (e);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new QException("Failed to create OpenSearch transport: " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
static JsonpMapper buildMapper()
|
||||
{
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
|
||||
return (new JacksonJsonpMapper(objectMapper));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static OpenSearchTransport buildApacheTransport(QuickSearchQBitConfig config, JsonpMapper mapper) throws Exception
|
||||
{
|
||||
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
|
||||
|
||||
HttpHost httpHost = new HttpHost(config.getEffectiveScheme(), config.getEffectiveHost(), config.getEffectivePort());
|
||||
|
||||
BasicCredentialsProvider credentialsProvider = null;
|
||||
if(config.getEffectiveAuthMode() == QuickSearchAuthMode.BASIC)
|
||||
{
|
||||
String username = requireResolved(interpreter, config.getOpensearchUsername(), "opensearchUsername");
|
||||
String password = requireResolved(interpreter, config.getOpensearchPassword(), "opensearchPassword");
|
||||
credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(new AuthScope(httpHost), new UsernamePasswordCredentials(username, password.toCharArray()));
|
||||
}
|
||||
|
||||
OpenSearchTransportCustomizer customizer = loadCustomizer(config);
|
||||
SSLContext sslContext = buildSslContext(config.getTls(), interpreter);
|
||||
HostnameVerifier verifier = buildHostnameVerifier(config.getTls());
|
||||
|
||||
BasicCredentialsProvider finalCredentialsProvider = credentialsProvider;
|
||||
int maxConnections = config.getMaxConnections() == null ? ApacheHttpClient5TransportBuilder.DEFAULT_MAX_CONN_TOTAL : config.getMaxConnections();
|
||||
int connectTimeout = config.getConnectTimeoutMillis() == null ? ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS : config.getConnectTimeoutMillis();
|
||||
int responseTimeout = config.getResponseTimeoutMillis() == null ? ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS : config.getResponseTimeoutMillis();
|
||||
|
||||
ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder
|
||||
.builder(httpHost)
|
||||
.setMapper(mapper)
|
||||
.setDefaultHeaders(new BasicHeader[] { new BasicHeader("User-Agent", USER_AGENT) })
|
||||
.setRequestConfigCallback(requestConfig -> requestConfig
|
||||
.setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
|
||||
.setResponseTimeout(Timeout.ofMilliseconds(responseTimeout)))
|
||||
.setHttpClientConfigCallback(httpClientBuilder ->
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// httpclient5 5.6+ decompresses gzip responses itself but leaves //
|
||||
// the Content-Encoding header, so the OpenSearch transport would //
|
||||
// gunzip the body a second time and fail. Let the transport alone //
|
||||
// handle compression. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
httpClientBuilder.disableContentCompression();
|
||||
|
||||
if(finalCredentialsProvider != null)
|
||||
{
|
||||
httpClientBuilder.setDefaultCredentialsProvider(finalCredentialsProvider);
|
||||
}
|
||||
|
||||
ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create();
|
||||
if(sslContext != null)
|
||||
{
|
||||
tlsStrategyBuilder.setSslContext(sslContext);
|
||||
}
|
||||
if(verifier != null)
|
||||
{
|
||||
tlsStrategyBuilder.setHostnameVerifier(verifier);
|
||||
}
|
||||
|
||||
httpClientBuilder.setConnectionManager(PoolingAsyncClientConnectionManagerBuilder.create()
|
||||
.setTlsStrategy(tlsStrategyBuilder.build())
|
||||
.setMaxConnTotal(maxConnections)
|
||||
.setMaxConnPerRoute(maxConnections)
|
||||
.build());
|
||||
|
||||
return (customizer == null ? httpClientBuilder : customizer.customizeHttpClient(httpClientBuilder));
|
||||
});
|
||||
|
||||
if(customizer != null)
|
||||
{
|
||||
builder = customizer.customizeTransport(builder);
|
||||
}
|
||||
|
||||
return (builder.build());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** AWS Signature V4 over the AWS SDK's Apache HTTP client. Credentials come
|
||||
** from the default chain, or from STS when awsAssumeRoleArn is set.
|
||||
*******************************************************************************/
|
||||
private static OpenSearchTransport buildAwsTransport(QuickSearchQBitConfig config, JsonpMapper mapper) throws Exception
|
||||
{
|
||||
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
|
||||
|
||||
String region = config.getEffectiveAwsRegion();
|
||||
String service = config.getEffectiveAwsServiceName();
|
||||
String host = config.getEffectiveHost();
|
||||
if(config.getEffectivePort() != null && config.getEffectivePort() != 443)
|
||||
{
|
||||
host = host + ":" + config.getEffectivePort();
|
||||
}
|
||||
|
||||
AwsCredentialsProvider credentials = DefaultCredentialsProvider.create();
|
||||
String roleArn = interpreter.interpret(config.getAwsAssumeRoleArn());
|
||||
if(StringUtils.hasContent(roleArn))
|
||||
{
|
||||
StsClient stsClient = StsClient.builder().region(Region.of(region)).credentialsProvider(credentials).build();
|
||||
credentials = StsAssumeRoleCredentialsProvider.builder()
|
||||
.stsClient(stsClient)
|
||||
.refreshRequest(AssumeRoleRequest.builder().roleArn(roleArn).roleSessionName("qbit-quick-search").build())
|
||||
.build();
|
||||
}
|
||||
|
||||
LOG.info("Building AWS SigV4 OpenSearch transport", logPair("host", host), logPair("region", region), logPair("service", service), logPair("assumeRole", StringUtils.hasContent(roleArn)));
|
||||
|
||||
return (new AwsSdk2Transport(
|
||||
ApacheHttpClient.builder().build(),
|
||||
host,
|
||||
service,
|
||||
Region.of(region),
|
||||
AwsSdk2TransportOptions.builder().setCredentials(credentials).setMapper(mapper).build()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build an SSLContext for the tls block, or null for JVM defaults.
|
||||
*******************************************************************************/
|
||||
static SSLContext buildSslContext(QuickSearchTlsConfig tls, QMetaDataVariableInterpreter interpreter) throws Exception
|
||||
{
|
||||
if(tls == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
SSLContextBuilder builder = SSLContextBuilder.create();
|
||||
boolean custom = false;
|
||||
|
||||
if(Boolean.TRUE.equals(tls.getInsecureSkipVerify()))
|
||||
{
|
||||
LOG.warn("tls.insecureSkipVerify is enabled: OpenSearch server certificates are NOT verified");
|
||||
builder.loadTrustMaterial(null, (chain, authType) -> true);
|
||||
custom = true;
|
||||
}
|
||||
else if(StringUtils.hasContent(tls.getCaCertificatePath()))
|
||||
{
|
||||
builder.loadTrustMaterial(loadPemTrustStore(requireResolved(interpreter, tls.getCaCertificatePath(), "tls.caCertificatePath")), null);
|
||||
custom = true;
|
||||
}
|
||||
else if(StringUtils.hasContent(tls.getTrustStorePath()))
|
||||
{
|
||||
String password = interpreter.interpret(tls.getTrustStorePassword());
|
||||
KeyStore trustStore = KeyStore.getInstance(StringUtils.hasContent(tls.getTrustStoreType()) ? tls.getTrustStoreType() : KeyStore.getDefaultType());
|
||||
try(InputStream inputStream = new FileInputStream(requireResolved(interpreter, tls.getTrustStorePath(), "tls.trustStorePath")))
|
||||
{
|
||||
trustStore.load(inputStream, password == null ? null : password.toCharArray());
|
||||
}
|
||||
builder.loadTrustMaterial(trustStore, null);
|
||||
custom = true;
|
||||
}
|
||||
|
||||
if(StringUtils.hasContent(tls.getKeyStorePath()))
|
||||
{
|
||||
String password = interpreter.interpret(tls.getKeyStorePassword());
|
||||
char[] chars = password == null ? new char[0] : password.toCharArray();
|
||||
KeyStore keyStore = KeyStore.getInstance(StringUtils.hasContent(tls.getKeyStoreType()) ? tls.getKeyStoreType() : KeyStore.getDefaultType());
|
||||
try(InputStream inputStream = new FileInputStream(requireResolved(interpreter, tls.getKeyStorePath(), "tls.keyStorePath")))
|
||||
{
|
||||
keyStore.load(inputStream, chars);
|
||||
}
|
||||
builder.loadKeyMaterial(keyStore, chars);
|
||||
custom = true;
|
||||
}
|
||||
|
||||
return (custom ? builder.build() : null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The no-op verifier when insecureSkipVerify (warned in buildSslContext) or
|
||||
** hostnameVerification=false is set, else null for the default verifier.
|
||||
*******************************************************************************/
|
||||
static HostnameVerifier buildHostnameVerifier(QuickSearchTlsConfig tls)
|
||||
{
|
||||
if(tls == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
if(Boolean.TRUE.equals(tls.getInsecureSkipVerify()))
|
||||
{
|
||||
return (NoopHostnameVerifier.INSTANCE);
|
||||
}
|
||||
|
||||
if(Boolean.FALSE.equals(tls.getHostnameVerification()))
|
||||
{
|
||||
LOG.warn("tls.hostnameVerification is disabled: OpenSearch server certificates are NOT checked against the host name");
|
||||
return (NoopHostnameVerifier.INSTANCE);
|
||||
}
|
||||
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Interpret a value that must resolve to content; null, empty and blank all
|
||||
** count as missing. The error names the field and, for a single ${env.X} or
|
||||
** ${prop.X} reference, the reference itself; never a resolved value and
|
||||
** never any part of a ?? fallback chain.
|
||||
*******************************************************************************/
|
||||
static String requireResolved(QMetaDataVariableInterpreter interpreter, String value, String fieldName) throws QException
|
||||
{
|
||||
String resolved = interpreter.interpret(value);
|
||||
if(StringUtils.hasContent(resolved))
|
||||
{
|
||||
return (resolved);
|
||||
}
|
||||
|
||||
String trimmed = value == null ? "" : value.trim();
|
||||
boolean isReference = SINGLE_REFERENCE.matcher(trimmed).matches();
|
||||
throw (new QException(fieldName + " is missing or empty" + (isReference ? " after resolving " + trimmed : "")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load one or more PEM certificates into an in-memory trust store.
|
||||
*******************************************************************************/
|
||||
static KeyStore loadPemTrustStore(String path) throws Exception
|
||||
{
|
||||
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
|
||||
trustStore.load(null, null);
|
||||
|
||||
try(InputStream inputStream = new FileInputStream(path))
|
||||
{
|
||||
Collection<? extends Certificate> certificates = CertificateFactory.getInstance("X.509").generateCertificates(inputStream);
|
||||
int i = 0;
|
||||
for(Certificate certificate : certificates)
|
||||
{
|
||||
trustStore.setCertificateEntry("quick-search-ca-" + (i++), certificate);
|
||||
}
|
||||
if(i == 0)
|
||||
{
|
||||
throw (new QException("No certificates found in tls.caCertificatePath"));
|
||||
}
|
||||
}
|
||||
|
||||
return (trustStore);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static OpenSearchTransportCustomizer loadCustomizer(QuickSearchQBitConfig config) throws QException
|
||||
{
|
||||
if(config.getTransportCustomizer() == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
OpenSearchTransportCustomizer customizer = QCodeLoader.getAdHoc(OpenSearchTransportCustomizer.class, config.getTransportCustomizer());
|
||||
if(customizer == null)
|
||||
{
|
||||
throw (new QException("Could not load transportCustomizer [" + config.getTransportCustomizer().getName() + "]"));
|
||||
}
|
||||
return (customizer);
|
||||
}
|
||||
|
||||
}
|
||||
+569
-260
File diff suppressed because it is too large.
Load diff
+347
-171
@@ -13,64 +13,83 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Abstract base class for Quick Search indexing process steps.
|
||||
**
|
||||
** Implements {@link BackendStep} but does NOT provide a run() implementation;
|
||||
** that is left to concrete subclasses (BasepullIndexStep, FullReindexStep).
|
||||
**
|
||||
** Provides protected helper methods for:
|
||||
** - Accessing QBit config, OpenSearch client, and discovered tables
|
||||
** - Ensuring quickSearchIndex rows exist for each table
|
||||
** - Creating and completing quickSearchIndexRun records
|
||||
** - Querying batches of records from source tables
|
||||
** Shared machinery for the indexing process steps: resolving the QBit's live
|
||||
** state, operational-table bookkeeping (index rows, run records, drift
|
||||
** detection, run-history purge), and primary-key-ordered paging of source
|
||||
** tables with display values.
|
||||
*******************************************************************************/
|
||||
public abstract class AbstractIndexingStep implements BackendStep
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(AbstractIndexingStep.class);
|
||||
|
||||
public static final String STATUS_ACTIVE = "ACTIVE";
|
||||
public static final String STATUS_NEEDS_REINDEX = "NEEDS_REINDEX";
|
||||
public static final String STATUS_REBUILDING = "REBUILDING";
|
||||
public static final String RUN_COMPLETED = "COMPLETED";
|
||||
public static final String RUN_FAILED = "FAILED";
|
||||
public static final String RUN_RUNNING = "RUNNING";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Counts from indexing one table.
|
||||
*******************************************************************************/
|
||||
public record IndexCounts(Integer processed, Integer indexed, Integer errors, Integer skipped, String firstError)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return the active QuickSearchQBitConfig from context.
|
||||
**
|
||||
** @throws QException if config is null (QBit not initialized)
|
||||
*******************************************************************************/
|
||||
protected QuickSearchQBitConfig getConfig() throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = QuickSearchQBitContext.getConfig();
|
||||
if(config == null)
|
||||
{
|
||||
throw new QException("QuickSearchQBitConfig is not initialized in QuickSearchQBitContext");
|
||||
throw new QException("Quick Search QBit is not initialized (no config found in the QInstance)");
|
||||
}
|
||||
return (config);
|
||||
}
|
||||
@@ -78,16 +97,14 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return the active OpenSearch client cast from context.
|
||||
**
|
||||
** @throws QException if client is null (not initialized)
|
||||
*******************************************************************************/
|
||||
protected QuickSearchOpenSearchClient getClient() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient client = QuickSearchQBitContext.getClient();
|
||||
if(client == null)
|
||||
{
|
||||
throw new QException("QuickSearchOpenSearchClient is not initialized in QuickSearchQBitContext");
|
||||
throw new QException("Quick Search OpenSearch client is not available");
|
||||
}
|
||||
return (client);
|
||||
}
|
||||
@@ -95,27 +112,18 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return the list of discovered table configs from context.
|
||||
**
|
||||
** Returns an empty list if discoveredTables is null.
|
||||
*******************************************************************************/
|
||||
protected List<QuickSearchableTableConfig> getDiscoveredTables()
|
||||
{
|
||||
List<QuickSearchableTableConfig> tables = QuickSearchQBitContext.getDiscoveredTables();
|
||||
if(tables == null)
|
||||
{
|
||||
return (Collections.emptyList());
|
||||
}
|
||||
return (tables);
|
||||
return (tables == null ? Collections.emptyList() : tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return the table config for the given table name from context.
|
||||
**
|
||||
** Delegates to QuickSearchQBitContext.getTableConfig().
|
||||
** Returns null if not found.
|
||||
*******************************************************************************/
|
||||
protected QuickSearchableTableConfig getTableConfig(String tableName)
|
||||
{
|
||||
@@ -125,97 +133,162 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ensure a quickSearchIndex row exists for the given table.
|
||||
**
|
||||
** Queries the quickSearchIndex table for a row matching tableName.
|
||||
** If no row is found, inserts one with:
|
||||
** - tableName
|
||||
** - enabled = tableConfig.getEnabledByDefault()
|
||||
** - basepullIntervalMinutes from tableConfig
|
||||
** - basepullTimestampField from tableConfig
|
||||
** - searchableFieldsJson built from tableConfig's fields/weights/labels
|
||||
** - status = "ACTIVE"
|
||||
**
|
||||
** @param tableName the name of the source table
|
||||
** @param tableConfig the configuration for that table
|
||||
** @throws QException if the query or insert fails
|
||||
** Make sure a quickSearchIndex row exists for the table. The table's unique
|
||||
** key on tableName makes a concurrent first insert fail, which is handled by
|
||||
** re-querying.
|
||||
*******************************************************************************/
|
||||
protected void ensureIndexRowExists(String tableName, QuickSearchableTableConfig tableConfig) throws QException
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// Query for an existing index row //
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
queryInput.setFilter(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
|
||||
if(!queryOutput.getRecords().isEmpty())
|
||||
if(queryIndexRow(tableName) != null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// Build and insert a new index row. Wrapped in try/catch to handle the //
|
||||
// race condition where another thread inserts between our check and our //
|
||||
// insert -- a duplicate key exception in that case is harmless. //
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
try
|
||||
{
|
||||
String searchableFieldsJson = buildSearchableFieldsJson(tableConfig);
|
||||
|
||||
QRecord record = new QRecord()
|
||||
.withValue("tableName", tableName)
|
||||
.withValue("enabled", tableConfig.getEnabledByDefault())
|
||||
.withValue("basepullIntervalMinutes", tableConfig.getBasepullIntervalMinutes())
|
||||
.withValue("basepullTimestampField", tableConfig.getBasepullTimestampField())
|
||||
.withValue("searchableFieldsJson", searchableFieldsJson)
|
||||
.withValue("status", "ACTIVE");
|
||||
.withValue("searchableFieldsJson", buildSearchableFieldsJson(tableConfig))
|
||||
.withValue("realTimeErrorCount", 0)
|
||||
.withValue("status", STATUS_ACTIVE);
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
insertInput.setRecords(List.of(record));
|
||||
InsertOutput insertOutput = new InsertAction().execute(insertInput);
|
||||
|
||||
new InsertAction().execute(insertInput);
|
||||
if(insertOutput.getRecords().isEmpty() || CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
|
||||
{
|
||||
throw (new QException("Insert returned errors"));
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Re-query to check if another thread inserted the row //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
QueryOutput recheck = new QueryAction().execute(queryInput);
|
||||
if(recheck.getRecords().isEmpty())
|
||||
if(queryIndexRow(tableName) == null)
|
||||
{
|
||||
throw new QException("Failed to create QuickSearchIndex row for table [" + tableName + "]", e);
|
||||
}
|
||||
LOG.debug("QuickSearchIndex row already exists (concurrent insert)", "tableName", tableName);
|
||||
LOG.debug("QuickSearchIndex row already exists (concurrent insert)", logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a JSON array string representing the searchable fields configuration
|
||||
** for a table.
|
||||
**
|
||||
** Each element looks like:
|
||||
** {"fieldName":"name","weight":1,"includeLabel":false}
|
||||
**
|
||||
** @param tableConfig the table configuration containing field lists/maps
|
||||
** @return JSON string, e.g. [{"fieldName":"name","weight":1,"includeLabel":false}]
|
||||
** The quickSearchIndex row for a table, or null.
|
||||
*******************************************************************************/
|
||||
private String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
|
||||
protected QRecord queryIndexRow(String tableName) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
return (CollectionUtils.nullSafeIsEmpty(rows) ? null : rows.get(0));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The id of the quickSearchIndex row for a table.
|
||||
*******************************************************************************/
|
||||
protected Integer queryIndexId(String tableName) throws QException
|
||||
{
|
||||
QRecord row = queryIndexRow(tableName);
|
||||
if(row == null)
|
||||
{
|
||||
throw new QException("No QuickSearchIndex row found for table: " + tableName);
|
||||
}
|
||||
return (row.getValueInteger("id"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether an index row is enabled (null counts as enabled).
|
||||
*******************************************************************************/
|
||||
protected static boolean isRowEnabled(QRecord row)
|
||||
{
|
||||
Boolean enabled = row == null ? null : row.getValueBoolean("enabled");
|
||||
return (enabled == null || enabled);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compare the configured fields with what the row was created with; mark
|
||||
** the row NEEDS_REINDEX when they differ. Returns true when drift was found.
|
||||
** A REBUILDING row is left alone: the full reindex in progress records the
|
||||
** current fields, and the status tells the listener to capture deletes.
|
||||
*******************************************************************************/
|
||||
protected boolean detectDrift(QRecord row, QuickSearchableTableConfig tableConfig) throws QException
|
||||
{
|
||||
if(row == null || tableConfig == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
String current = buildSearchableFieldsJson(tableConfig);
|
||||
if(!isUnmarkedDrift(row, current))
|
||||
{
|
||||
return (hasDrift(row, current));
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// the row may be a snapshot taken before a full reindex started or //
|
||||
// finished; decide on a fresh read, so REBUILDING is kept and fields a //
|
||||
// reindex has since recorded are not flagged //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
QRecord latest = queryIndexRow(tableConfig.getTableName());
|
||||
if(latest == null || !isUnmarkedDrift(latest, current))
|
||||
{
|
||||
return (latest != null && hasDrift(latest, current));
|
||||
}
|
||||
|
||||
// a status change in the moment between this read and the update below is not guarded
|
||||
LOG.warn("Searchable field configuration changed since the index row was created; marking NEEDS_REINDEX", logPair("tableName", tableConfig.getTableName()));
|
||||
updateIndexRow(latest.getValueInteger("id"), Map.of("status", STATUS_NEEDS_REINDEX));
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the row records fields that differ from the configured ones.
|
||||
*******************************************************************************/
|
||||
private static boolean hasDrift(QRecord row, String current)
|
||||
{
|
||||
String recorded = row.getValueString("searchableFieldsJson");
|
||||
return (recorded != null && !recorded.equals(current));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the row has drifted and is not yet NEEDS_REINDEX or REBUILDING.
|
||||
*******************************************************************************/
|
||||
private static boolean isUnmarkedDrift(QRecord row, String current)
|
||||
{
|
||||
String status = row.getValueString("status");
|
||||
return (hasDrift(row, current) && !STATUS_NEEDS_REINDEX.equals(status) && !STATUS_REBUILDING.equals(status));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** JSON array describing a table's searchable fields, weights and labels.
|
||||
*******************************************************************************/
|
||||
protected static String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig)
|
||||
{
|
||||
if(tableConfig.getSearchableFields() == null || tableConfig.getSearchableFields().isEmpty())
|
||||
{
|
||||
return ("[]");
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder("[");
|
||||
boolean first = true;
|
||||
|
||||
StringBuilder sb = new StringBuilder("[");
|
||||
boolean first = true;
|
||||
for(String fieldName : tableConfig.getSearchableFields())
|
||||
{
|
||||
if(!first)
|
||||
@@ -224,153 +297,256 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
}
|
||||
first = false;
|
||||
|
||||
Integer weight = 1;
|
||||
if(tableConfig.getFieldWeights() != null && tableConfig.getFieldWeights().containsKey(fieldName))
|
||||
{
|
||||
weight = tableConfig.getFieldWeights().get(fieldName);
|
||||
}
|
||||
Integer weight = tableConfig.getFieldWeights() != null && tableConfig.getFieldWeights().containsKey(fieldName) ? tableConfig.getFieldWeights().get(fieldName) : 1;
|
||||
Boolean includeLabel = tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName) ? tableConfig.getFieldIncludeLabels().get(fieldName) : false;
|
||||
String escapedName = fieldName.replace("\\", "\\\\").replace("\"", "\\\"");
|
||||
|
||||
Boolean includeLabel = false;
|
||||
if(tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName))
|
||||
{
|
||||
includeLabel = tableConfig.getFieldIncludeLabels().get(fieldName);
|
||||
}
|
||||
|
||||
String escapedName = fieldName.replace("\\", "\\\\").replace("\"", "\\\"");
|
||||
|
||||
sb.append("{\"fieldName\":\"").append(escapedName).append("\"")
|
||||
.append(",\"weight\":").append(weight)
|
||||
.append(",\"includeLabel\":").append(includeLabel)
|
||||
.append("}");
|
||||
sb.append("{\"fieldName\":\"").append(escapedName).append("\",\"weight\":").append(weight).append(",\"includeLabel\":").append(includeLabel).append("}");
|
||||
}
|
||||
|
||||
sb.append("]");
|
||||
return (sb.toString());
|
||||
return (sb.append("]").toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a quickSearchIndexRun record and return it with its generated ID.
|
||||
**
|
||||
** Inserts a new record with status=RUNNING and startTime=now, then reads
|
||||
** back the inserted record to capture the auto-generated primary key.
|
||||
**
|
||||
** @param indexId the ID of the parent quickSearchIndex row
|
||||
** @param runType the type label for this run (e.g. "BASEPULL", "FULL_REINDEX")
|
||||
** @return a QuickSearchIndexRun entity with id populated
|
||||
** @throws QException if insert fails or no record is returned
|
||||
** Insert a RUNNING run record and return it with its id.
|
||||
*******************************************************************************/
|
||||
protected QuickSearchIndexRun createRunRecord(Integer indexId, String runType) throws QException
|
||||
{
|
||||
QRecord record = new QRecord()
|
||||
.withValue("quickSearchIndexId", indexId)
|
||||
.withValue("runType", runType)
|
||||
.withValue("status", "RUNNING")
|
||||
.withValue("startTime", Instant.now());
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
insertInput.setRecords(List.of(record));
|
||||
insertInput.setRecords(List.of(new QRecord()
|
||||
.withValue("quickSearchIndexId", indexId)
|
||||
.withValue("runType", runType)
|
||||
.withValue("status", RUN_RUNNING)
|
||||
.withValue("startTime", Instant.now())));
|
||||
|
||||
InsertOutput insertOutput = new InsertAction().execute(insertInput);
|
||||
|
||||
if(insertOutput.getRecords().isEmpty())
|
||||
{
|
||||
throw new QException("Failed to create quickSearchIndexRun record: no record returned from insert");
|
||||
}
|
||||
|
||||
QRecord insertedRecord = insertOutput.getRecords().get(0);
|
||||
|
||||
QuickSearchIndexRun run = new QuickSearchIndexRun()
|
||||
.withId(insertedRecord.getValueInteger("id"))
|
||||
.withQuickSearchIndexId(insertedRecord.getValueInteger("quickSearchIndexId"))
|
||||
.withRunType(insertedRecord.getValueString("runType"))
|
||||
.withStatus(insertedRecord.getValueString("status"))
|
||||
.withStartTime(insertedRecord.getValueInstant("startTime"));
|
||||
return (run);
|
||||
QRecord inserted = insertOutput.getRecords().get(0);
|
||||
return (new QuickSearchIndexRun()
|
||||
.withId(inserted.getValueInteger("id"))
|
||||
.withQuickSearchIndexId(inserted.getValueInteger("quickSearchIndexId"))
|
||||
.withRunType(inserted.getValueString("runType"))
|
||||
.withStatus(inserted.getValueString("status"))
|
||||
.withStartTime(inserted.getValueInstant("startTime")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Mark a quickSearchIndexRun as complete by updating its status, endTime,
|
||||
** record counts, and error information.
|
||||
**
|
||||
** Wrapped in try/catch so that failures to update the run record do not
|
||||
** mask original exceptions when called from finally blocks.
|
||||
**
|
||||
** @param run the run record to update
|
||||
** @param status the final status (e.g. "SUCCESS", "ERROR")
|
||||
** @param recordsProcessed number of records examined
|
||||
** @param recordsIndexed number of records successfully indexed
|
||||
** @param errorCount number of errors encountered
|
||||
** @param errorMessage summary error message, or null
|
||||
** Finish a run record and mirror its status onto the index row. Never
|
||||
** throws, so it is safe from finally and catch blocks.
|
||||
*******************************************************************************/
|
||||
protected void completeRunRecord(QuickSearchIndexRun run, String status, Integer recordsProcessed,
|
||||
Integer recordsIndexed, Integer errorCount, String errorMessage)
|
||||
protected void completeRunRecord(QuickSearchIndexRun run, String status, Integer recordsProcessed, Integer recordsIndexed, Integer errorCount, String errorMessage)
|
||||
{
|
||||
run.withStatus(status);
|
||||
|
||||
try
|
||||
{
|
||||
QRecord record = new QRecord()
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
updateInput.setRecords(List.of(new QRecord()
|
||||
.withValue("id", run.getId())
|
||||
.withValue("status", status)
|
||||
.withValue("endTime", Instant.now())
|
||||
.withValue("recordsProcessed", recordsProcessed)
|
||||
.withValue("recordsIndexed", recordsIndexed)
|
||||
.withValue("errorCount", errorCount)
|
||||
.withValue("errorMessage", errorMessage);
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
updateInput.setRecords(List.of(record));
|
||||
|
||||
.withValue("errorMessage", truncate(errorMessage, 4000))));
|
||||
new UpdateAction().execute(updateInput);
|
||||
|
||||
if(run.getQuickSearchIndexId() != null)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("lastRunStatus", run.getRunType() + " " + status);
|
||||
values.put("lastErrorMessage", truncate(errorMessage, 4000));
|
||||
updateIndexRow(run.getQuickSearchIndexId(), values);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to complete run record", e, "runId", run.getId(), "status", status);
|
||||
LOG.warn("Failed to complete run record", e, logPair("runId", run.getId()), logPair("status", status));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Query a batch of records from the given source table.
|
||||
**
|
||||
** @param tableName the name of the source table to query
|
||||
** @param filter the filter to apply (may be null for no filtering)
|
||||
** @param limit maximum number of records to return
|
||||
** @param offset number of records to skip (for pagination)
|
||||
** @return list of QRecord from the query
|
||||
** @throws QException if the query fails
|
||||
** Update fields on a quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
protected List<QRecord> querySourceTableBatch(String tableName, QQueryFilter filter, Integer limit, Integer offset) throws QException
|
||||
protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
|
||||
{
|
||||
QQueryFilter effectiveFilter = new QQueryFilter();
|
||||
if(filter != null && filter.getCriteria() != null)
|
||||
QRecord record = new QRecord().withValue("id", indexId);
|
||||
for(Map.Entry<String, ? extends Serializable> entry : values.entrySet())
|
||||
{
|
||||
for(QFilterCriteria criteria : filter.getCriteria())
|
||||
{
|
||||
effectiveFilter.withCriteria(criteria);
|
||||
}
|
||||
record.setValue(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
updateInput.setRecords(List.of(record));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Next page of source records in primary-key order after afterPrimaryKey
|
||||
** (null starts at the beginning), with the extra criteria applied and
|
||||
** display values generated.
|
||||
*******************************************************************************/
|
||||
protected List<QRecord> querySourceBatchAfter(String tableName, String primaryKeyField, Serializable afterPrimaryKey, List<QFilterCriteria> extraCriteria, Integer limit) throws QException
|
||||
{
|
||||
QQueryFilter filter = new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy(primaryKeyField, true))
|
||||
.withLimit(limit);
|
||||
|
||||
if(afterPrimaryKey != null)
|
||||
{
|
||||
filter.withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.GREATER_THAN, afterPrimaryKey));
|
||||
}
|
||||
for(QFilterCriteria criteria : CollectionUtils.nonNullList(extraCriteria))
|
||||
{
|
||||
filter.withCriteria(criteria);
|
||||
}
|
||||
effectiveFilter.setLimit(limit);
|
||||
effectiveFilter.setSkip(offset);
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(effectiveFilter);
|
||||
queryInput.setFilter(filter);
|
||||
queryInput.setShouldGenerateDisplayValues(true);
|
||||
queryInput.setShouldTranslatePossibleValues(true);
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
|
||||
}
|
||||
|
||||
List<QRecord> records = queryOutput.getRecords();
|
||||
if(records == null)
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Index every source record matching the criteria into targetIndex (null
|
||||
** for the alias), paging by primary key so each record is read once even
|
||||
** while rows are inserted or deleted.
|
||||
*******************************************************************************/
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String tableName = tableConfig.getTableName();
|
||||
String primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
|
||||
|
||||
int processed = 0;
|
||||
int indexed = 0;
|
||||
int errors = 0;
|
||||
int skipped = 0;
|
||||
String firstError = null;
|
||||
|
||||
Serializable lastPrimaryKey = null;
|
||||
while(true)
|
||||
{
|
||||
return (new ArrayList<>());
|
||||
List<QRecord> batch = querySourceBatchAfter(tableName, primaryKeyField, lastPrimaryKey, extraCriteria, config.getSourceBatchSize());
|
||||
if(batch.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
List<OpenSearchDocument> documents = new ArrayList<>();
|
||||
for(QRecord record : batch)
|
||||
{
|
||||
try
|
||||
{
|
||||
OpenSearchDocument document = IndexingUtils.buildDocument(record, tableConfig);
|
||||
if(document != null)
|
||||
{
|
||||
documents.add(document);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
errors++;
|
||||
if(firstError == null)
|
||||
{
|
||||
firstError = "Could not build document for " + tableName + ":" + record.getValueString(primaryKeyField) + ": " + e.getMessage();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!documents.isEmpty())
|
||||
{
|
||||
BulkIndexResult result = targetIndex == null ? client.indexDocuments(documents, config.getBulkBatchSize()) : client.indexDocuments(targetIndex, documents, config.getBulkBatchSize());
|
||||
indexed += result.getSuccessCount();
|
||||
errors += result.getFailureCount();
|
||||
skipped += result.getSkippedCount();
|
||||
if(firstError == null && !result.getErrors().isEmpty())
|
||||
{
|
||||
firstError = result.getErrors().get(0);
|
||||
}
|
||||
}
|
||||
|
||||
processed += batch.size();
|
||||
lastPrimaryKey = batch.get(batch.size() - 1).getValue(primaryKeyField);
|
||||
if(lastPrimaryKey == null)
|
||||
{
|
||||
throw new QException("Source record has no value in primary key field [" + primaryKeyField + "]");
|
||||
}
|
||||
}
|
||||
|
||||
return (records);
|
||||
return (new IndexCounts(processed, indexed, errors, skipped, firstError));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Delete run records older than runHistoryRetentionDays. Never throws.
|
||||
*******************************************************************************/
|
||||
protected void purgeOldRuns()
|
||||
{
|
||||
try
|
||||
{
|
||||
Integer retentionDays = getConfig().getRunHistoryRetentionDays();
|
||||
if(retentionDays == null || retentionDays <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Instant cutoff = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("startTime", QCriteriaOperator.LESS_THAN, cutoff)).withLimit(5000));
|
||||
List<QRecord> old = new QueryAction().execute(queryInput).getRecords();
|
||||
if(CollectionUtils.nullSafeIsEmpty(old))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
for(QRecord record : old)
|
||||
{
|
||||
ids.add(record.getValue("id"));
|
||||
}
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
deleteInput.setPrimaryKeys(ids);
|
||||
new DeleteAction().execute(deleteInput);
|
||||
LOG.info("Purged old Quick Search run records", logPair("count", ids.size()), logPair("olderThanDays", retentionDays));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to purge old run records", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected static String truncate(String s, int max)
|
||||
{
|
||||
return (s == null || s.length() <= max ? s : s.substring(0, max));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -13,96 +13,104 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Incremental basepull indexing process step.
|
||||
** Scheduled catch-up. For each enabled table that is due, re-indexes records
|
||||
** whose basepull timestamp is at or after the previous run's start minus the
|
||||
** configured overlap, paging by primary key. First replays failed real-time
|
||||
** events, and afterwards purges old run history.
|
||||
**
|
||||
** Queries all enabled QuickSearchIndex rows, checks whether each is due for a
|
||||
** basepull run, and for those that are, queries the source table for records
|
||||
** modified since the last basepull, builds OpenSearch documents, and bulk-indexes
|
||||
** them.
|
||||
** The watermark stored on the quickSearchIndex row is the run's start time,
|
||||
** captured before the first query, so rows modified during the run are read
|
||||
** again next time (re-indexing is idempotent). A table without a timestamp
|
||||
** field is left to the reconcile process.
|
||||
**
|
||||
** Run records are created at the start of each per-table run and updated with
|
||||
** final status and counts on completion. Per-table failures are caught and
|
||||
** logged so that one bad table does not prevent others from being processed.
|
||||
** One table's failure does not stop the others, but the step fails at the end
|
||||
** if any table failed, so the scheduler and run history show it.
|
||||
*******************************************************************************/
|
||||
public class BasepullIndexStep extends AbstractIndexingStep
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(BasepullIndexStep.class);
|
||||
|
||||
public static final String RUN_TYPE = "BASEPULL";
|
||||
public static final int MAX_REPLAY_ATTEMPTS = 10;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Main entry point called by the QQQ process engine.
|
||||
**
|
||||
** 1. For each discovered table, ensure a quickSearchIndex row exists.
|
||||
** 2. Query all enabled quickSearchIndex rows.
|
||||
** 3. For each enabled row, check isDueForBasepull and run basepullIndex.
|
||||
**
|
||||
** @param input process step input (not directly used)
|
||||
** @param output process step output (not directly used)
|
||||
** @throws QException if an unrecoverable error occurs (per-table errors are caught)
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// 1. Ensure index rows exist for all tables //
|
||||
////////////////////////////////////////////////////
|
||||
List<QuickSearchableTableConfig> discoveredTables = getDiscoveredTables();
|
||||
for(QuickSearchableTableConfig tableConfig : discoveredTables)
|
||||
List<String> failures = new ArrayList<>();
|
||||
|
||||
try
|
||||
{
|
||||
replayFailedEvents(output);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Replaying failed events did not complete", e);
|
||||
failures.add("failed-event replay: " + e.getMessage());
|
||||
}
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
|
||||
{
|
||||
ensureIndexRowExists(tableConfig.getTableName(), tableConfig);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 2. Query all enabled quickSearchIndex rows //
|
||||
////////////////////////////////////////////////////
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
queryInput.setFilter(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)));
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)));
|
||||
List<QRecord> indexRecords = new QueryAction().execute(queryInput).getRecords();
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
List<QRecord> indexRecords = queryOutput.getRecords();
|
||||
|
||||
if(indexRecords == null || indexRecords.isEmpty())
|
||||
if(CollectionUtils.nullSafeIsEmpty(indexRecords))
|
||||
{
|
||||
LOG.info("No enabled quickSearchIndex rows found; nothing to basepull");
|
||||
purgeOldRuns();
|
||||
return;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// 3. Process each enabled index row //
|
||||
////////////////////////////////////////////////////
|
||||
Integer tablesIndexed = 0;
|
||||
for(QRecord indexRecord : indexRecords)
|
||||
{
|
||||
QuickSearchIndex index = new QuickSearchIndex()
|
||||
@@ -113,188 +121,299 @@ public class BasepullIndexStep extends AbstractIndexingStep
|
||||
.withBasepullTimestampField(indexRecord.getValueString("basepullTimestampField"))
|
||||
.withLastBasepullTime(indexRecord.getValueInstant("lastBasepullTime"));
|
||||
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(index.getTableName());
|
||||
detectDrift(indexRecord, tableConfig);
|
||||
|
||||
if(!isDueForBasepull(index))
|
||||
{
|
||||
LOG.info("Index not due for basepull; skipping",
|
||||
"tableName", index.getTableName(),
|
||||
"lastBasepullTime", index.getLastBasepullTime());
|
||||
LOG.debug("Index not due for basepull; skipping", logPair("tableName", index.getTableName()), logPair("lastBasepullTime", index.getLastBasepullTime()));
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
basepullIndex(index);
|
||||
tablesIndexed++;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Error during basepull for table; continuing with next table",
|
||||
e, "tableName", index.getTableName());
|
||||
LOG.warn("Error during basepull for table; continuing with next table", e, logPair("tableName", index.getTableName()));
|
||||
failures.add(index.getTableName() + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
purgeOldRuns();
|
||||
output.addValue("tablesIndexed", tablesIndexed);
|
||||
|
||||
if(!failures.isEmpty())
|
||||
{
|
||||
throw (new QException("Basepull failed for " + failures.size() + " item(s): " + String.join("; ", failures)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Determine whether this index is due for a basepull run.
|
||||
**
|
||||
** Returns true when lastBasepullTime is null (never run), or when the elapsed
|
||||
** time since the last run has exceeded the configured interval.
|
||||
**
|
||||
** @param index the QuickSearchIndex row to evaluate
|
||||
** @return true if a basepull should be executed for this index
|
||||
** Due when never run, or when the interval has elapsed since the last run.
|
||||
*******************************************************************************/
|
||||
boolean isDueForBasepull(QuickSearchIndex index)
|
||||
{
|
||||
Instant lastBasepullTime = index.getLastBasepullTime();
|
||||
|
||||
if(lastBasepullTime == null)
|
||||
Integer intervalMinutes = index.getBasepullIntervalMinutes();
|
||||
if(lastBasepullTime == null || intervalMinutes == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
Integer intervalMinutes = index.getBasepullIntervalMinutes();
|
||||
if(intervalMinutes == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
return (lastBasepullTime.plus(intervalMinutes, ChronoUnit.MINUTES).isBefore(Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute a basepull indexing run for one QuickSearchIndex row.
|
||||
**
|
||||
** Steps:
|
||||
** 1. Resolve table config and client.
|
||||
** 2. Create a run record (status=RUNNING).
|
||||
** 3. Build an optional QQueryFilter using lastBasepullTime.
|
||||
** 4. Paginate through the source table, building and indexing documents.
|
||||
** 5. Update lastBasepullTime on the QuickSearchIndex row.
|
||||
** 6. Complete the run record with final status and counts.
|
||||
**
|
||||
** @param index the QuickSearchIndex row driving this run
|
||||
** @throws QException if an error occurs that cannot be handled locally
|
||||
** One table's basepull run.
|
||||
*******************************************************************************/
|
||||
void basepullIndex(QuickSearchIndex index) throws QException
|
||||
{
|
||||
String tableName = index.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String tableName = index.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
|
||||
QuickSearchIndexRun run = createRunRecord(index.getId(), "BASEPULL");
|
||||
QuickSearchIndexRun run = createRunRecord(index.getId(), RUN_TYPE);
|
||||
|
||||
Integer totalProcessed = 0;
|
||||
Integer totalIndexed = 0;
|
||||
Integer totalErrors = 0;
|
||||
String errorMessage = null;
|
||||
if(tableConfig == null)
|
||||
{
|
||||
String message = "Table is no longer configured for Quick Search; disable or remove its quickSearchIndex row";
|
||||
completeRunRecord(run, RUN_FAILED, 0, 0, 1, message);
|
||||
throw (new QException(tableName + ": " + message));
|
||||
}
|
||||
|
||||
Instant runStart = Instant.now();
|
||||
|
||||
try
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Build filter: timestamp field > lastBasepullTime (if set) //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
QQueryFilter filter = null;
|
||||
|
||||
if(index.getLastBasepullTime() != null && tableConfig != null
|
||||
&& tableConfig.getBasepullTimestampField() != null)
|
||||
String timestampField = tableConfig.getBasepullTimestampField();
|
||||
if(timestampField == null && index.getLastBasepullTime() != null)
|
||||
{
|
||||
filter = new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria(
|
||||
tableConfig.getBasepullTimestampField(),
|
||||
QCriteriaOperator.GREATER_THAN,
|
||||
index.getLastBasepullTime()));
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// no change-detection field: only the first run indexes everything; //
|
||||
// later changes are caught by reconcile //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
completeRunRecord(run, RUN_COMPLETED, 0, 0, 0, null);
|
||||
return;
|
||||
}
|
||||
|
||||
Integer sourceBatchSize = config.getSourceBatchSize();
|
||||
Integer bulkBatchSize = config.getBulkBatchSize();
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Paginate through the source table //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
int offset = 0;
|
||||
|
||||
while(true)
|
||||
List<QFilterCriteria> criteria = new ArrayList<>();
|
||||
if(timestampField != null && index.getLastBasepullTime() != null)
|
||||
{
|
||||
List<QRecord> batch = querySourceTableBatch(tableName, filter, sourceBatchSize, offset);
|
||||
|
||||
if(batch.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Build documents for this batch //
|
||||
////////////////////////////////////////////////////
|
||||
List<OpenSearchDocument> documents = new ArrayList<>();
|
||||
|
||||
for(QRecord record : batch)
|
||||
{
|
||||
if(tableConfig != null)
|
||||
{
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
documents.add(doc);
|
||||
}
|
||||
}
|
||||
|
||||
documents.removeIf(doc -> doc == null);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Bulk-index the batch //
|
||||
////////////////////////////////////////////////////
|
||||
if(!documents.isEmpty())
|
||||
{
|
||||
BulkIndexResult result = client.indexDocuments(documents, bulkBatchSize);
|
||||
totalIndexed = totalIndexed + result.getSuccessCount();
|
||||
totalErrors = totalErrors + result.getFailureCount();
|
||||
|
||||
if(!result.getErrors().isEmpty() && errorMessage == null)
|
||||
{
|
||||
errorMessage = result.getErrors().get(0);
|
||||
}
|
||||
}
|
||||
|
||||
totalProcessed = totalProcessed + batch.size();
|
||||
offset = offset + batch.size();
|
||||
int overlap = config.getBasepullOverlapSeconds() == null ? 0 : config.getBasepullOverlapSeconds();
|
||||
Instant since = index.getLastBasepullTime().minusSeconds(overlap);
|
||||
criteria.add(new QFilterCriteria(timestampField, QCriteriaOperator.GREATER_THAN_OR_EQUALS, since));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Update lastBasepullTime on the index row //
|
||||
// only when no indexing errors occurred //
|
||||
////////////////////////////////////////////////////
|
||||
if(totalErrors.equals(0))
|
||||
IndexCounts counts = indexAllRecords(client, tableConfig, criteria, null);
|
||||
|
||||
if(counts.errors() == 0)
|
||||
{
|
||||
QRecord updateRecord = new QRecord()
|
||||
.withValue("id", index.getId())
|
||||
.withValue("lastBasepullTime", Instant.now());
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
updateInput.setRecords(List.of(updateRecord));
|
||||
|
||||
new UpdateAction().execute(updateInput);
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("lastBasepullTime", runStart);
|
||||
Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
|
||||
if(documentCount != null)
|
||||
{
|
||||
values.put("documentCount", documentCount);
|
||||
}
|
||||
updateIndexRow(index.getId(), values);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Complete the run record //
|
||||
////////////////////////////////////////////////////
|
||||
String finalStatus = (totalErrors > 0) ? "FAILED" : "COMPLETED";
|
||||
completeRunRecord(run, finalStatus, totalProcessed, totalIndexed, totalErrors, errorMessage);
|
||||
String status = counts.errors() > 0 ? RUN_FAILED : RUN_COMPLETED;
|
||||
completeRunRecord(run, status, counts.processed(), counts.indexed(), counts.errors(), counts.firstError());
|
||||
|
||||
LOG.info("Basepull complete",
|
||||
"tableName", tableName,
|
||||
"totalProcessed", totalProcessed,
|
||||
"totalIndexed", totalIndexed,
|
||||
"totalErrors", totalErrors);
|
||||
LOG.info("Basepull complete", logPair("tableName", tableName), logPair("processed", counts.processed()), logPair("indexed", counts.indexed()),
|
||||
logPair("skippedStale", counts.skipped()), logPair("errors", counts.errors()));
|
||||
|
||||
if(counts.errors() > 0)
|
||||
{
|
||||
throw (new QException("Basepull indexed " + counts.indexed() + " of " + counts.processed() + " records for table [" + tableName + "]; first error: " + counts.firstError()));
|
||||
}
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
if(!RUN_FAILED.equals(run.getStatus()))
|
||||
{
|
||||
completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
|
||||
run.withStatus(RUN_FAILED);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Basepull failed for table", e, "tableName", tableName);
|
||||
completeRunRecord(run, "FAILED", totalProcessed, totalIndexed, totalErrors, e.getMessage());
|
||||
throw e;
|
||||
completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
|
||||
throw (new QException("Basepull failed for table [" + tableName + "]: " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Replay PENDING failed real-time events: re-read and index records for
|
||||
** INDEX events, delete documents for DELETE events. Successes are removed;
|
||||
** failures bump attempts and become EXHAUSTED at the limit.
|
||||
*******************************************************************************/
|
||||
void replayFailedEvents(RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(failedEventTable);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_PENDING))
|
||||
.withOrderBy(new QFilterOrderBy("id", true))
|
||||
.withLimit(config.getSourceBatchSize()));
|
||||
List<QRecord> events = new QueryAction().execute(queryInput).getRecords();
|
||||
|
||||
if(CollectionUtils.nullSafeIsEmpty(events))
|
||||
{
|
||||
output.addValue("failedEventsReplayed", 0);
|
||||
return;
|
||||
}
|
||||
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
|
||||
Map<String, List<QRecord>> indexByTable = new LinkedHashMap<>();
|
||||
Map<String, List<QRecord>> deleteByTable = new LinkedHashMap<>();
|
||||
for(QRecord event : events)
|
||||
{
|
||||
Map<String, List<QRecord>> target = "DELETE".equals(event.getValueString("action")) ? deleteByTable : indexByTable;
|
||||
target.computeIfAbsent(event.getValueString("tableName"), k -> new ArrayList<>()).add(event);
|
||||
}
|
||||
|
||||
List<Serializable> succeeded = new ArrayList<>();
|
||||
List<QRecord> failed = new ArrayList<>();
|
||||
|
||||
for(Map.Entry<String, List<QRecord>> entry : deleteByTable.entrySet())
|
||||
{
|
||||
List<String> documentIds = new ArrayList<>();
|
||||
for(QRecord event : entry.getValue())
|
||||
{
|
||||
documentIds.add(OpenSearchDocument.buildDocumentId(entry.getKey(), event.getValueString("recordId")));
|
||||
}
|
||||
try
|
||||
{
|
||||
BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
|
||||
markReplayOutcome(entry.getValue(), result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
markReplayOutcome(entry.getValue(), false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
|
||||
}
|
||||
}
|
||||
|
||||
for(Map.Entry<String, List<QRecord>> entry : indexByTable.entrySet())
|
||||
{
|
||||
String tableName = entry.getKey();
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
markReplayOutcome(entry.getValue(), false, List.of("table is no longer configured"), succeeded, failed);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
String primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
for(QRecord event : entry.getValue())
|
||||
{
|
||||
ids.add(event.getValueString("recordId"));
|
||||
}
|
||||
|
||||
QueryInput sourceQuery = new QueryInput();
|
||||
sourceQuery.setTableName(tableName);
|
||||
sourceQuery.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, ids)));
|
||||
sourceQuery.setShouldGenerateDisplayValues(true);
|
||||
sourceQuery.setShouldTranslatePossibleValues(true);
|
||||
List<QRecord> records = CollectionUtils.nonNullList(new QueryAction().execute(sourceQuery).getRecords());
|
||||
|
||||
List<OpenSearchDocument> documents = new ArrayList<>();
|
||||
List<String> foundIds = new ArrayList<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
OpenSearchDocument document = IndexingUtils.buildDocument(record, tableConfig);
|
||||
if(document != null)
|
||||
{
|
||||
documents.add(document);
|
||||
foundIds.add(document.getRecordId());
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// a record deleted since the event was recorded has no document to //
|
||||
// index; remove any stale document for it instead //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
List<String> goneDocumentIds = new ArrayList<>();
|
||||
for(Serializable id : ids)
|
||||
{
|
||||
if(!foundIds.contains(String.valueOf(id)))
|
||||
{
|
||||
goneDocumentIds.add(OpenSearchDocument.buildDocumentId(tableName, String.valueOf(id)));
|
||||
}
|
||||
}
|
||||
|
||||
BulkIndexResult result = client.indexDocuments(documents, config.getBulkBatchSize());
|
||||
if(!goneDocumentIds.isEmpty())
|
||||
{
|
||||
BulkIndexResult deleteResult = client.deleteDocuments(goneDocumentIds, config.getBulkBatchSize());
|
||||
deleteResult.getErrors().forEach(result::addFailure);
|
||||
}
|
||||
markReplayOutcome(entry.getValue(), result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
markReplayOutcome(entry.getValue(), false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
|
||||
}
|
||||
}
|
||||
|
||||
if(!succeeded.isEmpty())
|
||||
{
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(failedEventTable);
|
||||
deleteInput.setPrimaryKeys(succeeded);
|
||||
new DeleteAction().execute(deleteInput);
|
||||
}
|
||||
|
||||
if(!failed.isEmpty())
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(failedEventTable);
|
||||
updateInput.setRecords(failed);
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
LOG.info("Replayed failed real-time events", logPair("succeeded", succeeded.size()), logPair("stillFailing", failed.size()));
|
||||
output.addValue("failedEventsReplayed", succeeded.size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void markReplayOutcome(List<QRecord> events, boolean success, List<String> errors, List<Serializable> succeeded, List<QRecord> failed)
|
||||
{
|
||||
for(QRecord event : events)
|
||||
{
|
||||
if(success)
|
||||
{
|
||||
succeeded.add(event.getValue("id"));
|
||||
continue;
|
||||
}
|
||||
|
||||
int attempts = (event.getValueInteger("attempts") == null ? 0 : event.getValueInteger("attempts")) + 1;
|
||||
failed.add(new QRecord()
|
||||
.withValue("id", event.getValue("id"))
|
||||
.withValue("attempts", attempts)
|
||||
.withValue("errorMessage", truncate(errors == null || errors.isEmpty() ? "replay failed" : errors.get(0), 4000))
|
||||
.withValue("status", attempts >= MAX_REPLAY_ATTEMPTS ? QuickSearchFailedEvent.STATUS_EXHAUSTED : QuickSearchFailedEvent.STATUS_PENDING));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,207 +17,478 @@
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Process step that performs a full reindex of one or all tables.
|
||||
** Rebuild the index from source.
|
||||
**
|
||||
** Deletes all existing OpenSearch documents for each target table, then
|
||||
** re-fetches and re-indexes every record in batches. Updates the
|
||||
** QuickSearchIndex row with lastFullReindexTime and recordCount when done.
|
||||
** Without a "tableName" input, every enabled table is indexed into a fresh
|
||||
** physical index with the current mapping; when all of them succeed the alias
|
||||
** is swapped to it and the old index is deleted, so search keeps serving the
|
||||
** old documents until the new set is complete. This is also how an index
|
||||
** created by an older release (outdated mapping) is upgraded. On failure the
|
||||
** new physical index is deleted and nothing changes.
|
||||
**
|
||||
** An optional "tableName" input field narrows the run to a single table.
|
||||
** When absent, all discovered tables are reindexed.
|
||||
** While the new index is built, real-time writes still go through the alias
|
||||
** to the old one. Updates are caught by the next basepull (lastBasepullTime
|
||||
** is set to each table's start). Deletes are caught here: each table's
|
||||
** quickSearchIndex row is REBUILDING for the whole rebuild, which makes the
|
||||
** record change listener (on any node) also record each delete as an
|
||||
** AWAITING_REINDEX failed event. Once the rebuild's own writes are done and
|
||||
** the alias is swapped, those deletes are applied through the alias.
|
||||
**
|
||||
** With a "tableName" input, that table is rebuilt in place with the reconcile
|
||||
** algorithm (index everything, then delete what was not re-indexed).
|
||||
*******************************************************************************/
|
||||
public class FullReindexStep extends AbstractIndexingStep
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(FullReindexStep.class);
|
||||
|
||||
public static final String RUN_TYPE = "FULL_REINDEX";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run the full reindex step.
|
||||
**
|
||||
** Reads the optional "tableName" field from the input. When present, only
|
||||
** that table is reindexed; otherwise every discovered table is processed.
|
||||
**
|
||||
** @param input the backend step input (may contain "tableName")
|
||||
** @param output the backend step output (unused but required by interface)
|
||||
** @throws QException if a fatal error occurs preventing any reindex
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
String tableNameFilter = input.getValueString("tableName");
|
||||
|
||||
List<QuickSearchableTableConfig> tables = getDiscoveredTables();
|
||||
List<QuickSearchableTableConfig> tables = new ArrayList<>();
|
||||
for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
|
||||
{
|
||||
if(tableNameFilter != null && !tableNameFilter.isBlank() && !tableNameFilter.equals(tableConfig.getTableName()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
ensureIndexRowExists(tableConfig.getTableName(), tableConfig);
|
||||
if(isRowEnabled(queryIndexRow(tableConfig.getTableName())))
|
||||
{
|
||||
tables.add(tableConfig);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.info("Table is disabled; skipping full reindex", logPair("tableName", tableConfig.getTableName()));
|
||||
}
|
||||
}
|
||||
|
||||
if(tableNameFilter != null && !tableNameFilter.isBlank())
|
||||
{
|
||||
List<QuickSearchableTableConfig> filtered = new ArrayList<>();
|
||||
Integer recordsIndexed = 0;
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
if(tableNameFilter.equals(tableConfig.getTableName()))
|
||||
{
|
||||
filtered.add(tableConfig);
|
||||
}
|
||||
recordsIndexed = recordsIndexed + new ReconcileIndexStep().reconcileTable(tableConfig, RUN_TYPE).recordsIndexed();
|
||||
}
|
||||
tables = filtered;
|
||||
output.addValue("recordsIndexed", recordsIndexed);
|
||||
output.addValue("aliasSwapped", false);
|
||||
return;
|
||||
}
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
output.addValue("recordsIndexed", rebuildAllTables(tables));
|
||||
output.addValue("aliasSwapped", true);
|
||||
}
|
||||
|
||||
ensureIndexRowExists(tableName, tableConfig);
|
||||
reindexTable(tableName, tableConfig);
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Index every table into a new physical index, then swap the alias.
|
||||
*******************************************************************************/
|
||||
Integer rebuildAllTables(List<QuickSearchableTableConfig> tables) throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
String physicalIndex = client.newPhysicalIndexName();
|
||||
|
||||
client.createPhysicalIndex(physicalIndex);
|
||||
LOG.info("Rebuilding Quick Search index", logPair("physicalIndex", physicalIndex), logPair("tables", tables.size()));
|
||||
|
||||
Map<QuickSearchableTableConfig, QuickSearchIndexRun> runs = new LinkedHashMap<>();
|
||||
Map<String, IndexCounts> counts = new LinkedHashMap<>();
|
||||
Map<String, Instant> startTimes = new LinkedHashMap<>();
|
||||
Map<String, String> priorStatus = new LinkedHashMap<>();
|
||||
Integer indexed = 0;
|
||||
|
||||
boolean swapped = false;
|
||||
|
||||
try
|
||||
{
|
||||
markRebuilding(tables, priorStatus);
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
QuickSearchIndexRun run = createRunRecord(queryIndexId(tableName), RUN_TYPE);
|
||||
runs.put(tableConfig, run);
|
||||
startTimes.put(tableName, Instant.now());
|
||||
|
||||
IndexCounts tableCounts = indexAllRecords(client, tableConfig, List.of(), physicalIndex);
|
||||
counts.put(tableName, tableCounts);
|
||||
indexed = indexed + tableCounts.indexed();
|
||||
|
||||
if(tableCounts.errors() > 0)
|
||||
{
|
||||
throw (new QException("Full reindex indexed " + tableCounts.indexed() + " of " + tableCounts.processed() + " records for table [" + tableName + "]; first error: " + tableCounts.firstError()));
|
||||
}
|
||||
}
|
||||
|
||||
client.refreshIndex(physicalIndex);
|
||||
client.swapAliasTo(physicalIndex);
|
||||
swapped = true;
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
IndexCounts tableCounts = counts.get(tableName);
|
||||
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("lastFullReindexTime", Instant.now());
|
||||
values.put("lastBasepullTime", startTimes.get(tableName));
|
||||
values.put("recordCount", tableCounts.processed());
|
||||
Integer documentCount = ReconcileIndexStep.safeCount(client, tableName);
|
||||
if(documentCount != null)
|
||||
{
|
||||
values.put("documentCount", documentCount);
|
||||
}
|
||||
values.put("searchableFieldsJson", buildSearchableFieldsJson(tableConfig));
|
||||
values.put("status", STATUS_ACTIVE);
|
||||
updateIndexRow(runs.get(tableConfig).getQuickSearchIndexId(), values);
|
||||
priorStatus.remove(tableName);
|
||||
|
||||
completeRunRecord(runs.get(tableConfig), RUN_COMPLETED, tableCounts.processed(), tableCounts.indexed(), 0, null);
|
||||
}
|
||||
|
||||
applyCapturedDeletes(client, tables);
|
||||
|
||||
LOG.info("Full reindex complete", logPair("physicalIndex", physicalIndex), logPair("recordsIndexed", indexed));
|
||||
return (indexed);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn(swapped ? "Full reindex failed after the alias swap; the new index stays in service" : "Full reindex failed; the previous index stays in service", e, logPair("physicalIndex", physicalIndex));
|
||||
|
||||
for(Map.Entry<QuickSearchableTableConfig, QuickSearchIndexRun> entry : runs.entrySet())
|
||||
{
|
||||
IndexCounts tableCounts = counts.get(entry.getKey().getTableName());
|
||||
completeRunRecord(entry.getValue(), RUN_FAILED,
|
||||
tableCounts == null ? 0 : tableCounts.processed(),
|
||||
tableCounts == null ? 0 : tableCounts.indexed(),
|
||||
tableCounts == null ? 1 : Math.max(1, tableCounts.errors()),
|
||||
e.getMessage());
|
||||
}
|
||||
|
||||
if(!swapped)
|
||||
{
|
||||
try
|
||||
{
|
||||
client.deletePhysicalIndex(physicalIndex);
|
||||
}
|
||||
catch(Exception cleanup)
|
||||
{
|
||||
LOG.warn("Could not delete the incomplete physical index", cleanup, logPair("physicalIndex", physicalIndex));
|
||||
}
|
||||
}
|
||||
|
||||
restoreStatus(priorStatus);
|
||||
applyCapturedDeletes(client, tables);
|
||||
|
||||
if(e instanceof QException qException)
|
||||
{
|
||||
throw qException;
|
||||
}
|
||||
throw new QException("Full reindex failed: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reindex all records for a single table.
|
||||
**
|
||||
** Flow:
|
||||
** 1. Obtain the QuickSearchIndex row for this table (to get its ID).
|
||||
** 2. Create a FULL_REINDEX run record.
|
||||
** 3. Delete all existing OpenSearch documents for the table.
|
||||
** 4. Paginate through all source records and index them in batches.
|
||||
** 5. Update lastFullReindexTime and recordCount on the QuickSearchIndex row.
|
||||
** 6. Complete the run record with final stats.
|
||||
**
|
||||
** On any exception, the run record is marked FAILED before re-throwing.
|
||||
**
|
||||
** @param tableName the name of the source table
|
||||
** @param tableConfig the configuration for that table
|
||||
** @throws QException if a fatal indexing error occurs
|
||||
** Mark each table's index row REBUILDING before any of them is read,
|
||||
** remembering the status it had. A stale REBUILDING left by a crashed run
|
||||
** is remembered as ACTIVE.
|
||||
*******************************************************************************/
|
||||
void reindexTable(String tableName, QuickSearchableTableConfig tableConfig) throws QException
|
||||
private void markRebuilding(List<QuickSearchableTableConfig> tables, Map<String, String> priorStatus) throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Query for the index row to get its ID //
|
||||
////////////////////////////////////////////////////
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
queryInput.setFilter(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
|
||||
if(queryOutput.getRecords().isEmpty())
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
throw new QException("No QuickSearchIndex row found for table: " + tableName);
|
||||
QRecord row = queryIndexRow(tableConfig.getTableName());
|
||||
String status = row.getValueString("status");
|
||||
priorStatus.put(tableConfig.getTableName(), status == null || STATUS_REBUILDING.equals(status) ? STATUS_ACTIVE : status);
|
||||
updateIndexRow(row.getValueInteger("id"), Map.of("status", STATUS_REBUILDING));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** After a failed rebuild, put back the status of each table still marked
|
||||
** REBUILDING. Never throws.
|
||||
*******************************************************************************/
|
||||
private void restoreStatus(Map<String, String> priorStatus)
|
||||
{
|
||||
for(Map.Entry<String, String> entry : priorStatus.entrySet())
|
||||
{
|
||||
try
|
||||
{
|
||||
updateIndexRow(queryIndexId(entry.getKey()), Map.of("status", entry.getValue()));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not restore the index row status after a failed full reindex", e, logPair("tableName", entry.getKey()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply the deletes the listener captured during the rebuild (failed-event
|
||||
** rows with status AWAITING_REINDEX) through the alias, after the rebuild's
|
||||
** own writes, so an in-flight page cannot bring a deleted record back.
|
||||
** Called once the tables are no longer REBUILDING, so the listener stops
|
||||
** adding rows (one that races the status change hands its own rows to
|
||||
** basepull).
|
||||
**
|
||||
** Rows are read in id order, past the previous batch, so a batch whose rows
|
||||
** could not be removed or updated (QQQ reports those as per-row errors, not
|
||||
** exceptions) is not read again; such rows stay AWAITING_REINDEX for the next
|
||||
** full reindex. Never throws.
|
||||
*******************************************************************************/
|
||||
void applyCapturedDeletes(QuickSearchOpenSearchClient client, List<QuickSearchableTableConfig> tables)
|
||||
{
|
||||
List<Serializable> tableNames = new ArrayList<>();
|
||||
tables.forEach(tableConfig -> tableNames.add(tableConfig.getTableName()));
|
||||
if(tableNames.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
QRecord indexRecord = queryOutput.getRecords().get(0);
|
||||
Integer indexId = indexRecord.getValueInteger("id");
|
||||
|
||||
QuickSearchIndexRun run = createRunRecord(indexId, "FULL_REINDEX");
|
||||
|
||||
Integer totalProcessed = 0;
|
||||
Integer totalIndexed = 0;
|
||||
Integer totalErrors = 0;
|
||||
|
||||
Integer applied = 0;
|
||||
try
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// Delete existing documents for this table //
|
||||
////////////////////////////////////////////////////
|
||||
client.deleteDocumentsForTable(tableName);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Paginate through all records and index them //
|
||||
////////////////////////////////////////////////////
|
||||
Integer sourceBatchSize = config.getSourceBatchSize();
|
||||
Integer offset = 0;
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
|
||||
Serializable afterId = null;
|
||||
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> batch = querySourceTableBatch(tableName, new QQueryFilter(), sourceBatchSize, offset);
|
||||
QQueryFilter filter = new QQueryFilter(
|
||||
new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX),
|
||||
new QFilterCriteria("tableName", QCriteriaOperator.IN, tableNames))
|
||||
.withOrderBy(new QFilterOrderBy("id", true))
|
||||
.withLimit(config.getSourceBatchSize());
|
||||
if(afterId != null)
|
||||
{
|
||||
filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, afterId));
|
||||
}
|
||||
|
||||
if(batch.isEmpty())
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(failedEventTable);
|
||||
queryInput.setFilter(filter);
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
if(CollectionUtils.nullSafeIsEmpty(rows))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
List<OpenSearchDocument> documents = new ArrayList<>();
|
||||
for(QRecord record : batch)
|
||||
afterId = rows.get(rows.size() - 1).getValue("id");
|
||||
try
|
||||
{
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
documents.add(doc);
|
||||
applied = applied + applyCapturedDeleteBatch(client, config, failedEventTable, rows);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not apply a batch of deletes captured during the full reindex; they stay AWAITING_REINDEX for the next full reindex", e,
|
||||
logPair("tableNames", rows.stream().map(row -> row.getValueString("tableName")).distinct().toList()), logPair("count", rows.size()));
|
||||
}
|
||||
|
||||
documents.removeIf(doc -> doc == null);
|
||||
|
||||
BulkIndexResult result = client.indexDocuments(documents, config.getBulkBatchSize());
|
||||
|
||||
totalProcessed += batch.size();
|
||||
totalIndexed += result.getSuccessCount();
|
||||
totalErrors += result.getFailureCount();
|
||||
|
||||
offset += batch.size();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Update the QuickSearchIndex row //
|
||||
// only when no indexing errors occurred //
|
||||
////////////////////////////////////////////////////
|
||||
if(totalErrors.equals(0))
|
||||
{
|
||||
QRecord updateRecord = new QRecord()
|
||||
.withValue("id", indexId)
|
||||
.withValue("lastFullReindexTime", Instant.now())
|
||||
.withValue("lastBasepullTime", Instant.now())
|
||||
.withValue("recordCount", totalProcessed);
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
updateInput.setRecords(List.of(updateRecord));
|
||||
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Complete the run record //
|
||||
////////////////////////////////////////////////////
|
||||
String finalStatus = totalErrors.equals(0) ? "COMPLETED" : "FAILED";
|
||||
completeRunRecord(run, finalStatus, totalProcessed, totalIndexed, totalErrors, null);
|
||||
|
||||
LOG.info("Full reindex complete", "tableName", tableName, "totalProcessed", totalProcessed,
|
||||
"totalIndexed", totalIndexed, "totalErrors", totalErrors);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Full reindex failed", e, "tableName", tableName);
|
||||
completeRunRecord(run, "FAILED", totalProcessed, totalIndexed, totalErrors, e.getMessage());
|
||||
throw new QException("Full reindex failed for table [" + tableName + "]: " + e.getMessage(), e);
|
||||
LOG.warn("Could not apply deletes captured during the full reindex; they stay AWAITING_REINDEX for the next full reindex", e);
|
||||
}
|
||||
|
||||
LOG.info("Applied deletes captured during the full reindex", logPair("count", applied));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply one batch of captured deletes; returns how many were applied.
|
||||
** A record that exists again (re-created with the same key since it was
|
||||
** deleted) keeps its document, and its row is just removed. Deleting a
|
||||
** missing document is harmless. Applied rows are removed; rows the cluster
|
||||
** rejects become PENDING for basepull to retry.
|
||||
*******************************************************************************/
|
||||
private Integer applyCapturedDeleteBatch(QuickSearchOpenSearchClient client, QuickSearchQBitConfig config, String failedEventTable, List<QRecord> rows) throws QException
|
||||
{
|
||||
Map<String, List<QRecord>> rowsByTable = new LinkedHashMap<>();
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
rowsByTable.computeIfAbsent(row.getValueString("tableName"), k -> new ArrayList<>()).add(row);
|
||||
}
|
||||
|
||||
List<QRecord> finished = new ArrayList<>();
|
||||
List<QRecord> toApply = new ArrayList<>();
|
||||
for(Map.Entry<String, List<QRecord>> entry : rowsByTable.entrySet())
|
||||
{
|
||||
Set<String> existingIds = queryExistingRecordIds(entry.getKey(), entry.getValue());
|
||||
for(QRecord row : entry.getValue())
|
||||
{
|
||||
if(existingIds.contains(row.getValueString("recordId")))
|
||||
{
|
||||
finished.add(row);
|
||||
}
|
||||
else
|
||||
{
|
||||
toApply.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String error = null;
|
||||
if(!toApply.isEmpty())
|
||||
{
|
||||
List<String> documentIds = new ArrayList<>();
|
||||
for(QRecord row : toApply)
|
||||
{
|
||||
documentIds.add(OpenSearchDocument.buildDocumentId(row.getValueString("tableName"), row.getValueString("recordId")));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
|
||||
error = result.getFailureCount() == 0 ? null : String.join("; ", result.getErrors());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
error = e.getMessage() == null ? e.toString() : e.getMessage();
|
||||
}
|
||||
|
||||
if(error == null)
|
||||
{
|
||||
finished.addAll(toApply);
|
||||
}
|
||||
}
|
||||
|
||||
if(!finished.isEmpty())
|
||||
{
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(failedEventTable);
|
||||
deleteInput.setPrimaryKeys(finished.stream().map(row -> row.getValue("id")).toList());
|
||||
DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
|
||||
if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
|
||||
{
|
||||
LOG.warn("Could not remove applied captured-delete rows; they stay AWAITING_REINDEX and are applied again by the next full reindex",
|
||||
logPair("count", deleteOutput.getRecordsWithErrors().size()), logPair("firstError", deleteOutput.getRecordsWithErrors().get(0).getErrors()));
|
||||
}
|
||||
}
|
||||
|
||||
if(error != null)
|
||||
{
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
for(QRecord row : toApply)
|
||||
{
|
||||
updates.add(new QRecord()
|
||||
.withValue("id", row.getValue("id"))
|
||||
.withValue("errorMessage", truncate(error, 4000))
|
||||
.withValue("status", QuickSearchFailedEvent.STATUS_PENDING));
|
||||
}
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(failedEventTable);
|
||||
updateInput.setRecords(updates);
|
||||
List<QRecord> notUpdated = CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords()).stream()
|
||||
.filter(record -> CollectionUtils.nullSafeHasContents(record.getErrors()))
|
||||
.toList();
|
||||
LOG.warn("Could not apply deletes captured during the full reindex; basepull will retry them", logPair("count", toApply.size()), logPair("error", error));
|
||||
if(!notUpdated.isEmpty())
|
||||
{
|
||||
LOG.warn("Could not hand captured deletes to basepull; they stay AWAITING_REINDEX for the next full reindex",
|
||||
logPair("count", notUpdated.size()), logPair("firstError", notUpdated.get(0).getErrors()));
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
return (toApply.size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record IDs, among the rows' recordIds, that exist in the source table.
|
||||
*******************************************************************************/
|
||||
Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
|
||||
{
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
return (Set.of());
|
||||
}
|
||||
|
||||
String primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
|
||||
List<Serializable> recordIds = new ArrayList<>();
|
||||
rows.forEach(row -> recordIds.add(row.getValueString("recordId")));
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, recordIds)));
|
||||
|
||||
Set<String> existingIds = new HashSet<>();
|
||||
for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
existingIds.add(record.getValueString(primaryKeyField));
|
||||
}
|
||||
return (existingIds);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Rebuild one table in place (kept for callers that used this directly).
|
||||
*******************************************************************************/
|
||||
void reindexTable(String tableName, QuickSearchableTableConfig tableConfig) throws QException
|
||||
{
|
||||
QRecord row = queryIndexRow(tableName);
|
||||
if(row == null)
|
||||
{
|
||||
ensureIndexRowExists(tableName, tableConfig);
|
||||
}
|
||||
new ReconcileIndexStep().reconcileTable(tableConfig, RUN_TYPE);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -18,26 +18,38 @@ package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Static utility class for text normalization and OpenSearch document building.
|
||||
** Text normalization and OpenSearch document building.
|
||||
**
|
||||
** Provides helpers used during the indexing pipeline to normalize field values
|
||||
** into searchable text and to assemble OpenSearchDocument instances from
|
||||
** QRecord data.
|
||||
** Indexed text uses a record's display value when the query generated one
|
||||
** (possible-value labels, formatted dates) and the raw value otherwise. Every
|
||||
** value is stored as a string (fieldValues are mapped as text), truncated to
|
||||
** the table's maxFieldLength when set.
|
||||
*******************************************************************************/
|
||||
public class IndexingUtils
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(IndexingUtils.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Private constructor to prevent instantiation of this utility class.
|
||||
** Private constructor to prevent instantiation.
|
||||
*******************************************************************************/
|
||||
private IndexingUtils()
|
||||
{
|
||||
@@ -46,14 +58,7 @@ public class IndexingUtils
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Normalize text for search indexing.
|
||||
**
|
||||
** Converts to lowercase, trims leading/trailing whitespace, and collapses
|
||||
** runs of internal whitespace to a single space. Returns empty string for
|
||||
** null or blank input.
|
||||
**
|
||||
** @param text the raw text to normalize
|
||||
** @return the normalized text, never null
|
||||
** Trim and collapse whitespace. Lower-casing is left to the analyzers.
|
||||
*******************************************************************************/
|
||||
public static String normalizeSearchText(String text)
|
||||
{
|
||||
@@ -61,39 +66,39 @@ public class IndexingUtils
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
|
||||
return (text.toLowerCase().trim().replaceAll("\\s+", " "));
|
||||
return (text.trim().replaceAll("\\s+", " "));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a concatenated searchable text string from selected record fields.
|
||||
**
|
||||
** Iterates over each field name in {@code fields}, retrieves the string value
|
||||
** from the record, and appends it to the result. Fields with a null or blank
|
||||
** value are skipped. When {@code includeLabels} maps a field name to
|
||||
** {@code true}, the value is prefixed with {@code "fieldName: "}.
|
||||
**
|
||||
** @param record the QRecord to read field values from
|
||||
** @param fields ordered list of field names to include
|
||||
** @param includeLabels map of field name to whether its label should be prefixed
|
||||
** @return the concatenated text, trimmed; empty string if fields is null or empty
|
||||
** Concatenate the display values of the given fields. A field whose
|
||||
** includeLabels entry is true is prefixed with the field's label (or name).
|
||||
*******************************************************************************/
|
||||
public static String buildSearchableText(QRecord record, List<String> fields, Map<String, Boolean> includeLabels)
|
||||
{
|
||||
return (buildSearchableText(record, fields, includeLabels, null, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
static String buildSearchableText(QRecord record, List<String> fields, Map<String, Boolean> includeLabels, QTableMetaData table, Integer maxFieldLength)
|
||||
{
|
||||
if(fields == null || fields.isEmpty())
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Map<String, Boolean> labels = includeLabels == null ? Collections.emptyMap() : includeLabels;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
for(String fieldName : fields)
|
||||
{
|
||||
String value = record.getValueString(fieldName);
|
||||
|
||||
if(value == null || value.isBlank())
|
||||
String value = displayValue(record, fieldName, maxFieldLength);
|
||||
if(value == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -103,9 +108,11 @@ public class IndexingUtils
|
||||
sb.append(" ");
|
||||
}
|
||||
|
||||
if(Boolean.TRUE.equals(includeLabels.get(fieldName)))
|
||||
if(Boolean.TRUE.equals(labels.get(fieldName)))
|
||||
{
|
||||
sb.append(fieldName).append(": ");
|
||||
String label = (table != null && table.getFields().containsKey(fieldName) && StringUtils.hasContent(table.getField(fieldName).getLabel()))
|
||||
? table.getField(fieldName).getLabel() : fieldName;
|
||||
sb.append(label).append(": ");
|
||||
}
|
||||
|
||||
sb.append(value);
|
||||
@@ -117,42 +124,105 @@ public class IndexingUtils
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build an OpenSearchDocument from a QRecord.
|
||||
**
|
||||
** Delegates to {@link #buildSearchableText} for the searchable text, reads the
|
||||
** record ID from {@code primaryKeyField}, and populates {@code fieldValues} with
|
||||
** only those entries where the record's value is non-null.
|
||||
**
|
||||
** @param record the source QRecord
|
||||
** @param tableName the QQQ table name used as the document's sourceTable
|
||||
** @param primaryKeyField the field name whose value becomes the document's recordId
|
||||
** @param fields ordered list of field names for searchable text
|
||||
** @param fieldWeights map of field name to search weight (reserved for future use)
|
||||
** @param includeLabels map of field name to whether its label should be prefixed
|
||||
** @return a fully populated OpenSearchDocument with indexedAt set to now
|
||||
** Display value when present, else the raw value as a string; blank is null.
|
||||
*******************************************************************************/
|
||||
static String displayValue(QRecord record, String fieldName, Integer maxFieldLength)
|
||||
{
|
||||
String value = record.getDisplayValue(fieldName);
|
||||
if(!StringUtils.hasContent(value))
|
||||
{
|
||||
value = record.getValueString(fieldName);
|
||||
}
|
||||
if(!StringUtils.hasContent(value))
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
if(maxFieldLength != null && value.length() > maxFieldLength)
|
||||
{
|
||||
value = value.substring(0, maxFieldLength);
|
||||
}
|
||||
return (value);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a document from explicit field lists (no table config).
|
||||
*******************************************************************************/
|
||||
public static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List<String> fields, Map<String, Integer> fieldWeights, Map<String, Boolean> includeLabels)
|
||||
{
|
||||
String searchableText = buildSearchableText(record, fields, includeLabels);
|
||||
return (buildDocument(record, tableName, primaryKeyField, fields, includeLabels, lookupTable(tableName), null, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a document for a configured table: display values, label fallback,
|
||||
** truncation, and an external version from the basepull timestamp field.
|
||||
** Returns null when the record has no primary key.
|
||||
*******************************************************************************/
|
||||
public static OpenSearchDocument buildDocument(QRecord record, QuickSearchableTableConfig tableConfig)
|
||||
{
|
||||
QTableMetaData table = lookupTable(tableConfig.getTableName());
|
||||
OpenSearchDocument doc = buildDocument(record, tableConfig.getTableName(), tableConfig.getPrimaryKeyField(), tableConfig.getSearchableFields(),
|
||||
tableConfig.getFieldIncludeLabels(), table, tableConfig.getMaxFieldLength(), tableConfig.getBasepullTimestampField());
|
||||
|
||||
if(doc != null && tableConfig.getRecordLabelFormat() != null && tableConfig.getRecordLabelFields() != null)
|
||||
{
|
||||
doc.withRecordLabel(formatLabel(record, tableConfig, doc.getRecordLabel()));
|
||||
}
|
||||
|
||||
return (doc);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List<String> fields, Map<String, Boolean> includeLabels,
|
||||
QTableMetaData table, Integer maxFieldLength, String timestampField)
|
||||
{
|
||||
String recordId = record.getValueString(primaryKeyField);
|
||||
if(recordId == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
String recordLabel = record.getRecordLabel();
|
||||
|
||||
Map<String, Object> fieldValues = new HashMap<>();
|
||||
|
||||
if(fields != null)
|
||||
for(String fieldName : fields == null ? List.<String>of() : fields)
|
||||
{
|
||||
for(String fieldName : fields)
|
||||
String value = displayValue(record, fieldName, maxFieldLength);
|
||||
if(value != null)
|
||||
{
|
||||
Object value = record.getValue(fieldName);
|
||||
fieldValues.put(fieldName, value);
|
||||
}
|
||||
}
|
||||
|
||||
if(value != null)
|
||||
{
|
||||
fieldValues.put(fieldName, value);
|
||||
}
|
||||
String recordLabel = record.getRecordLabel();
|
||||
if(!StringUtils.hasContent(recordLabel) && table != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
recordLabel = QValueFormatter.formatRecordLabel(table, record);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.debug("Could not format record label", e, logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
Long version = null;
|
||||
if(timestampField != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
Instant timestamp = record.getValueInstant(timestampField);
|
||||
version = timestamp == null ? null : timestamp.toEpochMilli();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
version = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,47 +230,45 @@ public class IndexingUtils
|
||||
.withSourceTable(tableName)
|
||||
.withRecordId(recordId)
|
||||
.withRecordLabel(recordLabel)
|
||||
.withSearchableText(searchableText)
|
||||
.withSearchableText(buildSearchableText(record, fields, includeLabels, table, maxFieldLength))
|
||||
.withIndexedAt(Instant.now())
|
||||
.withFieldValues(fieldValues));
|
||||
.withFieldValues(fieldValues)
|
||||
.withVersion(version));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build an OpenSearchDocument from a QRecord using a QuickSearchableTableConfig.
|
||||
**
|
||||
** Delegates to the existing buildDocument overload for core document
|
||||
** construction. When recordLabelFormat is set on the table config, overrides
|
||||
** the record label by formatting the specified field values using
|
||||
** String.format. Falls back to QRecord.getRecordLabel() when no format is set.
|
||||
**
|
||||
** @param record the source QRecord
|
||||
** @param tableConfig the table configuration containing fields, weights, and label format
|
||||
** @return a fully populated OpenSearchDocument, or null if the primary key is null
|
||||
** Apply recordLabelFormat; a bad format or null value never breaks indexing.
|
||||
*******************************************************************************/
|
||||
public static OpenSearchDocument buildDocument(QRecord record, QuickSearchableTableConfig tableConfig)
|
||||
private static String formatLabel(QRecord record, QuickSearchableTableConfig tableConfig, String fallback)
|
||||
{
|
||||
OpenSearchDocument doc = buildDocument(
|
||||
record,
|
||||
tableConfig.getTableName(),
|
||||
tableConfig.getPrimaryKeyField(),
|
||||
tableConfig.getSearchableFields(),
|
||||
tableConfig.getFieldWeights(),
|
||||
tableConfig.getFieldIncludeLabels() != null ? tableConfig.getFieldIncludeLabels() : java.util.Collections.emptyMap());
|
||||
|
||||
if(doc != null && tableConfig.getRecordLabelFormat() != null && tableConfig.getRecordLabelFields() != null)
|
||||
try
|
||||
{
|
||||
Object[] labelValues = new Object[tableConfig.getRecordLabelFields().size()];
|
||||
for(int i = 0; i < tableConfig.getRecordLabelFields().size(); i++)
|
||||
for(int i = 0; i < labelValues.length; i++)
|
||||
{
|
||||
Object v = record.getValue(tableConfig.getRecordLabelFields().get(i));
|
||||
labelValues[i] = v != null ? v : "";
|
||||
String value = displayValue(record, tableConfig.getRecordLabelFields().get(i), null);
|
||||
labelValues[i] = value == null ? "" : value;
|
||||
}
|
||||
doc.withRecordLabel(String.format(tableConfig.getRecordLabelFormat(), labelValues));
|
||||
return (String.format(tableConfig.getRecordLabelFormat(), labelValues));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("recordLabelFormat could not be applied; using default label", e, logPair("tableName", tableConfig.getTableName()));
|
||||
return (fallback);
|
||||
}
|
||||
}
|
||||
|
||||
return (doc);
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData lookupTable(String tableName)
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
return (qInstance == null || tableName == null ? null : qInstance.getTable(tableName));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reconcile the index with its source tables without a search blackout.
|
||||
**
|
||||
** For each enabled table (or the one named by the optional "tableName"
|
||||
** input): re-index every source record in primary-key order, then, if every
|
||||
** document indexed, refresh and delete the table's documents this run did not
|
||||
** touch (indexedAt before the run started). When no table filter is given,
|
||||
** documents of tables that are no longer configured are removed too.
|
||||
**
|
||||
** Used by the full reindex of a single table as well, with its own run type.
|
||||
*******************************************************************************/
|
||||
public class ReconcileIndexStep extends AbstractIndexingStep
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(ReconcileIndexStep.class);
|
||||
|
||||
public static final String RUN_TYPE = "RECONCILE";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Totals from reconciling one table.
|
||||
*******************************************************************************/
|
||||
public record ReconcileCounts(Integer recordsIndexed, Long documentsRemoved)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
String tableNameFilter = input.getValueString("tableName");
|
||||
boolean allTables = tableNameFilter == null || tableNameFilter.isBlank();
|
||||
|
||||
Integer recordsIndexed = 0;
|
||||
Long documentsRemoved = 0L;
|
||||
Integer tablesSkipped = 0;
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
|
||||
{
|
||||
if(!allTables && !tableNameFilter.equals(tableConfig.getTableName()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ensureIndexRowExists(tableConfig.getTableName(), tableConfig);
|
||||
QRecord row = queryIndexRow(tableConfig.getTableName());
|
||||
if(!isRowEnabled(row))
|
||||
{
|
||||
LOG.info("Table is disabled; skipping reconcile", logPair("tableName", tableConfig.getTableName()));
|
||||
tablesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
ReconcileCounts counts = reconcileTable(tableConfig, RUN_TYPE);
|
||||
recordsIndexed = recordsIndexed + counts.recordsIndexed();
|
||||
documentsRemoved = documentsRemoved + counts.documentsRemoved();
|
||||
}
|
||||
|
||||
Long orphanDocumentsRemoved = 0L;
|
||||
if(allTables)
|
||||
{
|
||||
List<String> configured = new ArrayList<>();
|
||||
getDiscoveredTables().forEach(tc -> configured.add(tc.getTableName()));
|
||||
orphanDocumentsRemoved = getClient().deleteDocumentsForTablesNotIn(configured);
|
||||
if(orphanDocumentsRemoved > 0)
|
||||
{
|
||||
LOG.info("Removed documents of tables that are no longer configured", logPair("count", orphanDocumentsRemoved));
|
||||
}
|
||||
}
|
||||
|
||||
purgeOldRuns();
|
||||
|
||||
output.addValue("recordsIndexed", recordsIndexed);
|
||||
output.addValue("documentsRemoved", documentsRemoved);
|
||||
output.addValue("orphanDocumentsRemoved", orphanDocumentsRemoved);
|
||||
output.addValue("tablesSkipped", tablesSkipped);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reconcile one table, recording the run with the given run type.
|
||||
*******************************************************************************/
|
||||
ReconcileCounts reconcileTable(QuickSearchableTableConfig tableConfig, String runType) throws QException
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
|
||||
Integer indexId = queryIndexId(tableName);
|
||||
QuickSearchIndexRun run = createRunRecord(indexId, runType);
|
||||
|
||||
try
|
||||
{
|
||||
Instant reconcileStart = Instant.now();
|
||||
IndexCounts counts = indexAllRecords(client, tableConfig, List.of(), null);
|
||||
|
||||
Long documentsRemoved = 0L;
|
||||
if(counts.errors() == 0)
|
||||
{
|
||||
client.refreshIndex();
|
||||
documentsRemoved = client.deleteDocumentsIndexedBefore(tableName, reconcileStart);
|
||||
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("lastReconcileTime", Instant.now());
|
||||
values.put("lastBasepullTime", reconcileStart);
|
||||
values.put("recordCount", counts.processed());
|
||||
Integer documentCount = safeCount(client, tableName);
|
||||
if(documentCount != null)
|
||||
{
|
||||
values.put("documentCount", documentCount);
|
||||
}
|
||||
values.put("searchableFieldsJson", buildSearchableFieldsJson(tableConfig));
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// a full reindex running alongside owns the status until it ends //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
if(!STATUS_REBUILDING.equals(queryIndexRow(tableName).getValueString("status")))
|
||||
{
|
||||
values.put("status", STATUS_ACTIVE);
|
||||
}
|
||||
if(ReconcileIndexStep.RUN_TYPE.equals(runType) == false)
|
||||
{
|
||||
values.put("lastFullReindexTime", Instant.now());
|
||||
}
|
||||
updateIndexRow(indexId, values);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.warn("Reconcile had indexing errors; stale documents were not removed", logPair("tableName", tableName), logPair("errorCount", counts.errors()));
|
||||
}
|
||||
|
||||
String status = counts.errors() == 0 ? RUN_COMPLETED : RUN_FAILED;
|
||||
completeRunRecord(run, status, counts.processed(), counts.indexed(), counts.errors(), counts.firstError());
|
||||
|
||||
LOG.info("Reconcile complete", logPair("tableName", tableName), logPair("processed", counts.processed()), logPair("indexed", counts.indexed()),
|
||||
logPair("skippedStale", counts.skipped()), logPair("errors", counts.errors()), logPair("documentsRemoved", documentsRemoved));
|
||||
|
||||
if(counts.errors() > 0)
|
||||
{
|
||||
throw (new QException("Reconcile indexed " + counts.indexed() + " of " + counts.processed() + " records for table [" + tableName + "]; first error: " + counts.firstError()));
|
||||
}
|
||||
|
||||
return (new ReconcileCounts(counts.indexed(), documentsRemoved));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
if(!RUN_FAILED.equals(run.getStatus()))
|
||||
{
|
||||
completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
|
||||
run.withStatus(RUN_FAILED);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Reconcile failed", e, logPair("tableName", tableName));
|
||||
completeRunRecord(run, RUN_FAILED, 0, 0, 1, e.getMessage());
|
||||
throw new QException("Reconcile failed for table [" + tableName + "]: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Number of documents indexed for the table, or null when the count fails
|
||||
** (logged), so callers keep the previous documentCount.
|
||||
*******************************************************************************/
|
||||
static Integer safeCount(QuickSearchOpenSearchClient client, String tableName)
|
||||
{
|
||||
try
|
||||
{
|
||||
Long count = client.countDocumentsForTable(tableName);
|
||||
return (count == null ? null : count.intValue());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not count indexed documents for table; keeping the previous documentCount", e, logPair("tableName", tableName));
|
||||
return (null);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+33
-5
@@ -24,6 +24,7 @@ import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
@@ -35,13 +36,15 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** immediately against the OpenSearch client.
|
||||
**
|
||||
** Index events are grouped by table, converted to OpenSearchDocument instances
|
||||
** using IndexingUtils, and bulk-indexed. Delete events are processed
|
||||
** individually.
|
||||
** using IndexingUtils, and bulk-indexed. Delete events are bulk-deleted.
|
||||
** Either way, per-document failures are raised as a QException.
|
||||
*******************************************************************************/
|
||||
public class SynchronousIndexEventPublisher implements IndexEventPublisher
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(SynchronousIndexEventPublisher.class);
|
||||
|
||||
private static final Integer MAX_ERRORS_IN_MESSAGE = 10;
|
||||
|
||||
private final QuickSearchOpenSearchClient client;
|
||||
private final Map<String, QuickSearchableTableConfig> tableConfigsByName;
|
||||
private final Integer bulkBatchSize;
|
||||
@@ -119,14 +122,16 @@ public class SynchronousIndexEventPublisher implements IndexEventPublisher
|
||||
|
||||
if(!allDocuments.isEmpty())
|
||||
{
|
||||
client.indexDocuments(allDocuments, bulkBatchSize);
|
||||
BulkIndexResult result = client.indexDocuments(allDocuments, bulkBatchSize);
|
||||
throwIfAnyFailed(result, "index", allDocuments.size());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish delete events synchronously by calling deleteDocument for each event.
|
||||
** Publish delete events synchronously as one bulk delete, so every document
|
||||
** is attempted even when some fail. Throws if any document failed.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void publishDeleteEvents(List<IndexEvent> events) throws QException
|
||||
@@ -136,10 +141,33 @@ public class SynchronousIndexEventPublisher implements IndexEventPublisher
|
||||
return;
|
||||
}
|
||||
|
||||
List<String> documentIds = new ArrayList<>();
|
||||
for(IndexEvent event : events)
|
||||
{
|
||||
client.deleteDocument(event.getTableName(), event.getRecordId());
|
||||
documentIds.add(OpenSearchDocument.buildDocumentId(event.getTableName(), event.getRecordId()));
|
||||
}
|
||||
|
||||
BulkIndexResult result = client.deleteDocuments(documentIds, bulkBatchSize);
|
||||
throwIfAnyFailed(result, "delete", documentIds.size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Throw a QException naming the failed documents, if any bulk item failed,
|
||||
** so failures reach the caller instead of leaving the index silently stale.
|
||||
*******************************************************************************/
|
||||
private void throwIfAnyFailed(BulkIndexResult result, String operationName, Integer attemptedCount) throws QException
|
||||
{
|
||||
if(result == null || Boolean.TRUE.equals(result.isFullySuccessful()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
List<String> errors = result.getErrors();
|
||||
List<String> errorsShown = errors.subList(0, Math.min(errors.size(), MAX_ERRORS_IN_MESSAGE));
|
||||
throw (new QException("Failed to " + operationName + " " + result.getFailureCount() + " of " + attemptedCount
|
||||
+ " search documents: " + String.join("; ", errorsShown)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
version=${project.version}
|
||||
@@ -28,8 +28,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchTableMetaDataHelper;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -79,19 +78,18 @@ public class BaseQuickSearchTest
|
||||
////////////////////////////////////////////////////
|
||||
// Set up fully-anonymous authentication //
|
||||
////////////////////////////////////////////////////
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData()
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// quickSearchIndex table //
|
||||
// operational tables, as the QBit produces them //
|
||||
////////////////////////////////////////////////////
|
||||
qInstance.addTable(buildQuickSearchIndexTable());
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// quickSearchIndexRun table //
|
||||
////////////////////////////////////////////////////
|
||||
qInstance.addTable(buildQuickSearchIndexRunTable());
|
||||
QuickSearchQBitConfig tableConfig = new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME);
|
||||
qInstance.addTable(QuickSearchTableMetaDataHelper.buildIndexTable(tableConfig));
|
||||
qInstance.addTable(QuickSearchTableMetaDataHelper.buildIndexRunTable(tableConfig));
|
||||
qInstance.addTable(QuickSearchTableMetaDataHelper.buildFailedEventTable(tableConfig));
|
||||
qInstance.addPossibleValueSource(QuickSearchTableMetaDataHelper.buildIndexPossibleValueSource(tableConfig));
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// testEntity source table //
|
||||
@@ -103,56 +101,6 @@ public class BaseQuickSearchTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build QTableMetaData for quickSearchIndex.
|
||||
*******************************************************************************/
|
||||
private QTableMetaData buildQuickSearchIndexTable()
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName(QuickSearchIndex.TABLE_NAME)
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tableName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN))
|
||||
.withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("searchableFieldsJson", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("recordCount", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build QTableMetaData for quickSearchIndexRun.
|
||||
*******************************************************************************/
|
||||
private QTableMetaData buildQuickSearchIndexRunTable()
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName(QuickSearchIndexRun.TABLE_NAME)
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("runType", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("errorCount", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("errorMessage", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build QTableMetaData for the test source entity table.
|
||||
*******************************************************************************/
|
||||
|
||||
+7
-5
@@ -46,6 +46,7 @@ import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
@@ -61,7 +62,8 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
** records, runs FullReindexStep, and verifies search results including custom
|
||||
** recordLabelFormat.
|
||||
*******************************************************************************/
|
||||
@Testcontainers(disabledWithoutDocker = true)
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class ConfigDrivenIndexingIntegrationTest
|
||||
{
|
||||
@@ -71,11 +73,11 @@ class ConfigDrivenIndexingIntegrationTest
|
||||
private static final String INDEX_NAME = "config_driven_integration_test";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>("opensearchproject/opensearch:2.11.0")
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("plugins.security.disabled", "true")
|
||||
.withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "Admin123!")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
|
||||
@@ -98,7 +100,7 @@ class ConfigDrivenIndexingIntegrationTest
|
||||
.withName(BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData()
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
|
||||
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Integration test, against a real OpenSearch, that a record deleted while a
|
||||
** full reindex rebuilds the index is gone from the new physical index after
|
||||
** the alias swap, even though the delete itself only reached the old one.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
class FullReindexDeleteReplayIntegrationTest
|
||||
{
|
||||
private static final String BACKEND_NAME = "reindexDeleteMemoryBackend";
|
||||
private static final String TABLE_NAME = "reindexCustomer";
|
||||
private static final String INDEX_NAME = "quick_search_reindex_delete_test";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Entity annotated for quick search indexing.
|
||||
*******************************************************************************/
|
||||
@QuickSearchable(tableName = TABLE_NAME)
|
||||
static class ReindexCustomer
|
||||
{
|
||||
@QuickSearchField(weight = 2)
|
||||
private String name;
|
||||
|
||||
@QuickSearchField
|
||||
private String city;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the instance, produce the QBit with real-time indexing on and the
|
||||
** record-security re-read off (search returns whatever the index holds),
|
||||
** and initialize QContext.
|
||||
*******************************************************************************/
|
||||
@BeforeAll
|
||||
static void setUpAll() throws Exception
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData()
|
||||
.withName(BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("city", QFieldType.STRING)));
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost(opensearch.getHost())
|
||||
.withOpensearchPort(opensearch.getMappedPort(9200))
|
||||
.withOpensearchIndexName(INDEX_NAME)
|
||||
.withSearchableEntityClasses(List.of(ReindexCustomer.class))
|
||||
.withEnableRealTimeIndexing(true)
|
||||
.withApplyRecordSecurityLocks(false);
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clean up QContext and QBit state after all tests complete.
|
||||
*******************************************************************************/
|
||||
@AfterAll
|
||||
static void tearDownAll()
|
||||
{
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a record deleted after the rebuild read it (its document is already
|
||||
** in the new physical index) is absent from the index and from search once
|
||||
** the rebuild has swapped the alias.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeleteDuringFullReindex_isAbsentAfterTheSwap() throws Exception
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("name", "Gwendolyn").withValue("city", "Rebuildton"),
|
||||
new QRecord().withValue("id", 2).withValue("name", "Stanford").withValue("city", "Rebuildton")));
|
||||
new InsertAction().execute(insertInput);
|
||||
assertThat(searchRecordIds("rebuildton")).containsExactlyInAnyOrder("1", "2");
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep()
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// delete a record once its document is in the new physical index, //
|
||||
// while the alias still points at the old one //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex);
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
deleteInput.setPrimaryKeys(List.<Serializable>of(1));
|
||||
new DeleteAction().execute(deleteInput);
|
||||
|
||||
return (counts);
|
||||
}
|
||||
}.run(new RunBackendStepInput(), output);
|
||||
|
||||
assertThat(output.getValueBoolean("aliasSwapped")).isTrue();
|
||||
assertThat(documentRecordIds("rebuildton"))
|
||||
.as("the deleted record's document must not survive the rebuild")
|
||||
.containsExactly("2");
|
||||
assertThat(searchRecordIds("rebuildton")).containsExactly("2");
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchFailedEvent.TABLE_NAME);
|
||||
assertThat(new QueryAction().execute(queryInput).getRecords())
|
||||
.as("the captured delete was applied and its row removed")
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the record IDs search returns for the term.
|
||||
*******************************************************************************/
|
||||
private List<String> searchRecordIds(String term) throws QException
|
||||
{
|
||||
client().refreshIndex();
|
||||
return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME))
|
||||
.getResults().stream()
|
||||
.map(QuickSearchResult::getRecordId)
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the record IDs of the documents matching
|
||||
** the term, straight from OpenSearch.
|
||||
*******************************************************************************/
|
||||
private List<String> documentRecordIds(String term) throws QException
|
||||
{
|
||||
client().refreshIndex();
|
||||
return (client().search(term, List.of(TABLE_NAME), 10, 0, QuickSearchQBitContext.getDiscoveredTables())
|
||||
.hits().hits().stream()
|
||||
.map(hit -> hit.source().getRecordId())
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live OpenSearch client from the QBit context.
|
||||
*******************************************************************************/
|
||||
private QuickSearchOpenSearchClient client()
|
||||
{
|
||||
return (QuickSearchQBitContext.getClient());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.ReconcileIndexStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Integration tests, against a real OpenSearch, that real-time indexing keeps
|
||||
** the index in step with the source table through inserts (with display
|
||||
** values), updates and deletes, and
|
||||
** that the reconcile process repairs an index that has drifted.
|
||||
**
|
||||
** Each test uses its own record IDs and search terms, so tests are independent.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
class IndexDriftIntegrationTest
|
||||
{
|
||||
private static final String BACKEND_NAME = "driftMemoryBackend";
|
||||
private static final String TABLE_NAME = "driftCustomer";
|
||||
private static final String REGION_TABLE = "driftRegion";
|
||||
private static final String INDEX_NAME = "quick_search_drift_test";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Entity annotated for quick search indexing.
|
||||
*******************************************************************************/
|
||||
@QuickSearchable(tableName = TABLE_NAME)
|
||||
static class DriftCustomer
|
||||
{
|
||||
@QuickSearchField(weight = 2)
|
||||
private String name;
|
||||
|
||||
@QuickSearchField
|
||||
private String city;
|
||||
|
||||
@QuickSearchField
|
||||
private Integer regionId;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the instance, produce the QBit with real-time indexing on, and
|
||||
** initialize QContext.
|
||||
*******************************************************************************/
|
||||
@BeforeAll
|
||||
static void setUpAll() throws Exception
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData()
|
||||
.withName(BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("city", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("regionId", QFieldType.INTEGER).withPossibleValueSourceName(REGION_TABLE))
|
||||
.withRecordLabelFormat("%s - %s")
|
||||
.withRecordLabelFields("name", "regionId"));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(REGION_TABLE)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s")
|
||||
.withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(REGION_TABLE));
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost(opensearch.getHost())
|
||||
.withOpensearchPort(opensearch.getMappedPort(9200))
|
||||
.withOpensearchIndexName(INDEX_NAME)
|
||||
.withSearchableEntityClasses(List.of(DriftCustomer.class))
|
||||
.withEnableRealTimeIndexing(true);
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clean up QContext and QBit state after all tests complete.
|
||||
*******************************************************************************/
|
||||
@AfterAll
|
||||
static void tearDownAll()
|
||||
{
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a just-inserted record's document carries display values: the
|
||||
** possible-value label of a foreign key is searchable and is in the record
|
||||
** label, with no basepull in between (#28).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testInsert_documentCarriesDisplayValuesAndPossibleValueLabel() throws Exception
|
||||
{
|
||||
InsertInput regionInsert = new InsertInput();
|
||||
regionInsert.setTableName(REGION_TABLE);
|
||||
regionInsert.setRecords(List.of(new QRecord().withValue("id", 7).withValue("name", "Northwind Territory")));
|
||||
new InsertAction().execute(regionInsert);
|
||||
|
||||
insert(new QRecord().withValue("id", 31).withValue("name", "Rhoda").withValue("city", "Labelburg").withValue("regionId", 7));
|
||||
|
||||
client().refreshIndex();
|
||||
List<QuickSearchResult> results = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("northwind").withTableName(TABLE_NAME)).getResults();
|
||||
assertThat(results).as("the region's label, not its key, is indexed").extracting(QuickSearchResult::getRecordId).containsExactly("31");
|
||||
assertThat(results.get(0).getRecordLabel()).isEqualTo("Rhoda - Northwind Territory");
|
||||
|
||||
Map<String, Object> fieldValues = client().search("labelburg", List.of(TABLE_NAME), 10, 0, QuickSearchQBitContext.getDiscoveredTables())
|
||||
.hits().hits().get(0).source().getFieldValues();
|
||||
assertThat(fieldValues).containsEntry("regionId", "Northwind Territory");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a partial update (one field) keeps the other fields searchable.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPartialUpdate_unchangedFieldsStaySearchable() throws Exception
|
||||
{
|
||||
insert(new QRecord().withValue("id", 1).withValue("name", "Ada").withValue("city", "Lovelaceville"));
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("name", "Adeline")));
|
||||
new UpdateAction().execute(updateInput);
|
||||
|
||||
assertThat(searchRecordIds("lovelaceville"))
|
||||
.as("the unchanged city must stay searchable after updating only the name")
|
||||
.containsExactly("1");
|
||||
assertThat(searchRecordIds("adeline")).containsExactly("1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: deleting records removes their documents, even when the batch also
|
||||
** holds a record whose document is already missing from the index.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDelete_removesDocuments_evenWithMissingDocumentInBatch() throws Exception
|
||||
{
|
||||
insert(
|
||||
new QRecord().withValue("id", 10).withValue("name", "Tenley").withValue("city", "Deleteton"),
|
||||
new QRecord().withValue("id", 11).withValue("name", "Elvira").withValue("city", "Deleteton"),
|
||||
new QRecord().withValue("id", 12).withValue("name", "Twila").withValue("city", "Deleteton"));
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// simulate earlier drift: record 12's document is already gone //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
client().deleteDocument(TABLE_NAME, "12");
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
deleteInput.setPrimaryKeys(List.<Serializable>of(12, 10));
|
||||
new DeleteAction().execute(deleteInput);
|
||||
|
||||
assertThat(searchRecordIds("deleteton"))
|
||||
.as("only the record that was not deleted should remain")
|
||||
.containsExactly("11");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the reconcile process repairs each kind of drift from the source
|
||||
** table: an orphan document (record gone), a missing document, and a stale
|
||||
** document (record changed since it was indexed).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_repairsOrphanMissingAndStaleDocuments() throws Exception
|
||||
{
|
||||
insert(
|
||||
new QRecord().withValue("id", 21).withValue("name", "Stella").withValue("city", "Freshford"),
|
||||
new QRecord().withValue("id", 22).withValue("name", "Mona").withValue("city", "Missingham"));
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// drift the index away from the source table three ways //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
Instant anHourAgo = Instant.now().minus(1, ChronoUnit.HOURS);
|
||||
client().indexDocuments(List.of(
|
||||
new OpenSearchDocument().withSourceTable(TABLE_NAME).withRecordId("99")
|
||||
.withSearchableText("Ghostly Phantomburg").withIndexedAt(anHourAgo).withFieldValues(Map.of()),
|
||||
new OpenSearchDocument().withSourceTable(TABLE_NAME).withRecordId("21")
|
||||
.withSearchableText("Stella Oldtown").withIndexedAt(anHourAgo).withFieldValues(Map.of())), 10);
|
||||
client().deleteDocument(TABLE_NAME, "22");
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// check the drift at the index level: the search action hides the orphan, //
|
||||
// since its record cannot be read back from the source table //
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
assertThat(documentRecordIds("phantomburg")).containsExactly("99");
|
||||
assertThat(documentRecordIds("oldtown")).containsExactly("21");
|
||||
assertThat(searchRecordIds("phantomburg")).as("the orphan is never shown to a searcher").isEmpty();
|
||||
assertThat(searchRecordIds("missingham")).isEmpty();
|
||||
|
||||
RunBackendStepInput input = new RunBackendStepInput();
|
||||
input.addValue("tableName", TABLE_NAME);
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new ReconcileIndexStep().run(input, output);
|
||||
|
||||
assertThat(documentRecordIds("phantomburg")).as("orphan document removed").isEmpty();
|
||||
assertThat(documentRecordIds("oldtown")).as("stale document replaced").isEmpty();
|
||||
assertThat(searchRecordIds("freshford")).as("stale document replaced").containsExactly("21");
|
||||
assertThat(searchRecordIds("missingham")).as("missing document restored").containsExactly("22");
|
||||
assertThat(output.getValue("documentsRemoved")).isEqualTo(1L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert records through InsertAction (so real-time indexing runs).
|
||||
*******************************************************************************/
|
||||
private void insert(QRecord... records) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
insertInput.setRecords(List.of(records));
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the record IDs matching the term.
|
||||
*******************************************************************************/
|
||||
private List<String> searchRecordIds(String term) throws QException
|
||||
{
|
||||
client().refreshIndex();
|
||||
return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME))
|
||||
.getResults().stream()
|
||||
.map(QuickSearchResult::getRecordId)
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the record IDs of the documents matching
|
||||
** the term, straight from OpenSearch (no record-security post-filter).
|
||||
*******************************************************************************/
|
||||
private List<String> documentRecordIds(String term) throws QException
|
||||
{
|
||||
client().refreshIndex();
|
||||
return (client().search(term, List.of(TABLE_NAME), 10, 0, QuickSearchQBitContext.getDiscoveredTables())
|
||||
.hits().hits().stream()
|
||||
.map(hit -> hit.source().getRecordId())
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live OpenSearch client from the QBit context.
|
||||
*******************************************************************************/
|
||||
private QuickSearchOpenSearchClient client()
|
||||
{
|
||||
return (QuickSearchQBitContext.getClient());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchTransportCustomizer;
|
||||
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test transport customizer that fails linkage while the OpenSearch client is
|
||||
** built, as a missing optional library (the AWS SDK for AWS_SIGV4) does.
|
||||
*******************************************************************************/
|
||||
public class MissingLibraryTransportCustomizer implements OpenSearchTransportCustomizer
|
||||
{
|
||||
public static final String MISSING_CLASS = "software/amazon/awssdk/http/SdkHttpClient";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ApacheHttpClient5TransportBuilder customizeTransport(ApacheHttpClient5TransportBuilder builder)
|
||||
{
|
||||
throw (new NoClassDefFoundError(MISSING_CLASS));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -16,8 +16,14 @@
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
@@ -37,19 +43,25 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.Mem
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchOutput;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.ReconcileIndexStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -59,9 +71,11 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
** OpenSearch instance managed by Testcontainers.
|
||||
**
|
||||
** Tests run in order: index creation, full reindex, search with results,
|
||||
** pagination, and no-results search.
|
||||
** pagination, no-results search, then analyzer precision, mixed field
|
||||
** types, external versioning, alias swap, orphan sweep and table labels.
|
||||
*******************************************************************************/
|
||||
@Testcontainers(disabledWithoutDocker = true)
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class OpenSearchIntegrationTest
|
||||
{
|
||||
@@ -69,13 +83,15 @@ class OpenSearchIntegrationTest
|
||||
private static final String BACKEND_NAME = "testMemoryBackend";
|
||||
private static final String TABLE_NAME = "testProduct";
|
||||
private static final String INDEX_NAME = "quick_search_integration_test";
|
||||
private static final String EVENT_TABLE = "testEvent";
|
||||
private static final String TABLE_LABEL = "Test Product";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>("opensearchproject/opensearch:2.11.0")
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("plugins.security.disabled", "true")
|
||||
.withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "Admin123!")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
|
||||
@@ -115,7 +131,7 @@ class OpenSearchIntegrationTest
|
||||
.withName(BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData()
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
|
||||
@@ -124,6 +140,7 @@ class OpenSearchIntegrationTest
|
||||
////////////////////////////////////////////////////
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withLabel(TABLE_LABEL)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
@@ -132,6 +149,19 @@ class OpenSearchIntegrationTest
|
||||
.withField(new QFieldMetaData("sku", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// a config-driven table with non-text searchable fields //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(EVENT_TABLE)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("title", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("quantity", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("occurredAt", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Configure and produce the QBit //
|
||||
////////////////////////////////////////////////////
|
||||
@@ -141,6 +171,10 @@ class OpenSearchIntegrationTest
|
||||
.withOpensearchPort(opensearch.getMappedPort(9200))
|
||||
.withOpensearchIndexName(INDEX_NAME)
|
||||
.withSearchableEntityClasses(List.of(TestProduct.class))
|
||||
.withSearchableTable(new SearchableTableConfig(EVENT_TABLE, List.of(
|
||||
new SearchableFieldConfig("title").withWeight(2),
|
||||
new SearchableFieldConfig("quantity"),
|
||||
new SearchableFieldConfig("occurredAt"))))
|
||||
.withEnableRealTimeIndexing(false);
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
@@ -164,11 +198,12 @@ class OpenSearchIntegrationTest
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 1: Verify the OpenSearch client was created and the index exists.
|
||||
** Test 1: Verify the OpenSearch client was created and the alias points at
|
||||
** a physical index built with the current mapping.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(1)
|
||||
void testOpenSearchIndexCreated()
|
||||
void testOpenSearchIndexCreated() throws Exception
|
||||
{
|
||||
Object rawClient = QuickSearchQBitContext.getClient();
|
||||
assertThat(rawClient).isNotNull();
|
||||
@@ -177,6 +212,12 @@ class OpenSearchIntegrationTest
|
||||
QuickSearchQBitConfig storedConfig = QuickSearchQBitContext.getConfig();
|
||||
assertThat(storedConfig).isNotNull();
|
||||
assertThat(storedConfig.getOpensearchIndexName()).isEqualTo(INDEX_NAME);
|
||||
|
||||
Set<String> physicalIndexes = client().getPhysicalIndexes();
|
||||
assertThat(physicalIndexes).as("the configured name is an alias over one physical index").hasSize(1);
|
||||
assertThat(physicalIndexes.iterator().next()).startsWith(INDEX_NAME + "-v" + QuickSearchOpenSearchClient.MAPPING_VERSION + "-");
|
||||
assertThat(client().isMappingOutdated()).isFalse();
|
||||
assertThat(client().isHealthy()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
@@ -288,6 +329,20 @@ class OpenSearchIntegrationTest
|
||||
List<String> page2Ids = page2.getResults().stream().map(r -> r.getRecordId()).toList();
|
||||
|
||||
assertThat(page1Ids).doesNotContainAnyElementsOf(page2Ids);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// per-table mode honors offset the same way //
|
||||
////////////////////////////////////////////////////
|
||||
QuickSearchOutput perTable1 = new QuickSearchAction().execute(
|
||||
new QuickSearchInput().withSearchTerm("widget").withLimitPerTable(2).withOffset(0));
|
||||
QuickSearchOutput perTable2 = new QuickSearchAction().execute(
|
||||
new QuickSearchInput().withSearchTerm("widget").withLimitPerTable(2).withOffset(2));
|
||||
|
||||
List<String> perTable1Ids = perTable1.getResults().stream().map(r -> r.getTableName() + ":" + r.getRecordId()).toList();
|
||||
List<String> perTable2Ids = perTable2.getResults().stream().map(r -> r.getTableName() + ":" + r.getRecordId()).toList();
|
||||
|
||||
assertThat(perTable2Ids).isNotEmpty();
|
||||
assertThat(perTable1Ids).doesNotContainAnyElementsOf(perTable2Ids);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,4 +363,196 @@ class OpenSearchIntegrationTest
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 6: a term only matches documents containing a token that starts
|
||||
** with it; "stella" must not match "stone" through the shared "st" prefix.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(6)
|
||||
void testSearch_doesNotMatchOnSharedPrefixOnly() throws Exception
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("id", 6).withValue("name", "Stella").withValue("description", "Named Stella").withValue("sku", "ST-006"),
|
||||
new QRecord().withValue("id", 7).withValue("name", "Stone").withValue("description", "Named Stone").withValue("sku", "ST-007")));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
reindexTable(TABLE_NAME);
|
||||
|
||||
QuickSearchOutput output = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("stella"));
|
||||
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("6");
|
||||
assertThat(output.getTotalHits()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 7: a table with INTEGER and DATE_TIME searchable fields indexes and
|
||||
** searches without a mapping error, and the number is searchable as text.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(7)
|
||||
void testSearch_tableWithIntegerAndDateTimeSearchableFields() throws Exception
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(EVENT_TABLE);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("title", "Quarterly conference").withValue("quantity", 4242).withValue("occurredAt", Instant.parse("2026-03-01T10:00:00Z")),
|
||||
new QRecord().withValue("id", 2).withValue("title", "Board meeting").withValue("quantity", 7).withValue("occurredAt", Instant.parse("2026-04-01T10:00:00Z"))));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
reindexTable(EVENT_TABLE);
|
||||
|
||||
QuickSearchOutput byWord = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("conference").withTableName(EVENT_TABLE));
|
||||
assertThat(byWord.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1");
|
||||
|
||||
QuickSearchOutput byNumber = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("4242").withTableName(EVENT_TABLE));
|
||||
assertThat(byNumber.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1");
|
||||
|
||||
QuickSearchOutput byDate = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("2026-04").withTableName(EVENT_TABLE));
|
||||
assertThat(byDate.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("2");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 8: a document written with an older version than the one in the
|
||||
** index is skipped (not a failure) and the newer document stays.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(8)
|
||||
void testIndexDocuments_olderVersionSkipped_newerStays() throws Exception
|
||||
{
|
||||
OpenSearchDocument newer = new OpenSearchDocument().withSourceTable(TABLE_NAME).withRecordId("900")
|
||||
.withSearchableText("Versioned newer").withIndexedAt(Instant.now()).withFieldValues(Map.of()).withVersion(2000L);
|
||||
OpenSearchDocument older = new OpenSearchDocument().withSourceTable(TABLE_NAME).withRecordId("900")
|
||||
.withSearchableText("Versioned older").withIndexedAt(Instant.now()).withFieldValues(Map.of()).withVersion(1000L);
|
||||
|
||||
BulkIndexResult first = client().indexDocuments(List.of(newer), 10);
|
||||
assertThat(first.getSuccessCount()).isEqualTo(1);
|
||||
|
||||
BulkIndexResult second = client().indexDocuments(List.of(older), 10);
|
||||
assertThat(second.getSkippedCount()).isEqualTo(1);
|
||||
assertThat(second.getSuccessCount()).isEqualTo(0);
|
||||
assertThat(second.getFailureCount()).isEqualTo(0);
|
||||
assertThat(second.isFullySuccessful()).isTrue();
|
||||
|
||||
client().refreshIndex();
|
||||
SearchResponse<OpenSearchDocument> response = client().search("versioned", List.of(TABLE_NAME), 10, 0, QuickSearchQBitContext.getDiscoveredTables());
|
||||
assertThat(response.hits().hits()).hasSize(1);
|
||||
assertThat(response.hits().hits().get(0).source().getSearchableText()).isEqualTo("Versioned newer");
|
||||
|
||||
client().deleteDocument(TABLE_NAME, "900");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 9: a full reindex of all tables builds a new physical index, swaps
|
||||
** the alias to it and removes the old physical index; search keeps working.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(9)
|
||||
void testFullReindexAllTables_swapsAliasAndRemovesOldPhysicalIndex() throws Exception
|
||||
{
|
||||
Set<String> before = client().getPhysicalIndexes();
|
||||
assertThat(before).hasSize(1);
|
||||
String oldPhysicalIndex = before.iterator().next();
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
assertThat(output.getValueBoolean("aliasSwapped")).isTrue();
|
||||
|
||||
Set<String> after = client().getPhysicalIndexes();
|
||||
assertThat(after).hasSize(1);
|
||||
assertThat(after).doesNotContain(oldPhysicalIndex);
|
||||
assertThat(physicalIndexExists(oldPhysicalIndex)).as("old physical index is deleted after the swap").isFalse();
|
||||
assertThat(physicalIndexExists(after.iterator().next())).isTrue();
|
||||
|
||||
QuickSearchOutput searchOutput = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("widget"));
|
||||
assertThat(searchOutput.getTotalHits()).isGreaterThan(0L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 10: reconcile with no table filter removes documents of a table that
|
||||
** is no longer configured.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(10)
|
||||
void testReconcile_removesDocumentsOfUnconfiguredTable() throws Exception
|
||||
{
|
||||
client().indexDocuments(List.of(new OpenSearchDocument().withSourceTable("ghostTable").withRecordId("1")
|
||||
.withSearchableText("Ghostly leftover").withIndexedAt(Instant.now()).withFieldValues(Map.of())), 10);
|
||||
client().refreshIndex();
|
||||
assertThat(client().countDocumentsForTable("ghostTable")).isEqualTo(1L);
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
client().refreshIndex();
|
||||
assertThat(output.getValue("orphanDocumentsRemoved")).isEqualTo(1L);
|
||||
assertThat(client().countDocumentsForTable("ghostTable")).isEqualTo(0L);
|
||||
assertThat(client().countDocumentsForTable(TABLE_NAME)).as("configured tables keep their documents").isGreaterThan(0L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test 11: search results carry the table's label.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(11)
|
||||
void testSearchResults_haveTableLabel() throws Exception
|
||||
{
|
||||
QuickSearchOutput output = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("widget"));
|
||||
|
||||
assertThat(output.getResults()).isNotEmpty();
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getTableLabel).containsOnly(TABLE_LABEL);
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordLabel).allMatch(label -> label != null && !label.isBlank());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Rebuild one table in place and refresh so its documents are searchable.
|
||||
*******************************************************************************/
|
||||
private static void reindexTable(String tableName) throws Exception
|
||||
{
|
||||
RunBackendStepInput input = new RunBackendStepInput();
|
||||
input.addValue("tableName", tableName);
|
||||
new FullReindexStep().run(input, new RunBackendStepOutput());
|
||||
client().refreshIndex();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether a physical index exists, asked of OpenSearch over HTTP.
|
||||
*******************************************************************************/
|
||||
private static boolean physicalIndexExists(String physicalIndex) throws Exception
|
||||
{
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create("http://" + opensearch.getHost() + ":" + opensearch.getMappedPort(9200) + "/" + physicalIndex))
|
||||
.method("HEAD", HttpRequest.BodyPublishers.noBody())
|
||||
.build();
|
||||
HttpResponse<Void> response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.discarding());
|
||||
return (response.statusCode() == 200);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live OpenSearch client from the QBit context.
|
||||
*******************************************************************************/
|
||||
private static QuickSearchOpenSearchClient client()
|
||||
{
|
||||
return (QuickSearchQBitContext.getClient());
|
||||
}
|
||||
|
||||
}
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validation rules added for 1.0: URLs, auth modes, TLS, index names, field
|
||||
** existence, scheduling, secrets handling.
|
||||
*******************************************************************************/
|
||||
class QuickSearchQBitConfigValidationTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QInstance instanceWithBackendAndCustomer()
|
||||
{
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName("customer")
|
||||
.withBackendName("memory")
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
return (qInstance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QuickSearchQBitConfig baseConfig()
|
||||
{
|
||||
return (new QuickSearchQBitConfig()
|
||||
.withBackendName("memory")
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("app-search")
|
||||
.withSearchableTable("customer", List.of(new SearchableFieldConfig("name"))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<String> validate(QuickSearchQBitConfig config)
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
config.validate(instanceWithBackendAndCustomer(), errors);
|
||||
return (errors);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUrl_replacesHostPortAndScheme()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig().withOpensearchHost(null).withOpensearchPort(null).withOpensearchUrl("https://search.example.com");
|
||||
assertThat(validate(config)).isEmpty();
|
||||
assertThat(config.getEffectiveScheme()).isEqualTo("https");
|
||||
assertThat(config.getEffectiveHost()).isEqualTo("search.example.com");
|
||||
assertThat(config.getEffectivePort()).isEqualTo(443);
|
||||
|
||||
config.withOpensearchUrl("http://os.internal:9201");
|
||||
assertThat(config.getEffectivePort()).isEqualTo(9201);
|
||||
assertThat(config.getEffectiveScheme()).isEqualTo("http");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUrl_invalid_addsError()
|
||||
{
|
||||
assertThat(validate(baseConfig().withOpensearchUrl("ftp://nope"))).anyMatch(e -> e.contains("http:// or https://"));
|
||||
assertThat(validate(baseConfig().withOpensearchUrl("https://"))).anyMatch(e -> e.contains("host"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUrl_withUserInfo_isRejectedWithoutEchoingIt()
|
||||
{
|
||||
List<String> errors = validate(baseConfig().withOpensearchHost(null).withOpensearchPort(null).withOpensearchUrl("https://admin:s3cret@search.example.com"));
|
||||
assertThat(errors).anyMatch(e -> e.contains("opensearchUrl must not contain credentials"));
|
||||
assertThat(errors).noneMatch(e -> e.contains("s3cret") || e.contains("admin"));
|
||||
|
||||
assertThat(validate(baseConfig().withOpensearchUrl("https://admin@search.example.com"))).anyMatch(e -> e.contains("opensearchUrl must not contain credentials"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testAuthMode_inferredFromCredentials()
|
||||
{
|
||||
assertThat(baseConfig().getEffectiveAuthMode()).isEqualTo(QuickSearchAuthMode.NONE);
|
||||
assertThat(baseConfig().withOpensearchUsername("u").withOpensearchPassword("p").getEffectiveAuthMode()).isEqualTo(QuickSearchAuthMode.BASIC);
|
||||
assertThat(baseConfig().withAuthMode(QuickSearchAuthMode.AWS_SIGV4).getEffectiveAuthMode()).isEqualTo(QuickSearchAuthMode.AWS_SIGV4);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testBasic_overPlainHttp_isRefusedUnlessAllowed()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig().withOpensearchUsername("u").withOpensearchPassword("p");
|
||||
assertThat(validate(config)).anyMatch(e -> e.contains("plain http"));
|
||||
|
||||
assertThat(validate(config.withAllowPlaintextCredentials(true))).noneMatch(e -> e.contains("plain http"));
|
||||
assertThat(validate(baseConfig().withOpensearchUsername("u").withOpensearchPassword("p").withUseSsl(true))).noneMatch(e -> e.contains("plain http"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testBasic_explicitWithoutCredentials_addsError()
|
||||
{
|
||||
assertThat(validate(baseConfig().withAuthMode(QuickSearchAuthMode.BASIC).withUseSsl(true))).anyMatch(e -> e.contains("required for authMode BASIC"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testNone_withCredentials_addsError()
|
||||
{
|
||||
assertThat(validate(baseConfig().withAuthMode(QuickSearchAuthMode.NONE).withOpensearchUsername("u").withOpensearchPassword("p"))).anyMatch(e -> e.contains("authMode is NONE"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testSigV4_requirementsAndDefaults()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig().withAuthMode(QuickSearchAuthMode.AWS_SIGV4).withAwsRegion("us-east-1");
|
||||
List<String> errors = validate(config);
|
||||
assertThat(errors).anyMatch(e -> e.contains("requires an https connection"));
|
||||
assertThat(config.getEffectiveAwsServiceName()).isEqualTo("es");
|
||||
|
||||
errors = validate(config.withUseSsl(true).withOpensearchUsername("u").withOpensearchPassword("p"));
|
||||
assertThat(errors).anyMatch(e -> e.contains("not used with authMode AWS_SIGV4"));
|
||||
|
||||
errors = validate(baseConfig().withAuthMode(QuickSearchAuthMode.AWS_SIGV4).withUseSsl(true).withAwsRegion("us-east-1").withAwsServiceName("bogus"));
|
||||
assertThat(errors).anyMatch(e -> e.contains("es or aoss"));
|
||||
|
||||
errors = validate(baseConfig().withAuthMode(QuickSearchAuthMode.AWS_SIGV4).withUseSsl(true).withAwsRegion("us-east-1"));
|
||||
assertThat(errors).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testTls_insecureSkipVerify_onlyForLoopbackUnlessOverridden()
|
||||
{
|
||||
QuickSearchQBitConfig loopback = baseConfig().withTls(new QuickSearchTlsConfig().withInsecureSkipVerify(true));
|
||||
assertThat(validate(loopback)).isEmpty();
|
||||
|
||||
QuickSearchQBitConfig remote = baseConfig().withOpensearchHost("search.example.com").withTls(new QuickSearchTlsConfig().withInsecureSkipVerify(true));
|
||||
assertThat(validate(remote)).anyMatch(e -> e.contains("insecureSkipVerify"));
|
||||
|
||||
remote.getTls().withAllowInsecureInProduction(true);
|
||||
assertThat(validate(remote)).noneMatch(e -> e.contains("insecureSkipVerify"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testTls_caAndTrustStoreTogether_addsError()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig().withTls(new QuickSearchTlsConfig().withCaCertificatePath("/ca.pem").withTrustStorePath("/ts.p12"));
|
||||
assertThat(validate(config)).anyMatch(e -> e.contains("not both"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testIndexName_rules()
|
||||
{
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("app-search_v2.1+x")).isTrue();
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("App")).isFalse();
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("_hidden")).isFalse();
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("has space")).isFalse();
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("a/b")).isFalse();
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("..")).isFalse();
|
||||
assertThat(QuickSearchQBitConfig.isValidIndexName("a".repeat(256))).isFalse();
|
||||
|
||||
assertThat(validate(baseConfig().withOpensearchIndexName("Bad Name"))).anyMatch(e -> e.contains("opensearchIndexName must be lowercase"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testFieldExistence_checkedAgainstInstanceTables()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig().withSearchableTables(null)
|
||||
.withSearchableTable(new SearchableTableConfig("customer", List.of(new SearchableFieldConfig("name"), new SearchableFieldConfig("nope"))).withBasepullTimestampField("updatedAt"));
|
||||
List<String> errors = validate(config);
|
||||
assertThat(errors).anyMatch(e -> e.contains("[nope] does not exist on table [customer]"));
|
||||
assertThat(errors).anyMatch(e -> e.contains("basepullTimestampField [updatedAt] does not exist"));
|
||||
|
||||
QuickSearchQBitConfig ok = baseConfig().withSearchableTables(null)
|
||||
.withSearchableTable(new SearchableTableConfig("customer", List.of(new SearchableFieldConfig("name"))).withBasepullTimestampField(null));
|
||||
assertThat(validate(ok)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testTableName_withColon_addsError()
|
||||
{
|
||||
assertThat(validate(baseConfig().withSearchableTable("weird:name", List.of(new SearchableFieldConfig("x"))))).anyMatch(e -> e.contains("must not contain ':'"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testScheduling_schedulerMustExist_andCronNeedsScheduler()
|
||||
{
|
||||
QInstance qInstance = instanceWithBackendAndCustomer();
|
||||
qInstance.addScheduler(new SimpleSchedulerMetaData().withName("sched"));
|
||||
|
||||
List<String> errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("missing").validate(qInstance, errors);
|
||||
assertThat(errors).anyMatch(e -> e.contains("Scheduler not found: missing"));
|
||||
|
||||
errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").validate(qInstance, errors);
|
||||
assertThat(errors).isEmpty();
|
||||
|
||||
errors = new ArrayList<>();
|
||||
baseConfig().withReconcileCronExpression("0 0 3 * * ?").validate(qInstance, errors);
|
||||
assertThat(errors).anyMatch(e -> e.contains("requires schedulerName"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// an instance without schedulers cannot be checked; no error raised //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
assertThat(validate(baseConfig().withSchedulerName("anything"))).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testNumbers_mustBePositive()
|
||||
{
|
||||
List<String> errors = validate(baseConfig()
|
||||
.withConnectTimeoutMillis(0)
|
||||
.withResponseTimeoutMillis(-1)
|
||||
.withMaxConnections(0)
|
||||
.withMaxSearchLimit(0)
|
||||
.withBasepullRepeatSeconds(0)
|
||||
.withBasepullOverlapSeconds(-5)
|
||||
.withRunHistoryRetentionDays(0)
|
||||
.withMaxFieldLength(0));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("connectTimeoutMillis"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("responseTimeoutMillis"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("maxConnections"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("maxSearchLimit"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("basepullRepeatSeconds"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("basepullOverlapSeconds"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("runHistoryRetentionDays"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("maxFieldLength"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testValidate_nullInstance_isSafe()
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
baseConfig().validate(null, errors);
|
||||
assertThat(errors).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDeprecatedScheduledFlag_setsBothNewFlags()
|
||||
{
|
||||
@SuppressWarnings("deprecation")
|
||||
QuickSearchQBitConfig config = baseConfig().withEnableScheduledProcesses(false);
|
||||
assertThat(config.getEnableBasepullProcess()).isFalse();
|
||||
assertThat(config.getEnableMaintenanceProcesses()).isFalse();
|
||||
assertThat(config.getEnableScheduledProcesses()).isFalse();
|
||||
|
||||
config.withEnableBasepullProcess(true);
|
||||
assertThat(config.getEnableScheduledProcesses()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testToString_redactsSecrets()
|
||||
{
|
||||
String s = baseConfig().withOpensearchUsername("admin").withOpensearchPassword("hunter2").toString();
|
||||
assertThat(s).doesNotContain("hunter2").doesNotContain("admin").contains("[redacted]");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDefaults()
|
||||
{
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig();
|
||||
assertThat(config.getStartupMode()).isEqualTo(QuickSearchStartupMode.FAIL_FAST);
|
||||
assertThat(config.getEnableBasepullProcess()).isTrue();
|
||||
assertThat(config.getEnableMaintenanceProcesses()).isTrue();
|
||||
assertThat(config.getConnectTimeoutMillis()).isEqualTo(5000);
|
||||
assertThat(config.getResponseTimeoutMillis()).isEqualTo(60000);
|
||||
assertThat(config.getMaxSearchLimit()).isEqualTo(100);
|
||||
assertThat(config.getBasepullOverlapSeconds()).isEqualTo(300);
|
||||
assertThat(config.getAdminPermissionRules().getPermissionBaseName()).isEqualTo(QuickSearchQBitConfig.DEFAULT_ADMIN_PERMISSION_BASE_NAME);
|
||||
assertThat(config.getApplyRecordSecurityLocks()).isTrue();
|
||||
assertThat(config.getDefaultBackendNameForTables()).isNull();
|
||||
assertThat(config.withBackendName("memory").getDefaultBackendNameForTables()).isEqualTo("memory");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -32,9 +32,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.Mem
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostDeleteCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostInsertCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostUpdateCustomizer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -98,6 +99,15 @@ class QuickSearchQBitProducerTest
|
||||
|
||||
|
||||
|
||||
@QuickSearchable(tableName = "person")
|
||||
static class BadFieldEntity
|
||||
{
|
||||
@QuickSearchField
|
||||
private String nope;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
@@ -162,6 +172,7 @@ class QuickSearchQBitProducerTest
|
||||
assertThat(qInstance.getTable("myApp_quickSearchIndexRun")).isNotNull();
|
||||
assertThat(qInstance.getProcess("myApp_" + QuickSearchQBitProducer.BASEPULL_PROCESS_NAME)).isNotNull();
|
||||
assertThat(qInstance.getProcess("myApp_" + QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME)).isNotNull();
|
||||
assertThat(qInstance.getProcess("myApp_" + QuickSearchQBitProducer.RECONCILE_PROCESS_NAME)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
@@ -265,70 +276,31 @@ class QuickSearchQBitProducerTest
|
||||
** on source tables when enableRealTimeIndexing is true.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_registersCustomizers() throws QException
|
||||
void testProduce_registersRecordChangeListener() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withEnableRealTimeIndexing(true)).produce(qInstance);
|
||||
|
||||
QuickSearchQBitConfig config = buildValidConfig()
|
||||
.withEnableRealTimeIndexing(true);
|
||||
|
||||
QuickSearchQBitProducer producer = new QuickSearchQBitProducer()
|
||||
.withConfig(config);
|
||||
|
||||
producer.produce(qInstance);
|
||||
|
||||
QTableMetaData personTable = qInstance.getTable(SOURCE_TABLE);
|
||||
assertThat(personTable).isNotNull();
|
||||
|
||||
Map<String, com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference> customizers =
|
||||
personTable.getCustomizers();
|
||||
assertThat(customizers).isNotNull();
|
||||
assertThat(customizers).containsKey("postInsertRecord");
|
||||
assertThat(customizers).containsKey("postUpdateRecord");
|
||||
assertThat(customizers).containsKey("postDeleteRecord");
|
||||
|
||||
assertThat(customizers.get("postInsertRecord").getName())
|
||||
.isEqualTo(QuickSearchPostInsertCustomizer.class.getName());
|
||||
assertThat(customizers.get("postUpdateRecord").getName())
|
||||
.isEqualTo(QuickSearchPostUpdateCustomizer.class.getName());
|
||||
assertThat(customizers.get("postDeleteRecord").getName())
|
||||
.isEqualTo(QuickSearchPostDeleteCustomizer.class.getName());
|
||||
assertThat(qInstance.getRecordChangeListeners()).extracting(QCodeReference::getName).contains(QuickSearchRecordChangeListener.class.getName());
|
||||
assertThat(qInstance.getTable(SOURCE_TABLE).getCustomizers() == null || qInstance.getTable(SOURCE_TABLE).getCustomizers().isEmpty()).isTrue();
|
||||
assertThat(qInstance.getRuntimeServices()).extracting(QCodeReference::getName).contains(QuickSearchRuntimeService.class.getName());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Test that no customizers are registered when enableRealTimeIndexing
|
||||
** is false.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_realTimeIndexingDisabled_noCustomizers() throws QException
|
||||
void testProduce_realTimeIndexingDisabled_noListener() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withEnableRealTimeIndexing(false)).produce(qInstance);
|
||||
|
||||
QuickSearchQBitConfig config = buildValidConfig()
|
||||
.withEnableRealTimeIndexing(false);
|
||||
|
||||
QuickSearchQBitProducer producer = new QuickSearchQBitProducer()
|
||||
.withConfig(config);
|
||||
|
||||
producer.produce(qInstance);
|
||||
|
||||
QTableMetaData personTable = qInstance.getTable(SOURCE_TABLE);
|
||||
assertThat(personTable).isNotNull();
|
||||
|
||||
Map<String, com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference> customizers =
|
||||
personTable.getCustomizers();
|
||||
assertThat(customizers == null || customizers.isEmpty()).isTrue();
|
||||
assertThat(qInstance.getRecordChangeListeners() == null || qInstance.getRecordChangeListeners().isEmpty()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Test that produce creates processes in the QInstance.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_createsProcesses() throws QException
|
||||
{
|
||||
@@ -342,6 +314,7 @@ class QuickSearchQBitProducerTest
|
||||
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME)).isNotNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME)).isNotNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.RECONCILE_PROCESS_NAME)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
@@ -486,6 +459,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -512,39 +486,129 @@ class QuickSearchQBitProducerTest
|
||||
** Test that config-driven tables register customizers on source tables.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_configDriven_registersCustomizers() throws QException
|
||||
void testProduce_registersQBitMetaDataWithRealVersion() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
QuickSearchQBitConfig config = buildValidConfig();
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("test-config-driven")
|
||||
.withEnableRealTimeIndexing(true)
|
||||
.withSearchableTable(SOURCE_TABLE, List.of(new SearchableFieldConfig("firstName")));
|
||||
assertThat(qInstance.getQBits()).hasSize(1);
|
||||
var qBit = qInstance.getQBits().values().iterator().next();
|
||||
assertThat(qBit.getGroupId()).isEqualTo("com.kingsrook.qbits");
|
||||
assertThat(qBit.getArtifactId()).isEqualTo("quick-search");
|
||||
assertThat(qBit.getVersion()).isNotEqualTo("unknown").doesNotContain("${");
|
||||
assertThat(qBit.getConfig()).isSameAs(config);
|
||||
assertThat(config.getRuntime()).isNotNull();
|
||||
assertThat(QuickSearchRuntime.get()).isSameAs(config.getRuntime());
|
||||
|
||||
QuickSearchQBitProducer producer = new QuickSearchQBitProducer()
|
||||
.withConfig(config);
|
||||
assertThat(qInstance.getTable("quickSearchIndex").getSourceQBitName()).isEqualTo(qBit.getName());
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME).getSourceQBitName()).isEqualTo(qBit.getName());
|
||||
assertThat(qInstance.getTable("quickSearchFailedEvent")).isNotNull();
|
||||
}
|
||||
|
||||
producer.produce(qInstance);
|
||||
|
||||
QTableMetaData personTable = qInstance.getTable(SOURCE_TABLE);
|
||||
Map<String, com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference> customizers =
|
||||
personTable.getCustomizers();
|
||||
|
||||
assertThat(customizers).containsKey("postInsertRecord");
|
||||
assertThat(customizers).containsKey("postUpdateRecord");
|
||||
assertThat(customizers).containsKey("postDeleteRecord");
|
||||
@Test
|
||||
void testProduce_permissionRulesAndUniqueKey() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig()).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME).getPermissionRules().getLevel()).isEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME).getPermissionRules().getPermissionBaseName()).isEqualTo("quickSearchAdmin");
|
||||
assertThat(qInstance.getApp(QuickSearchQBitProducer.APP_NAME).getPermissionRules().getLevel()).isEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
|
||||
assertThat(qInstance.getTable("quickSearchIndex").getPermissionRules().getLevel()).isEqualTo(PermissionLevel.READ_WRITE_PERMISSIONS);
|
||||
assertThat(qInstance.getTable("quickSearchIndex").getUniqueKeys()).anyMatch(uk -> uk.getFieldNames().equals(List.of("tableName")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_schedulesWhenSchedulerConfigured() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
qInstance.addScheduler(new SimpleSchedulerMetaData().withName("sched"));
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withSchedulerName("sched").withBasepullRepeatSeconds(120).withReconcileCronExpression("0 0 3 * * ?")).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME).getSchedule().getSchedulerName()).isEqualTo("sched");
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME).getSchedule().getRepeatSeconds()).isEqualTo(120);
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.RECONCILE_PROCESS_NAME).getSchedule().getCronExpression()).isEqualTo("0 0 3 * * ?");
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME).getSchedule()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_noScheduler_processesUnscheduled() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig()).produce(qInstance);
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME).getSchedule()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_flagsControlProcessRegistration() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withEnableBasepullProcess(false)).produce(qInstance);
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME)).isNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME)).isNotNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.RECONCILE_PROCESS_NAME)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_failFast_unreachableCluster_throws()
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
QuickSearchQBitConfig config = buildValidConfig().withStartupMode(QuickSearchStartupMode.FAIL_FAST).withOpensearchHost("127.0.0.1").withOpensearchPort(1);
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config).produce(qInstance))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("startupMode=DEGRADED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_annotationFieldMissing_throws()
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
QuickSearchQBitConfig config = buildValidConfig().withSearchableEntityClasses(List.of(BadFieldEntity.class));
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config).produce(qInstance))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("[nope]");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testIsEnabled_falseWithoutConfig()
|
||||
{
|
||||
assertThat(new QuickSearchQBitProducer().isEnabled()).isFalse();
|
||||
assertThat(new QuickSearchQBitProducer().withConfig(buildValidConfig()).isEnabled()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_withoutConfig_throws()
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().produce(qInstance)).isInstanceOf(QException.class).hasMessageContaining("no config");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Test mixed mode: both annotation-based and config-driven tables
|
||||
** are discovered and merged.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_mixedMode_bothSourcesMerged() throws QException
|
||||
{
|
||||
@@ -606,6 +670,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -637,6 +702,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -668,6 +734,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -696,6 +763,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -725,6 +793,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -756,6 +825,7 @@ class QuickSearchQBitProducerTest
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
@@ -794,7 +864,7 @@ class QuickSearchQBitProducerTest
|
||||
.withName(BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData()
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
|
||||
@@ -836,6 +906,7 @@ class QuickSearchQBitProducerTest
|
||||
private QuickSearchQBitConfig buildValidConfig()
|
||||
{
|
||||
return new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for QuickSearchRuntime
|
||||
*******************************************************************************/
|
||||
class QuickSearchRuntimeTest extends BaseQuickSearchTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Register a QBit with a runtime on the current QInstance.
|
||||
*******************************************************************************/
|
||||
private QuickSearchRuntime registerRuntime(QuickSearchQBitConfig config)
|
||||
{
|
||||
QuickSearchRuntime runtime = new QuickSearchRuntime(config);
|
||||
runtime.setDiscoveredTables(List.of(new QuickSearchableTableConfig().withTableName(TEST_ENTITY_TABLE).withPrimaryKeyField("id").withSearchableFields(List.of("name"))));
|
||||
config.setRuntime(runtime);
|
||||
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
qInstance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("quick-search").withVersion("test").withConfig(config));
|
||||
return (runtime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testGet_resolvesThroughQInstanceQBits()
|
||||
{
|
||||
QuickSearchQBitContext.clear();
|
||||
assertThat(QuickSearchRuntime.get()).isNull();
|
||||
|
||||
QuickSearchRuntime runtime = registerRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("localhost").withOpensearchPort(9200).withOpensearchIndexName("t"));
|
||||
|
||||
assertThat(QuickSearchRuntime.get()).isSameAs(runtime);
|
||||
assertThat(QuickSearchQBitContext.getConfig()).isSameAs(runtime.getConfig());
|
||||
assertThat(QuickSearchQBitContext.getTableConfig(TEST_ENTITY_TABLE)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testGetOrThrow_withoutRuntime_throws()
|
||||
{
|
||||
QuickSearchQBitContext.clear();
|
||||
assertThatThrownBy(QuickSearchRuntime::getOrThrow).isInstanceOf(QException.class).hasMessageContaining("not registered");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testGetClient_isLazyAndDoesNotContactCluster() throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = registerRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("127.0.0.1").withOpensearchPort(1).withOpensearchIndexName("t"));
|
||||
assertThat(runtime.isIndexReady()).isFalse();
|
||||
assertThat(runtime.getClient()).isNotNull().isSameAs(runtime.getClient());
|
||||
assertThat(runtime.isIndexReady()).isFalse();
|
||||
runtime.close();
|
||||
runtime.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testGetPublisher_prefersHostSuppliedPublisher() throws QException
|
||||
{
|
||||
IndexEventPublisher custom = mock(IndexEventPublisher.class);
|
||||
QuickSearchRuntime runtime = registerRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("127.0.0.1").withOpensearchPort(1).withOpensearchIndexName("t").withIndexEventPublisher(custom));
|
||||
assertThat(runtime.getPublisher()).isSameAs(custom);
|
||||
|
||||
QuickSearchRuntime defaultRuntime = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("127.0.0.1").withOpensearchPort(1).withOpensearchIndexName("t"));
|
||||
assertThat(defaultRuntime.getPublisher()).isInstanceOf(SynchronousIndexEventPublisher.class);
|
||||
defaultRuntime.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testIsTableEnabled_readsRowAndCaches() throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = registerRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("localhost").withOpensearchPort(9200).withOpensearchIndexName("t"));
|
||||
|
||||
assertThat(runtime.isTableEnabled("notConfigured")).isFalse();
|
||||
assertThat(runtime.isTableEnabled(TEST_ENTITY_TABLE)).isTrue();
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("tableName", TEST_ENTITY_TABLE).withValue("enabled", false)));
|
||||
Integer id = new InsertAction().execute(insertInput).getRecords().get(0).getValueInteger("id");
|
||||
|
||||
//////////////////////////////////////////////////////////////
|
||||
// cached value survives the row change until invalidated //
|
||||
//////////////////////////////////////////////////////////////
|
||||
assertThat(runtime.isTableEnabled(TEST_ENTITY_TABLE)).isTrue();
|
||||
runtime.invalidateEnabledCache();
|
||||
assertThat(runtime.isTableEnabled(TEST_ENTITY_TABLE)).isFalse();
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", id).withValue("enabled", true)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
runtime.invalidateEnabledCache();
|
||||
assertThat(runtime.isTableEnabled(TEST_ENTITY_TABLE)).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testStart_degradedSwallowsConnectionFailure_failFastThrows()
|
||||
{
|
||||
QuickSearchRuntime degraded = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("127.0.0.1").withOpensearchPort(1).withOpensearchIndexName("t").withStartupMode(QuickSearchStartupMode.DEGRADED).withConnectTimeoutMillis(200).withResponseTimeoutMillis(500));
|
||||
assertThatCode(degraded::start).doesNotThrowAnyException();
|
||||
assertThat(degraded.isIndexReady()).isFalse();
|
||||
degraded.close();
|
||||
|
||||
QuickSearchRuntime failFast = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("127.0.0.1").withOpensearchPort(1).withOpensearchIndexName("t").withConnectTimeoutMillis(200).withResponseTimeoutMillis(500));
|
||||
assertThatThrownBy(failFast::start).isInstanceOf(QException.class).hasMessageContaining("startupMode=DEGRADED");
|
||||
failFast.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testStart_missingOptionalLibrary_degradedBoots_failFastThrows()
|
||||
{
|
||||
QCodeReference customizer = new QCodeReference(MissingLibraryTransportCustomizer.class);
|
||||
|
||||
QuickSearchRuntime degraded = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("localhost").withOpensearchPort(9200).withOpensearchIndexName("t").withStartupMode(QuickSearchStartupMode.DEGRADED).withTransportCustomizer(customizer));
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchRuntime.class);
|
||||
try
|
||||
{
|
||||
assertThatCode(degraded::start).doesNotThrowAnyException();
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(QuickSearchRuntime.class);
|
||||
}
|
||||
assertThat(degraded.isIndexReady()).isFalse();
|
||||
assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains(MissingLibraryTransportCustomizer.MISSING_CLASS).contains("AWS SDK v2").contains("apache-client, sts").doesNotContain("not reachable");
|
||||
});
|
||||
degraded.close();
|
||||
|
||||
QuickSearchRuntime failFast = new QuickSearchRuntime(new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withOpensearchHost("localhost").withOpensearchPort(9200).withOpensearchIndexName("t").withTransportCustomizer(customizer));
|
||||
assertThatThrownBy(failFast::start).isInstanceOf(QException.class)
|
||||
.hasMessageContaining(MissingLibraryTransportCustomizer.MISSING_CLASS)
|
||||
.hasMessageContaining("AWS SDK v2 dependencies (apache-client, sts)")
|
||||
.hasMessageNotContaining("startupMode=DEGRADED")
|
||||
.hasCauseInstanceOf(NoClassDefFoundError.class);
|
||||
failFast.close();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
|
||||
import org.junit.jupiter.api.extension.ExecutionCondition;
|
||||
import org.junit.jupiter.api.extension.ExtensionContext;
|
||||
import org.testcontainers.DockerClientFactory;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** JUnit condition for integration tests that need Docker.
|
||||
**
|
||||
** Without Docker, the tests are skipped on a developer machine, but fail when
|
||||
** CI=true, so a CI build can never pass with its integration tests silently
|
||||
** skipped. Register it ahead of @Testcontainers, so no container is started
|
||||
** when the tests are disabled.
|
||||
*******************************************************************************/
|
||||
public class RequiresDockerCondition implements ExecutionCondition
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Evaluate against the local Docker environment and the CI env var.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context)
|
||||
{
|
||||
return (evaluate(DockerClientFactory.instance().isDockerAvailable(), System.getenv("CI")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Decide whether Docker-backed tests run, skip, or fail.
|
||||
**
|
||||
** @param dockerAvailable whether a usable Docker environment was found
|
||||
** @param ciValue value of the CI environment variable (may be null)
|
||||
** @return enabled when Docker is available; disabled when it is not and CI
|
||||
** is not "true"
|
||||
** @throws IllegalStateException when Docker is unavailable and CI is "true"
|
||||
*******************************************************************************/
|
||||
static ConditionEvaluationResult evaluate(Boolean dockerAvailable, String ciValue)
|
||||
{
|
||||
if(Boolean.TRUE.equals(dockerAvailable))
|
||||
{
|
||||
return (ConditionEvaluationResult.enabled("Docker is available"));
|
||||
}
|
||||
|
||||
if("true".equalsIgnoreCase(ciValue))
|
||||
{
|
||||
throw (new IllegalStateException("Docker is required for integration tests when CI=true, but no usable Docker environment was found"));
|
||||
}
|
||||
|
||||
return (ConditionEvaluationResult.disabled("Docker is not available; skipping (with CI=true these tests fail instead)"));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests for RequiresDockerCondition.
|
||||
*******************************************************************************/
|
||||
class RequiresDockerConditionTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Docker available: integration tests run, whether or not CI is set.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testEvaluate_dockerAvailable_enabled()
|
||||
{
|
||||
ConditionEvaluationResult local = RequiresDockerCondition.evaluate(true, null);
|
||||
ConditionEvaluationResult ci = RequiresDockerCondition.evaluate(true, "true");
|
||||
|
||||
assertThat(local.isDisabled()).isFalse();
|
||||
assertThat(ci.isDisabled()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No Docker outside CI: integration tests are skipped.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testEvaluate_noDockerOutsideCi_disabled()
|
||||
{
|
||||
assertThat(RequiresDockerCondition.evaluate(false, null).isDisabled()).isTrue();
|
||||
assertThat(RequiresDockerCondition.evaluate(false, "").isDisabled()).isTrue();
|
||||
assertThat(RequiresDockerCondition.evaluate(false, "false").isDisabled()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No Docker with CI=true: evaluation throws, so the tests fail instead of
|
||||
** silently skipping.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testEvaluate_noDockerInCi_throws()
|
||||
{
|
||||
assertThatThrownBy(() -> RequiresDockerCondition.evaluate(false, "true"))
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("CI=true");
|
||||
|
||||
assertThatThrownBy(() -> RequiresDockerCondition.evaluate(false, "TRUE"))
|
||||
.isInstanceOf(IllegalStateException.class);
|
||||
}
|
||||
|
||||
}
|
||||
+678
@@ -0,0 +1,678 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.actions;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.IntStream;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.opensearch.client.opensearch.core.search.Hit;
|
||||
import org.opensearch.client.opensearch.core.search.HitsMetadata;
|
||||
import org.opensearch.client.opensearch.core.search.TotalHits;
|
||||
import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.clearInvocations;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests for the permission-aware parts of QuickSearchAction: table READ
|
||||
** permission, the record-security post-filter through QueryAction, the
|
||||
** access-safe totals it reports, and tableLabel from the QInstance.
|
||||
**
|
||||
** Uses the memory-backed QInstance, QSession and mock client from
|
||||
** BaseQuickSearchTest; record security locks are on (the default).
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
class QuickSearchActionSecurityTest extends BaseQuickSearchTest
|
||||
{
|
||||
private static final String TABLE_LABEL = "Test Entity";
|
||||
private static final String OTHER_TABLE = "otherEntity";
|
||||
|
||||
private QuickSearchOpenSearchClient mockClient;
|
||||
private QuickSearchAction action;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
mockClient = QuickSearchQBitContext.getClient();
|
||||
action = new QuickSearchAction();
|
||||
QContext.getQInstance().getTable(TEST_ENTITY_TABLE).withLabel(TABLE_LABEL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a mocked SearchResponse with the given hits and totalHits count.
|
||||
*******************************************************************************/
|
||||
private SearchResponse<OpenSearchDocument> buildMockResponse(List<Hit<OpenSearchDocument>> hits, long totalHits)
|
||||
{
|
||||
TotalHits mockTotal = mock(TotalHits.class);
|
||||
when(mockTotal.value()).thenReturn(totalHits);
|
||||
|
||||
HitsMetadata<OpenSearchDocument> mockHitsMeta = mock(HitsMetadata.class);
|
||||
when(mockHitsMeta.total()).thenReturn(mockTotal);
|
||||
when(mockHitsMeta.hits()).thenReturn(hits);
|
||||
|
||||
SearchResponse<OpenSearchDocument> mockResponse = mock(SearchResponse.class);
|
||||
when(mockResponse.hits()).thenReturn(mockHitsMeta);
|
||||
return (mockResponse);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stub the client search with a response built before the stubbing call,
|
||||
** so no mock is configured while another stubbing is in progress.
|
||||
*******************************************************************************/
|
||||
private void stubSearch(List<Hit<OpenSearchDocument>> hits, long totalHits) throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(hits, totalHits);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a mocked hit for a testEntity record.
|
||||
*******************************************************************************/
|
||||
private Hit<OpenSearchDocument> hitFor(String recordId)
|
||||
{
|
||||
return (hitFor(TEST_ENTITY_TABLE, recordId));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a mocked hit for a record of the given table.
|
||||
*******************************************************************************/
|
||||
private Hit<OpenSearchDocument> hitFor(String tableName, String recordId)
|
||||
{
|
||||
Hit<OpenSearchDocument> hit = mock(Hit.class);
|
||||
when(hit.source()).thenReturn(new OpenSearchDocument().withSourceTable(tableName).withRecordId(recordId).withRecordLabel("Entity " + recordId));
|
||||
when(hit.score()).thenReturn(1.0);
|
||||
when(hit.highlight()).thenReturn(Map.of());
|
||||
return (hit);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert testEntity records with the given ids.
|
||||
*******************************************************************************/
|
||||
private void insertEntities(Integer... ids) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TEST_ENTITY_TABLE);
|
||||
insertInput.setRecords(List.of(ids).stream().map(id -> new QRecord().withValue("id", id).withValue("name", "Entity " + id)).toList());
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a hit whose record the query cannot return is dropped, the fetch
|
||||
** over-reads by 2x, and the total reflects only verified-accessible hits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLockPostFilter_dropsHitForRecordTheQueryCannotReturn() throws QException
|
||||
{
|
||||
insertEntities(1);
|
||||
stubSearch(List.of(hitFor("1"), hitFor("2")), 2L);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity"));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of(TEST_ENTITY_TABLE)), eq(QuickSearchAction.DEFAULT_LIMIT * 2), eq(0), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1");
|
||||
assertThat(output.getTotalHits()).isEqualTo(1L);
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isFalse();
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: accessible hits beyond the limit are truncated; every hit was read,
|
||||
** so the total is exact.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_moreAccessibleHitsThanLimit_truncatedWithLowerBoundTotal() throws QException
|
||||
{
|
||||
insertEntities(1, 2, 3);
|
||||
stubSearch(List.of(hitFor("1"), hitFor("2"), hitFor("3")), 3L);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withLimit(2));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(4), eq(0), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1", "2");
|
||||
assertThat(output.getTotalHits()).isEqualTo(3L);
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isFalse();
|
||||
assertThat(output.getHasMore()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with locks disabled the raw OpenSearch total is reported and hits
|
||||
** are not re-read from the table.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLocksDisabled_rawTotalsAndNoPostFilter() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withApplyRecordSecurityLocks(false);
|
||||
stubSearch(List.of(hitFor("1"), hitFor("2")), 40L);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withLimit(10));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(10), eq(0), anyList(), any());
|
||||
assertThat(output.getResults()).hasSize(2);
|
||||
assertThat(output.getTotalHits()).isEqualTo(40L);
|
||||
assertThat(output.getHasMore()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a session without READ permission on a protected table gets no
|
||||
** hits from it, and the index is not even queried.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSessionWithoutReadPermission_tableNotSearched() throws QException
|
||||
{
|
||||
QContext.getQInstance().getTable(TEST_ENTITY_TABLE).withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS));
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity"));
|
||||
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
assertThat(output.getTotalHits()).isEqualTo(0L);
|
||||
verify(mockClient, never()).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the same protected table is searched once the session holds the
|
||||
** READ permission.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSessionWithReadPermission_tableSearched() throws QException
|
||||
{
|
||||
QContext.getQInstance().getTable(TEST_ENTITY_TABLE).withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS));
|
||||
QContext.getQSession().withPermission(PermissionsHelper.getTablePermissionName(TEST_ENTITY_TABLE, TablePermissionSubType.READ));
|
||||
|
||||
insertEntities(1);
|
||||
stubSearch(List.of(hitFor("1")), 1L);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity"));
|
||||
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: tableLabel comes from the QInstance table label.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableLabel_populatedFromQInstance() throws QException
|
||||
{
|
||||
insertEntities(1);
|
||||
stubSearch(List.of(hitFor("1")), 1L);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity"));
|
||||
|
||||
assertThat(output.getResults().get(0).getTableName()).isEqualTo(TEST_ENTITY_TABLE);
|
||||
assertThat(output.getResults().get(0).getTableLabel()).isEqualTo(TABLE_LABEL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: limitPerTable with locks on reports the filtered count per table;
|
||||
** with every candidate read, the count is exact.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_withLocks_totalIsFilteredCount() throws QException
|
||||
{
|
||||
insertEntities(1);
|
||||
stubSearch(List.of(hitFor("1"), hitFor("2")), 2L);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withLimitPerTable(5));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of(TEST_ENTITY_TABLE)), eq(10), eq(0), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1");
|
||||
assertThat(output.getTotalHits()).isEqualTo(1L);
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isFalse();
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stub the client search over an ordered list of testEntity record ids:
|
||||
** each call returns the slice at its from/size, as OpenSearch would.
|
||||
** Fails the call if from + size passes the result window.
|
||||
*******************************************************************************/
|
||||
private void stubPagedSearch(List<String> recordIds) throws QException
|
||||
{
|
||||
stubPagedSearch(Map.of(TEST_ENTITY_TABLE, recordIds), false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Paged stub over per-table id lists (a request naming several tables
|
||||
** reads the test entity list); totalIsFloor reports the total as Gte.
|
||||
*******************************************************************************/
|
||||
private void stubPagedSearch(Map<String, List<String>> idsByTable, boolean totalIsFloor) throws QException
|
||||
{
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenAnswer(invocation ->
|
||||
{
|
||||
List<String> tables = new ArrayList<>(invocation.<Collection<String>>getArgument(1));
|
||||
String table = tables.size() == 1 ? tables.get(0) : TEST_ENTITY_TABLE;
|
||||
List<String> ids = idsByTable.getOrDefault(table, List.of());
|
||||
int size = invocation.getArgument(2);
|
||||
int from = invocation.getArgument(3);
|
||||
assertThat(from + size).isLessThanOrEqualTo(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW);
|
||||
|
||||
List<Hit<OpenSearchDocument>> hits = new ArrayList<>();
|
||||
for(int i = from; i < Math.min(ids.size(), from + size); i++)
|
||||
{
|
||||
hits.add(hitFor(table, ids.get(i)));
|
||||
}
|
||||
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(hits, ids.size());
|
||||
if(totalIsFloor)
|
||||
{
|
||||
when(response.hits().total().relation()).thenReturn(TotalHitsRelation.Gte);
|
||||
}
|
||||
return (response);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run one search page and return its output.
|
||||
*******************************************************************************/
|
||||
private QuickSearchOutput page(Integer offset, Integer limit) throws QException
|
||||
{
|
||||
return (action.execute(new QuickSearchInput().withSearchTerm("entity").withOffset(offset).withLimit(limit)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with locks on, offset counts accessible results, so consecutive
|
||||
** pages neither overlap nor skip an accessible hit when the post-filter
|
||||
** removes hits, and hasMore is false only on the last page.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_pagesByAccessibleResults_noOverlapNoSkip() throws QException
|
||||
{
|
||||
insertEntities(1, 3, 5, 7, 9);
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 10).mapToObj(String::valueOf).toList());
|
||||
|
||||
QuickSearchOutput page1 = page(0, 2);
|
||||
QuickSearchOutput page2 = page(2, 2);
|
||||
QuickSearchOutput page3 = page(4, 2);
|
||||
|
||||
assertThat(page1.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1", "3");
|
||||
assertThat(page2.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("5", "7");
|
||||
assertThat(page3.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("9");
|
||||
assertThat(page1.getHasMore()).isTrue();
|
||||
assertThat(page2.getHasMore()).isTrue();
|
||||
assertThat(page3.getHasMore()).isFalse();
|
||||
assertThat(page3.getTotalHits()).isEqualTo(5L);
|
||||
assertThat(page3.getTotalHitsIsLowerBound()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: an offset past the last accessible result returns an empty page
|
||||
** with hasMore false.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_offsetPastAccessibleResults_emptyPage() throws QException
|
||||
{
|
||||
insertEntities(1, 3);
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 10).mapToObj(String::valueOf).toList());
|
||||
|
||||
QuickSearchOutput output = page(5, 2);
|
||||
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
assertThat(output.getTotalHits()).isEqualTo(2L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: per-table mode skips offset accessible results in each table.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_withLocks_honorsOffsetPerTable() throws QException
|
||||
{
|
||||
insertEntities(1, 3, 5, 7, 9);
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 10).mapToObj(String::valueOf).toList());
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withOffset(2).withLimitPerTable(2));
|
||||
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("5", "7");
|
||||
assertThat(output.getHasMore()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: per-table mode without locks passes offset straight to OpenSearch.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_locksDisabled_passesOffset() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withApplyRecordSecurityLocks(false);
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 10).mapToObj(String::valueOf).toList());
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withOffset(8).withLimitPerTable(5));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of(TEST_ENTITY_TABLE)), eq(5), eq(8), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("9", "10");
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with a maxSearchLimit above half the result window, no request
|
||||
** asks OpenSearch for more than the window (from + size at most 10,000),
|
||||
** and a locked batch is at most MAX_LOCKED_BATCH_SIZE.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLargeMaxSearchLimit_requestsCappedAtResultWindow() throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = QuickSearchQBitContext.getConfig().withMaxSearchLimit(8_000);
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 20).mapToObj(String::valueOf).toList());
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("entity").withLimitPerTable(8_000));
|
||||
action.execute(new QuickSearchInput().withSearchTerm("entity").withLimit(8_000));
|
||||
config.withApplyRecordSecurityLocks(false);
|
||||
action.execute(new QuickSearchInput().withSearchTerm("entity").withLimitPerTable(8_000).withOffset(5_000));
|
||||
|
||||
ArgumentCaptor<Integer> sizes = ArgumentCaptor.forClass(Integer.class);
|
||||
ArgumentCaptor<Integer> offsets = ArgumentCaptor.forClass(Integer.class);
|
||||
verify(mockClient, times(3)).search(anyString(), any(), sizes.capture(), offsets.capture(), anyList(), any());
|
||||
assertThat(sizes.getAllValues()).containsExactly(QuickSearchAction.MAX_LOCKED_BATCH_SIZE, QuickSearchAction.MAX_LOCKED_BATCH_SIZE, 5_000);
|
||||
assertThat(offsets.getAllValues()).containsExactly(0, 0, 5_000);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Assert the searches since the last clear stayed within one request's
|
||||
** budget and read up to, but not past, the result window.
|
||||
*******************************************************************************/
|
||||
private void assertScanWithinBudget() throws QException
|
||||
{
|
||||
ArgumentCaptor<Integer> sizes = ArgumentCaptor.forClass(Integer.class);
|
||||
ArgumentCaptor<Integer> offsets = ArgumentCaptor.forClass(Integer.class);
|
||||
verify(mockClient, atLeastOnce()).search(anyString(), any(), sizes.capture(), offsets.capture(), anyList(), any());
|
||||
|
||||
List<Integer> sizeValues = sizes.getAllValues();
|
||||
List<Integer> offsetValues = offsets.getAllValues();
|
||||
assertThat(sizeValues.size()).isLessThanOrEqualTo(QuickSearchAction.MAX_LOCKED_ROUND_TRIPS);
|
||||
assertThat(sizeValues).allMatch(size -> size <= QuickSearchAction.MAX_LOCKED_BATCH_SIZE);
|
||||
assertThat(sizeValues.stream().mapToInt(Integer::intValue).sum()).isEqualTo(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW);
|
||||
assertThat(offsetValues.get(offsetValues.size() - 1) + sizeValues.get(sizeValues.size() - 1)).isEqualTo(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when every hit is filtered out by a lock, one search stops at the
|
||||
** request budget: at most MAX_RESULT_WINDOW raw hits and
|
||||
** MAX_LOCKED_ROUND_TRIPS requests, in shared and per-table mode, with an
|
||||
** empty page and an honest hasMore.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_allHitsFiltered_workIsBounded() throws QException
|
||||
{
|
||||
stubPagedSearch(Map.of(TEST_ENTITY_TABLE, IntStream.rangeClosed(1, 50_000).mapToObj(String::valueOf).toList()), true);
|
||||
|
||||
QuickSearchOutput shared = action.execute(new QuickSearchInput().withSearchTerm("entity").withLimit(1));
|
||||
assertScanWithinBudget();
|
||||
clearInvocations(mockClient);
|
||||
|
||||
QuickSearchOutput perTable = action.execute(new QuickSearchInput().withSearchTerm("entity").withOffset(3).withLimitPerTable(1));
|
||||
assertScanWithinBudget();
|
||||
|
||||
assertThat(shared.getResults()).isEmpty();
|
||||
assertThat(perTable.getResults()).isEmpty();
|
||||
assertThat(shared.getTotalHits()).isEqualTo(0L);
|
||||
assertThat(shared.getHasMore()).isFalse();
|
||||
assertThat(shared.getTotalHitsIsLowerBound()).isTrue();
|
||||
assertThat(perTable.getHasMore()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a second memory table and make both quick-searchable.
|
||||
*******************************************************************************/
|
||||
private void addOtherTable()
|
||||
{
|
||||
QContext.getQInstance().addTable(new QTableMetaData()
|
||||
.withName(OTHER_TABLE)
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
QuickSearchQBitContext.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName(TEST_ENTITY_TABLE).withPrimaryKeyField("id").withEnabledByDefault(true),
|
||||
new QuickSearchableTableConfig().withTableName(OTHER_TABLE).withPrimaryKeyField("id").withEnabledByDefault(true)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: in per-table mode each table has its own scan budget, so a table
|
||||
** with many hits all locked out cannot starve the next table, which still
|
||||
** returns its accessible hits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_withLocks_lockedTableDoesNotStarveOthers() throws QException
|
||||
{
|
||||
addOtherTable();
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(OTHER_TABLE);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("name", "Other 1"), new QRecord().withValue("id", 2).withValue("name", "Other 2")));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
stubPagedSearch(Map.of(
|
||||
TEST_ENTITY_TABLE, IntStream.rangeClosed(1, 12_000).mapToObj(String::valueOf).toList(),
|
||||
OTHER_TABLE, List.of("1", "2")), false);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withTableNames(List.of(TEST_ENTITY_TABLE, OTHER_TABLE)).withLimitPerTable(5));
|
||||
|
||||
verify(mockClient, atLeastOnce()).search(anyString(), eq(List.of(OTHER_TABLE)), anyInt(), anyInt(), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getTableName).containsOnly(OTHER_TABLE);
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1", "2");
|
||||
assertThat(output.getTotalHits()).isEqualTo(2L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with both tables fully locked, each table stops at its equal share
|
||||
** and the raw hits fetched across tables stay within
|
||||
** max(10,000, T x 2 x limitPerTable); the share ends paging like the
|
||||
** window, so hasMore is false and the total is a lower bound.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_withLocks_totalWorkWithinBound() throws QException
|
||||
{
|
||||
addOtherTable();
|
||||
List<String> ids = IntStream.rangeClosed(1, 50_000).mapToObj(String::valueOf).toList();
|
||||
stubPagedSearch(Map.of(TEST_ENTITY_TABLE, ids, OTHER_TABLE, ids), false);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("entity").withTableNames(List.of(TEST_ENTITY_TABLE, OTHER_TABLE)).withOffset(5_000).withLimitPerTable(5));
|
||||
|
||||
int total = 0;
|
||||
for(String table : List.of(TEST_ENTITY_TABLE, OTHER_TABLE))
|
||||
{
|
||||
ArgumentCaptor<Integer> sizes = ArgumentCaptor.forClass(Integer.class);
|
||||
verify(mockClient, atLeastOnce()).search(anyString(), eq(List.of(table)), sizes.capture(), anyInt(), anyList(), any());
|
||||
assertThat(sizes.getAllValues().size()).isLessThanOrEqualTo(QuickSearchAction.MAX_LOCKED_ROUND_TRIPS / 2);
|
||||
assertThat(sizes.getAllValues()).allMatch(size -> size <= QuickSearchAction.MAX_LOCKED_BATCH_SIZE);
|
||||
assertThat(sizes.getAllValues().stream().mapToInt(Integer::intValue).sum()).isEqualTo(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW / 2);
|
||||
total += sizes.getAllValues().stream().mapToInt(Integer::intValue).sum();
|
||||
}
|
||||
assertThat(total).isLessThanOrEqualTo(Math.max(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW, 2 * 2 * 5));
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a deep offset under locks reads in capped batches rather than one
|
||||
** request (and one record-lock query) the size of the window.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_deepOffset_batchesCapped() throws QException
|
||||
{
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 20_000).mapToObj(String::valueOf).toList());
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("entity").withOffset(4_975).withLimit(25));
|
||||
|
||||
ArgumentCaptor<Integer> sizes = ArgumentCaptor.forClass(Integer.class);
|
||||
verify(mockClient, atLeastOnce()).search(anyString(), any(), sizes.capture(), anyInt(), anyList(), any());
|
||||
assertThat(sizes.getAllValues().get(0)).isEqualTo(QuickSearchAction.MAX_LOCKED_BATCH_SIZE);
|
||||
assertThat(sizes.getAllValues()).allMatch(size -> size <= QuickSearchAction.MAX_LOCKED_BATCH_SIZE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: accessible hits found only in later batches fill the page, and
|
||||
** unread candidates keep hasMore true with a lower-bound total.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_accessibleHitsInLaterBatches() throws QException
|
||||
{
|
||||
insertEntities(60, 70, 80);
|
||||
stubPagedSearch(IntStream.rangeClosed(1, 100).mapToObj(String::valueOf).toList());
|
||||
|
||||
QuickSearchOutput output = page(1, 2);
|
||||
|
||||
verify(mockClient, times(4)).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("70", "80");
|
||||
assertThat(output.getHasMore()).isTrue();
|
||||
assertThat(output.getTotalHits()).isEqualTo(3L);
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a hit returned again by a later batch is counted once, so the
|
||||
** offset skips it once and the page holds the next accessible hits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecordLock_duplicateAcrossBatches_returnedOnce() throws QException
|
||||
{
|
||||
insertEntities(1, 2, 3);
|
||||
stubPagedSearch(List.of("9", "8", "7", "6", "5", "1", "1", "2", "3"));
|
||||
|
||||
QuickSearchOutput output = page(1, 2);
|
||||
|
||||
verify(mockClient, times(2)).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("2", "3");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: paging a table with offset += results until hasMore is false
|
||||
** terminates at the table's share, with every result returned once.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_withLocks_pagingUntilNoMoreTerminates() throws QException
|
||||
{
|
||||
addOtherTable();
|
||||
QuickSearchQBitContext.getConfig().withMaxSearchLimit(1_000);
|
||||
insertEntities(IntStream.rangeClosed(1, 6_000).boxed().toArray(Integer[]::new));
|
||||
stubPagedSearch(Map.of(TEST_ENTITY_TABLE, IntStream.rangeClosed(1, 12_000).mapToObj(String::valueOf).toList()), false);
|
||||
|
||||
List<String> seen = new ArrayList<>();
|
||||
int pages = 0;
|
||||
QuickSearchOutput output;
|
||||
do
|
||||
{
|
||||
output = action.execute(new QuickSearchInput().withSearchTerm("entity").withTableNames(List.of(TEST_ENTITY_TABLE, OTHER_TABLE)).withOffset(seen.size()).withLimitPerTable(1_000));
|
||||
output.getResults().forEach(result -> seen.add(result.getRecordId()));
|
||||
pages++;
|
||||
}
|
||||
while(output.getHasMore() && pages < 20);
|
||||
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isTrue();
|
||||
assertThat(pages).isEqualTo(5);
|
||||
assertThat(seen).doesNotHaveDuplicates().hasSize(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW / 2);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
@@ -29,18 +30,24 @@ import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.opensearch.client.opensearch._types.ErrorResponse;
|
||||
import org.opensearch.client.opensearch._types.OpenSearchException;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.opensearch.client.opensearch.core.search.Hit;
|
||||
import org.opensearch.client.opensearch.core.search.HitsMetadata;
|
||||
import org.opensearch.client.opensearch.core.search.TotalHits;
|
||||
import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyCollection;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -49,7 +56,8 @@ import static org.mockito.Mockito.when;
|
||||
** Tests for QuickSearchAction.
|
||||
**
|
||||
** Uses Mockito to mock QuickSearchOpenSearchClient and the OpenSearch
|
||||
** SearchResponse chain (SearchResponse → HitsMetadata → TotalHits / Hit).
|
||||
** SearchResponse chain (SearchResponse, HitsMetadata, TotalHits, Hit). Record
|
||||
** security locks are off here; QuickSearchActionSecurityTest covers them.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
class QuickSearchActionTest
|
||||
@@ -74,6 +82,7 @@ class QuickSearchActionTest
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("orderNumber", "customerName")));
|
||||
|
||||
QuickSearchQBitContext.setConfig(new QuickSearchQBitConfig().withApplyRecordSecurityLocks(false));
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
QuickSearchQBitContext.setDiscoveredTables(tableConfigs);
|
||||
|
||||
@@ -192,7 +201,7 @@ class QuickSearchActionTest
|
||||
Hit<OpenSearchDocument> hit = buildMockHit(doc, 1.5, Map.of("searchableText", List.of("Order <em>42</em>")));
|
||||
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(List.of(hit), 1L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchInput input = new QuickSearchInput()
|
||||
.withSearchTerm("42")
|
||||
@@ -228,7 +237,7 @@ class QuickSearchActionTest
|
||||
Hit<OpenSearchDocument> hit = buildMockHit(doc, 1.0, Map.of("searchableText", List.of("fragment one", "fragment two")));
|
||||
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(List.of(hit), 1L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("one"));
|
||||
|
||||
@@ -251,7 +260,7 @@ class QuickSearchActionTest
|
||||
Hit<OpenSearchDocument> hit = buildMockHit(doc, 0.9, null);
|
||||
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(List.of(hit), 1L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("order"));
|
||||
|
||||
@@ -267,11 +276,11 @@ class QuickSearchActionTest
|
||||
void testNullLimit_defaultsTo25() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(null));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList());
|
||||
verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -283,11 +292,11 @@ class QuickSearchActionTest
|
||||
void testZeroLimit_defaultsTo25() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(0));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList());
|
||||
verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -299,11 +308,11 @@ class QuickSearchActionTest
|
||||
void testNegativeLimit_defaultsTo25() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(-5));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList());
|
||||
verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -315,11 +324,11 @@ class QuickSearchActionTest
|
||||
void testNullOffset_defaultsTo0() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withOffset(null));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), anyInt(), eq(0), anyList());
|
||||
verify(mockClient).search(anyString(), any(), anyInt(), eq(0), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -331,11 +340,11 @@ class QuickSearchActionTest
|
||||
void testNegativeOffset_defaultsTo0() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withOffset(-1));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), anyInt(), eq(0), anyList());
|
||||
verify(mockClient).search(anyString(), any(), anyInt(), eq(0), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -353,7 +362,7 @@ class QuickSearchActionTest
|
||||
|
||||
// 1 result, offset=0, totalHits=5 → hasMore = (0+1) < 5 = true
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(hits, 5L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput()
|
||||
.withSearchTerm("order")
|
||||
@@ -383,7 +392,7 @@ class QuickSearchActionTest
|
||||
|
||||
// 2 results, offset=3, totalHits=5 → hasMore = (3+2) < 5 = false
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(hits, 5L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput()
|
||||
.withSearchTerm("order")
|
||||
@@ -396,19 +405,20 @@ class QuickSearchActionTest
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tableName filter is passed through to client.search().
|
||||
** tableName filter is passed through to client.search() when it names a
|
||||
** configured table.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableNameFilter_passedToClient() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), anyString(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), anyCollection(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput()
|
||||
.withSearchTerm("widget")
|
||||
.withTableName("products"));
|
||||
.withTableName("orders"));
|
||||
|
||||
verify(mockClient).search(anyString(), eq("products"), anyInt(), anyInt(), anyList());
|
||||
verify(mockClient).search(anyString(), eq(List.of("orders")), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
@@ -417,35 +427,344 @@ class QuickSearchActionTest
|
||||
** null tableName is passed through to client.search() as null.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNullTableName_passedAsNullToClient() throws QException
|
||||
void testNullTableName_searchesAllConfiguredTables() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), isNull(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), anyCollection(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput()
|
||||
.withSearchTerm("widget")
|
||||
.withTableName(null));
|
||||
|
||||
verify(mockClient).search(anyString(), isNull(), anyInt(), anyInt(), anyList());
|
||||
verify(mockClient).search(anyString(), anyCollection(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Search term is normalized (lowercased, trimmed) before being passed to client.
|
||||
** Search term is normalized (trimmed, whitespace collapsed, case kept)
|
||||
** before being passed to client.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSearchTerm_isNormalizedBeforeClientCall() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm(" HELLO WORLD "));
|
||||
|
||||
ArgumentCaptor<String> termCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(mockClient).search(termCaptor.capture(), any(), anyInt(), anyInt(), anyList());
|
||||
verify(mockClient).search(termCaptor.capture(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
|
||||
assertThat(termCaptor.getValue()).isEqualTo("hello world");
|
||||
assertThat(termCaptor.getValue()).isEqualTo("HELLO WORLD");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A one-character term is below the minimum and returns empty output
|
||||
** without touching the client.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOneCharacterTerm_returnsEmptyWithoutSearching() throws QException
|
||||
{
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm(" a "));
|
||||
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
assertThat(output.getTotalHits()).isEqualTo(0L);
|
||||
verify(mockClient, never()).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A term over 100 characters is rejected.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTermOver100Characters_throws() throws QException
|
||||
{
|
||||
String longTerm = "x".repeat(QuickSearchAction.MAX_TERM_LENGTH + 1);
|
||||
|
||||
assertThatThrownBy(() -> action.execute(new QuickSearchInput().withSearchTerm(longTerm)))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("too long");
|
||||
|
||||
verify(mockClient, never()).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A term of exactly 100 characters is accepted.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTermOfExactly100Characters_accepted() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("x".repeat(QuickSearchAction.MAX_TERM_LENGTH)));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** limit above the configured maxSearchLimit is clamped to it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitAboveMaxSearchLimit_clamped() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setConfig(new QuickSearchQBitConfig().withApplyRecordSecurityLocks(false).withMaxSearchLimit(40));
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(500));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(40), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without a config the limit is still capped at the default maximum.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitAboveDefaultMax_noConfig_clampedTo100() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setConfig(null);
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(500));
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// no config means locks default to on, so the fetch size is doubled //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
verify(mockClient).search(anyString(), any(), eq(200), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** offset + limit beyond the OpenSearch result window shrinks the limit so
|
||||
** the window is not exceeded.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOffsetPlusLimit_beyondResultWindow_limitShrunk() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(25).withOffset(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW - 10));
|
||||
|
||||
verify(mockClient).search(anyString(), any(), eq(10), eq(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW - 10), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An offset at or past the result window returns empty output without
|
||||
** searching.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOffsetAtResultWindow_returnsEmptyWithoutSearching() throws QException
|
||||
{
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(25).withOffset(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW));
|
||||
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
assertThat(output.getHasMore()).isFalse();
|
||||
verify(mockClient, never()).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tableNames is intersected with the configured tables.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableNames_intersectedWithConfiguredTables() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), anyCollection(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("widget").withTableNames(List.of("orders", "notConfigured")));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of("orders")), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tableNames takes precedence over the single tableName.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableNames_takesPrecedenceOverTableName() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName("orders").withPrimaryKeyField("id").withSearchableFields(List.of("orderNumber")),
|
||||
new QuickSearchableTableConfig().withTableName("customers").withPrimaryKeyField("id").withSearchableFields(List.of("name"))));
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 0L);
|
||||
when(mockClient.search(anyString(), anyCollection(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
action.execute(new QuickSearchInput().withSearchTerm("widget").withTableName("orders").withTableNames(List.of("customers")));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of("customers")), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A request naming only unconfigured tables returns empty output without
|
||||
** searching.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableNames_noneConfigured_returnsEmptyWithoutSearching() throws QException
|
||||
{
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("widget").withTableNames(List.of("notConfigured")));
|
||||
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
verify(mockClient, never()).search(anyString(), any(), anyInt(), anyInt(), anyList(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** limitPerTable searches each allowed table separately with that limit and
|
||||
** merges the hits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_searchesEachTableSeparately() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName("orders").withPrimaryKeyField("id").withSearchableFields(List.of("orderNumber")),
|
||||
new QuickSearchableTableConfig().withTableName("customers").withPrimaryKeyField("id").withSearchableFields(List.of("name"))));
|
||||
|
||||
Hit<OpenSearchDocument> orderHit = buildMockHit(new OpenSearchDocument().withSourceTable("orders").withRecordId("1").withRecordLabel("Order 1"), 2.0, null);
|
||||
Hit<OpenSearchDocument> customerHit = buildMockHit(new OpenSearchDocument().withSourceTable("customers").withRecordId("9").withRecordLabel("Customer 9"), 1.0, null);
|
||||
|
||||
SearchResponse<OpenSearchDocument> orderResponse = buildMockResponse(List.of(orderHit), 7L);
|
||||
SearchResponse<OpenSearchDocument> customerResponse = buildMockResponse(List.of(customerHit), 1L);
|
||||
when(mockClient.search(anyString(), eq(List.of("orders")), anyInt(), anyInt(), anyList(), any())).thenReturn(orderResponse);
|
||||
when(mockClient.search(anyString(), eq(List.of("customers")), anyInt(), anyInt(), anyList(), any())).thenReturn(customerResponse);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("widget").withLimitPerTable(3));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of("orders")), eq(3), eq(0), anyList(), any());
|
||||
verify(mockClient).search(anyString(), eq(List.of("customers")), eq(3), eq(0), anyList(), any());
|
||||
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1", "9");
|
||||
assertThat(output.getTotalHits()).isEqualTo(8L);
|
||||
assertThat(output.getHasMore()).as("orders has 7 hits but only 1 was returned").isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** limitPerTable is capped at maxSearchLimit and trims each table's hits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLimitPerTable_aboveMax_clamped() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setConfig(new QuickSearchQBitConfig().withApplyRecordSecurityLocks(false).withMaxSearchLimit(2));
|
||||
|
||||
List<Hit<OpenSearchDocument>> hits = List.of(
|
||||
buildMockHit(new OpenSearchDocument().withSourceTable("orders").withRecordId("1"), 3.0, null),
|
||||
buildMockHit(new OpenSearchDocument().withSourceTable("orders").withRecordId("2"), 2.0, null));
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(hits, 3L);
|
||||
when(mockClient.search(anyString(), anyCollection(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("widget").withLimitPerTable(50));
|
||||
|
||||
verify(mockClient).search(anyString(), eq(List.of("orders")), eq(2), eq(0), anyList(), any());
|
||||
assertThat(output.getResults()).extracting(QuickSearchResult::getRecordId).containsExactly("1", "2");
|
||||
assertThat(output.getHasMore()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without a QInstance in context the table label falls back to the table
|
||||
** name.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableLabel_noQInstance_fallsBackToTableName() throws QException
|
||||
{
|
||||
Hit<OpenSearchDocument> hit = buildMockHit(new OpenSearchDocument().withSourceTable("orders").withRecordId("1"), 1.0, null);
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(List.of(hit), 1L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("widget"));
|
||||
|
||||
assertThat(output.getResults().get(0).getTableLabel()).isEqualTo("orders");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** totalHitsIsLowerBound mirrors a "gte" total relation from OpenSearch.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTotalHitsIsLowerBound_whenRelationIsGte() throws QException
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(Collections.emptyList(), 10000L);
|
||||
when(response.hits().total().relation()).thenReturn(TotalHitsRelation.Gte);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("widget"));
|
||||
|
||||
assertThat(output.getTotalHitsIsLowerBound()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A hit without a source document is skipped.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testHitWithoutSource_skipped() throws QException
|
||||
{
|
||||
Hit<OpenSearchDocument> hit = buildMockHit(null, 1.0, null);
|
||||
SearchResponse<OpenSearchDocument> response = buildMockResponse(List.of(hit), 1L);
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenReturn(response);
|
||||
|
||||
QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("widget"));
|
||||
|
||||
assertThat(output.getResults()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A missing client is reported as a QException.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoClient_throwsQException()
|
||||
{
|
||||
QuickSearchQBitContext.setClient(null);
|
||||
|
||||
assertThatThrownBy(() -> action.execute(new QuickSearchInput().withSearchTerm("widget")))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("not initialized");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An OpenSearchException escaping the client surfaces as a QException.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOpenSearchException_surfacesAsQException() throws QException
|
||||
{
|
||||
OpenSearchException openSearchException = new OpenSearchException(ErrorResponse.of(e -> e
|
||||
.status(500)
|
||||
.error(c -> c.type("search_phase_execution_exception").reason("all shards failed"))));
|
||||
when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList(), any())).thenThrow(openSearchException);
|
||||
|
||||
assertThatThrownBy(() -> action.execute(new QuickSearchInput().withSearchTerm("widget")))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("all shards failed");
|
||||
}
|
||||
|
||||
}
|
||||
-222
@@ -1,222 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.customizers;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatNoException;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests for QuickSearchPostDeleteCustomizer.
|
||||
*******************************************************************************/
|
||||
class QuickSearchPostDeleteCustomizerTest
|
||||
{
|
||||
private static final String TABLE_NAME = "orders";
|
||||
private static final String PK_FIELD = "id";
|
||||
|
||||
private IndexEventPublisher mockPublisher;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set up mock publisher in QuickSearchQBitContext and a QContext with a test
|
||||
** QInstance before each test.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
mockPublisher = mock(IndexEventPublisher.class);
|
||||
QuickSearchQBitContext.setPublisher(mockPublisher);
|
||||
QuickSearchQBitContext.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName(TABLE_NAME).withPrimaryKeyField(PK_FIELD)
|
||||
));
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withPrimaryKeyField(PK_FIELD)
|
||||
.withField(new QFieldMetaData(PK_FIELD, QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addBackend(new com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData().withName("testBackend"));
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.getTable(TABLE_NAME).withBackendName("testBackend");
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clear QuickSearchQBitContext and QContext after each test to prevent leakage.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QuickSearchQBitContext.clear();
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that no-arg constructor completes without throwing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoArgConstructor_doesNotThrow()
|
||||
{
|
||||
assertThatNoException().isThrownBy(QuickSearchPostDeleteCustomizer::new);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-delete with valid records builds correct DELETE events and
|
||||
** calls publisher.publishDeleteEvents. Records have no QRecord attached
|
||||
** (DELETE events only carry tableName and recordId).
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void testPostDelete_withRecords_publishesDeleteEvents() throws QException
|
||||
{
|
||||
QRecord record1 = new QRecord().withValue(PK_FIELD, 5);
|
||||
QRecord record2 = new QRecord().withValue(PK_FIELD, 6);
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer();
|
||||
List<QRecord> result = customizer.postDelete(deleteInput, List.of(record1, record2));
|
||||
|
||||
assertThat(result).containsExactly(record1, record2);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mockPublisher).publishDeleteEvents(captor.capture());
|
||||
|
||||
List<IndexEvent> events = captor.getValue();
|
||||
assertThat(events).hasSize(2);
|
||||
assertThat(events.get(0).getAction()).isEqualTo(IndexEventAction.DELETE);
|
||||
assertThat(events.get(0).getTableName()).isEqualTo(TABLE_NAME);
|
||||
assertThat(events.get(0).getRecordId()).isEqualTo("5");
|
||||
assertThat(events.get(0).getRecord()).isNull();
|
||||
assertThat(events.get(1).getAction()).isEqualTo(IndexEventAction.DELETE);
|
||||
assertThat(events.get(1).getRecordId()).isEqualTo("6");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-delete with empty records list returns input without calling
|
||||
** publisher.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostDelete_withEmptyRecords_returnsInputWithoutPublishing() throws QException
|
||||
{
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer();
|
||||
List<QRecord> result = customizer.postDelete(deleteInput, List.of());
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
verify(mockPublisher, never()).publishDeleteEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-delete with null records returns null without calling publisher.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostDelete_withNullRecords_returnsNullWithoutPublishing() throws QException
|
||||
{
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer();
|
||||
List<QRecord> result = customizer.postDelete(deleteInput, null);
|
||||
|
||||
assertThat(result).isNull();
|
||||
verify(mockPublisher, never()).publishDeleteEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that when publisher is null, records are still returned (no exception).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostDelete_publisherNull_returnsRecordsWithoutThrowing() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setPublisher(null);
|
||||
|
||||
QRecord record = new QRecord().withValue(PK_FIELD, 1);
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer();
|
||||
List<QRecord> result = customizer.postDelete(deleteInput, List.of(record));
|
||||
|
||||
assertThat(result).containsExactly(record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that when publisher throws, a warning is logged and records are returned
|
||||
** (no exception propagated to caller).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostDelete_publisherThrows_returnsRecordsWithoutThrowing() throws QException
|
||||
{
|
||||
doThrow(new QException("publish failed"))
|
||||
.when(mockPublisher).publishDeleteEvents(anyList());
|
||||
|
||||
QRecord record = new QRecord().withValue(PK_FIELD, 1);
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer();
|
||||
assertThatNoException().isThrownBy(() -> customizer.postDelete(deleteInput, List.of(record)));
|
||||
}
|
||||
|
||||
}
|
||||
-222
@@ -1,222 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.customizers;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatNoException;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests for QuickSearchPostInsertCustomizer.
|
||||
*******************************************************************************/
|
||||
class QuickSearchPostInsertCustomizerTest
|
||||
{
|
||||
private static final String TABLE_NAME = "orders";
|
||||
private static final String PK_FIELD = "id";
|
||||
|
||||
private IndexEventPublisher mockPublisher;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set up mock publisher in QuickSearchQBitContext and a QContext with a test
|
||||
** QInstance before each test.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
mockPublisher = mock(IndexEventPublisher.class);
|
||||
QuickSearchQBitContext.setPublisher(mockPublisher);
|
||||
QuickSearchQBitContext.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName(TABLE_NAME).withPrimaryKeyField(PK_FIELD)
|
||||
));
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withPrimaryKeyField(PK_FIELD)
|
||||
.withField(new QFieldMetaData(PK_FIELD, QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addBackend(new com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData().withName("testBackend"));
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.getTable(TABLE_NAME).withBackendName("testBackend");
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clear QuickSearchQBitContext and QContext after each test to prevent leakage.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QuickSearchQBitContext.clear();
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that no-arg constructor completes without throwing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoArgConstructor_doesNotThrow()
|
||||
{
|
||||
assertThatNoException().isThrownBy(QuickSearchPostInsertCustomizer::new);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-insert with valid records builds correct INDEX events and
|
||||
** calls publisher.publishIndexEvents.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void testPostInsert_withRecords_publishesIndexEvents() throws QException
|
||||
{
|
||||
QRecord record1 = new QRecord().withValue(PK_FIELD, 1).withValue("name", "Alice");
|
||||
QRecord record2 = new QRecord().withValue(PK_FIELD, 2).withValue("name", "Bob");
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer();
|
||||
List<QRecord> result = customizer.postInsert(insertInput, List.of(record1, record2));
|
||||
|
||||
assertThat(result).containsExactly(record1, record2);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mockPublisher).publishIndexEvents(captor.capture());
|
||||
|
||||
List<IndexEvent> events = captor.getValue();
|
||||
assertThat(events).hasSize(2);
|
||||
assertThat(events.get(0).getAction()).isEqualTo(IndexEventAction.INDEX);
|
||||
assertThat(events.get(0).getTableName()).isEqualTo(TABLE_NAME);
|
||||
assertThat(events.get(0).getRecordId()).isEqualTo("1");
|
||||
assertThat(events.get(0).getRecord()).isSameAs(record1);
|
||||
assertThat(events.get(1).getAction()).isEqualTo(IndexEventAction.INDEX);
|
||||
assertThat(events.get(1).getRecordId()).isEqualTo("2");
|
||||
assertThat(events.get(1).getRecord()).isSameAs(record2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-insert with empty records list returns input without calling
|
||||
** publisher.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostInsert_withEmptyRecords_returnsInputWithoutPublishing() throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer();
|
||||
List<QRecord> result = customizer.postInsert(insertInput, List.of());
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
verify(mockPublisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-insert with null records returns null without calling publisher.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostInsert_withNullRecords_returnsNullWithoutPublishing() throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer();
|
||||
List<QRecord> result = customizer.postInsert(insertInput, null);
|
||||
|
||||
assertThat(result).isNull();
|
||||
verify(mockPublisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that when publisher is null, records are still returned (no exception).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostInsert_publisherNull_returnsRecordsWithoutThrowing() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setPublisher(null);
|
||||
|
||||
QRecord record = new QRecord().withValue(PK_FIELD, 1);
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer();
|
||||
List<QRecord> result = customizer.postInsert(insertInput, List.of(record));
|
||||
|
||||
assertThat(result).containsExactly(record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that when publisher throws, a warning is logged and records are returned
|
||||
** (no exception propagated to caller).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostInsert_publisherThrows_returnsRecordsWithoutThrowing() throws QException
|
||||
{
|
||||
doThrow(new QException("publish failed"))
|
||||
.when(mockPublisher).publishIndexEvents(anyList());
|
||||
|
||||
QRecord record = new QRecord().withValue(PK_FIELD, 1);
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer();
|
||||
assertThatNoException().isThrownBy(() -> customizer.postInsert(insertInput, List.of(record)));
|
||||
}
|
||||
|
||||
}
|
||||
-223
@@ -1,223 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.customizers;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatNoException;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests for QuickSearchPostUpdateCustomizer.
|
||||
*******************************************************************************/
|
||||
class QuickSearchPostUpdateCustomizerTest
|
||||
{
|
||||
private static final String TABLE_NAME = "orders";
|
||||
private static final String PK_FIELD = "id";
|
||||
|
||||
private IndexEventPublisher mockPublisher;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set up mock publisher in QuickSearchQBitContext and a QContext with a test
|
||||
** QInstance before each test.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
mockPublisher = mock(IndexEventPublisher.class);
|
||||
QuickSearchQBitContext.setPublisher(mockPublisher);
|
||||
QuickSearchQBitContext.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName(TABLE_NAME).withPrimaryKeyField(PK_FIELD)
|
||||
));
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withPrimaryKeyField(PK_FIELD)
|
||||
.withField(new QFieldMetaData(PK_FIELD, QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addBackend(new com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData().withName("testBackend"));
|
||||
qInstance.setAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.getTable(TABLE_NAME).withBackendName("testBackend");
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clear QuickSearchQBitContext and QContext after each test to prevent leakage.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QuickSearchQBitContext.clear();
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that no-arg constructor completes without throwing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoArgConstructor_doesNotThrow()
|
||||
{
|
||||
assertThatNoException().isThrownBy(QuickSearchPostUpdateCustomizer::new);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-update with valid records builds correct INDEX events and
|
||||
** calls publisher.publishIndexEvents.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void testPostUpdate_withRecords_publishesIndexEvents() throws QException
|
||||
{
|
||||
QRecord record1 = new QRecord().withValue(PK_FIELD, 10).withValue("name", "Alice Updated");
|
||||
QRecord record2 = new QRecord().withValue(PK_FIELD, 20).withValue("name", "Bob Updated");
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer();
|
||||
List<QRecord> result = customizer.postUpdate(updateInput, List.of(record1, record2), Optional.empty());
|
||||
|
||||
assertThat(result).containsExactly(record1, record2);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mockPublisher).publishIndexEvents(captor.capture());
|
||||
|
||||
List<IndexEvent> events = captor.getValue();
|
||||
assertThat(events).hasSize(2);
|
||||
assertThat(events.get(0).getAction()).isEqualTo(IndexEventAction.INDEX);
|
||||
assertThat(events.get(0).getTableName()).isEqualTo(TABLE_NAME);
|
||||
assertThat(events.get(0).getRecordId()).isEqualTo("10");
|
||||
assertThat(events.get(0).getRecord()).isSameAs(record1);
|
||||
assertThat(events.get(1).getAction()).isEqualTo(IndexEventAction.INDEX);
|
||||
assertThat(events.get(1).getRecordId()).isEqualTo("20");
|
||||
assertThat(events.get(1).getRecord()).isSameAs(record2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-update with empty records list returns input without calling
|
||||
** publisher.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostUpdate_withEmptyRecords_returnsInputWithoutPublishing() throws QException
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer();
|
||||
List<QRecord> result = customizer.postUpdate(updateInput, List.of(), Optional.empty());
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
verify(mockPublisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that post-update with null records returns null without calling publisher.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostUpdate_withNullRecords_returnsNullWithoutPublishing() throws QException
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer();
|
||||
List<QRecord> result = customizer.postUpdate(updateInput, null, Optional.empty());
|
||||
|
||||
assertThat(result).isNull();
|
||||
verify(mockPublisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that when publisher is null, records are still returned (no exception).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostUpdate_publisherNull_returnsRecordsWithoutThrowing() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.setPublisher(null);
|
||||
|
||||
QRecord record = new QRecord().withValue(PK_FIELD, 1);
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer();
|
||||
List<QRecord> result = customizer.postUpdate(updateInput, List.of(record), Optional.empty());
|
||||
|
||||
assertThat(result).containsExactly(record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that when publisher throws, a warning is logged and records are returned
|
||||
** (no exception propagated to caller).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPostUpdate_publisherThrows_returnsRecordsWithoutThrowing() throws QException
|
||||
{
|
||||
doThrow(new QException("publish failed"))
|
||||
.when(mockPublisher).publishIndexEvents(anyList());
|
||||
|
||||
QRecord record = new QRecord().withValue(PK_FIELD, 1);
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TABLE_NAME);
|
||||
|
||||
QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer();
|
||||
assertThatNoException().isThrownBy(() -> customizer.postUpdate(updateInput, List.of(record), Optional.empty()));
|
||||
}
|
||||
|
||||
}
|
||||
+657
@@ -0,0 +1,657 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.listeners;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.MissingLibraryTransportCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.MockedConstruction;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockConstruction;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for QuickSearchRecordChangeListener against the memory backend.
|
||||
*******************************************************************************/
|
||||
class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
{
|
||||
private static final String COMPANY_TABLE = "company";
|
||||
|
||||
private IndexEventPublisher publisher;
|
||||
private QuickSearchRuntime runtime;
|
||||
|
||||
|
||||
|
||||
@BeforeEach
|
||||
void setUpListener()
|
||||
{
|
||||
QuickSearchQBitContext.clear();
|
||||
publisher = mock(IndexEventPublisher.class);
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("test")
|
||||
.withIndexEventPublisher(publisher);
|
||||
|
||||
runtime = new QuickSearchRuntime(config);
|
||||
runtime.setDiscoveredTables(List.of(new QuickSearchableTableConfig()
|
||||
.withTableName(TEST_ENTITY_TABLE)
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("name", "description"))));
|
||||
config.setRuntime(runtime);
|
||||
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// a foreign key to a company table, through a table PVS, and a //
|
||||
// record label over it, so display values can be asserted //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(COMPANY_TABLE)
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s")
|
||||
.withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(COMPANY_TABLE));
|
||||
qInstance.getTable(TEST_ENTITY_TABLE)
|
||||
.withField(new QFieldMetaData("companyId", QFieldType.INTEGER).withPossibleValueSourceName(COMPANY_TABLE))
|
||||
.withRecordLabelFormat("%s - %s")
|
||||
.withRecordLabelFields("name", "companyId");
|
||||
|
||||
qInstance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("quick-search").withVersion("test").withConfig(config));
|
||||
qInstance.withRecordChangeListener(new QCodeReference(QuickSearchRecordChangeListener.class));
|
||||
}
|
||||
|
||||
|
||||
|
||||
private Integer insert(String name, QBackendTransaction transaction) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TEST_ENTITY_TABLE);
|
||||
insertInput.setTransaction(transaction);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("name", name).withValue("description", "d")));
|
||||
return (new InsertAction().execute(insertInput).getRecords().get(0).getValueInteger("id"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testAppliesTo_onlyConfiguredTablesWhenEnabled()
|
||||
{
|
||||
QuickSearchRecordChangeListener listener = new QuickSearchRecordChangeListener();
|
||||
assertThat(listener.appliesTo(TEST_ENTITY_TABLE, RecordChangeType.INSERT)).isTrue();
|
||||
assertThat(listener.appliesTo(QuickSearchIndex.TABLE_NAME, RecordChangeType.INSERT)).isFalse();
|
||||
|
||||
runtime.getConfig().setEnableRealTimeIndexing(false);
|
||||
assertThat(listener.appliesTo(TEST_ENTITY_TABLE, RecordChangeType.INSERT)).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withoutTransaction_publishesImmediately() throws QException
|
||||
{
|
||||
Integer id = insert("widget", null);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(publisher, times(1)).publishIndexEvents(captor.capture());
|
||||
assertThat(captor.getValue()).hasSize(1);
|
||||
assertThat(captor.getValue().get(0).getAction()).isEqualTo(IndexEventAction.INDEX);
|
||||
assertThat(captor.getValue().get(0).getRecordId()).isEqualTo(String.valueOf(id));
|
||||
assertThat(captor.getValue().get(0).getRecord().getValueString("name")).isEqualTo("widget");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_indexesDisplayValuesAndPossibleValueLabels() throws QException
|
||||
{
|
||||
InsertInput companyInsert = new InsertInput();
|
||||
companyInsert.setTableName(COMPANY_TABLE);
|
||||
companyInsert.setRecords(List.of(new QRecord().withValue("name", "Acme Corp")));
|
||||
Integer companyId = new InsertAction().execute(companyInsert).getRecords().get(0).getValueInteger("id");
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TEST_ENTITY_TABLE);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("name", "widget").withValue("description", "d").withValue("companyId", companyId)));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(publisher, times(1)).publishIndexEvents(captor.capture());
|
||||
QRecord indexed = captor.getValue().get(0).getRecord();
|
||||
assertThat(indexed.getValueInteger("companyId")).isEqualTo(companyId);
|
||||
assertThat(indexed.getDisplayValue("companyId")).as("the PVS label, not the raw key").isEqualTo("Acme Corp");
|
||||
assertThat(indexed.getRecordLabel()).isEqualTo("widget - Acme Corp");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_reReadFails_recordsFailedEventsAfterCommit() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new QException("possible-value backend unavailable")))
|
||||
{
|
||||
assertThatCode(() -> new QuickSearchRecordChangeListener().onRecordsChanged(changeEvent(RecordChangeType.INSERT, transaction, 7, 8))).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
assertThat(queryFailedEvents()).isEmpty();
|
||||
transaction.commit();
|
||||
|
||||
List<QRecord> failed = queryFailedEvents();
|
||||
assertThat(failed).extracting(r -> r.getValueString("recordId")).containsExactlyInAnyOrder("7", "8");
|
||||
assertThat(failed).allSatisfy(r ->
|
||||
{
|
||||
assertThat(r.getValueString("action")).isEqualTo("INDEX");
|
||||
assertThat(r.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
|
||||
assertThat(r.getValueString("errorMessage")).contains("possible-value backend unavailable");
|
||||
});
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_reReadFailsThenRollback_logsCauseAndRecordsNothing() throws QException
|
||||
{
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new NoClassDefFoundError("com/example/pvs/Provider")))
|
||||
{
|
||||
new QuickSearchRecordChangeListener().onRecordsChanged(changeEvent(RecordChangeType.INSERT, transaction, 7));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
}
|
||||
transaction.rollback();
|
||||
|
||||
assertThat(queryFailedEvents()).isEmpty();
|
||||
assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains("Could not re-read inserted records").contains(TEST_ENTITY_TABLE).contains("com/example/pvs/Provider");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withTransaction_publishesOnlyAfterCommit() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
insert("widget", transaction);
|
||||
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
|
||||
transaction.commit();
|
||||
verify(publisher, times(1)).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withTransaction_rollbackNeverPublishes() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
insert("widget", transaction);
|
||||
transaction.rollback();
|
||||
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadsFullRecord() throws QException
|
||||
{
|
||||
Integer id = insert("widget", null);
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(TEST_ENTITY_TABLE);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", id).withValue("description", "changed")));
|
||||
new UpdateAction().execute(updateInput);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(publisher, times(2)).publishIndexEvents(captor.capture());
|
||||
IndexEvent updateEvent = captor.getAllValues().get(1).get(0);
|
||||
assertThat(updateEvent.getRecord().getValueString("name")).isEqualTo("widget");
|
||||
assertThat(updateEvent.getRecord().getValueString("description")).isEqualTo("changed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDelete_publishesDeleteEvents() throws QException
|
||||
{
|
||||
Integer id = insert("widget", null);
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TEST_ENTITY_TABLE);
|
||||
deleteInput.setPrimaryKeys(List.of(id));
|
||||
new DeleteAction().execute(deleteInput);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(publisher, times(1)).publishDeleteEvents(captor.capture());
|
||||
assertThat(captor.getValue()).hasSize(1);
|
||||
assertThat(captor.getValue().get(0).getAction()).isEqualTo(IndexEventAction.DELETE);
|
||||
assertThat(captor.getValue().get(0).getRecordId()).isEqualTo(String.valueOf(id));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testPublishFailure_writesFailedEventAndDoesNotFailWrite() throws QException
|
||||
{
|
||||
doThrow(new QException("cluster down")).when(publisher).publishIndexEvents(anyList());
|
||||
|
||||
Integer id = insert("widget", null);
|
||||
assertThat(id).isNotNull();
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchFailedEvent.TABLE_NAME);
|
||||
List<QRecord> failed = new QueryAction().execute(queryInput).getRecords();
|
||||
assertThat(failed).hasSize(1);
|
||||
assertThat(failed.get(0).getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE);
|
||||
assertThat(failed.get(0).getValueString("recordId")).isEqualTo(String.valueOf(id));
|
||||
assertThat(failed.get(0).getValueString("action")).isEqualTo("INDEX");
|
||||
assertThat(failed.get(0).getValueString("errorMessage")).contains("cluster down");
|
||||
assertThat(failed.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDisabledTable_isSkipped() throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("tableName", TEST_ENTITY_TABLE).withValue("enabled", false)));
|
||||
new InsertAction().execute(insertInput);
|
||||
runtime.invalidateEnabledCache();
|
||||
|
||||
insert("widget", null);
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDelete_duringFullReindex_isCapturedForTheNewIndex() throws QException
|
||||
{
|
||||
insertIndexRowWithStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
Integer id = insert("widget", null);
|
||||
delete(id);
|
||||
|
||||
verify(publisher, times(1)).publishDeleteEvents(anyList());
|
||||
|
||||
List<QRecord> captured = queryFailedEvents();
|
||||
assertThat(captured).hasSize(1);
|
||||
assertThat(captured.get(0).getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE);
|
||||
assertThat(captured.get(0).getValueString("recordId")).isEqualTo(String.valueOf(id));
|
||||
assertThat(captured.get(0).getValueString("action")).isEqualTo("DELETE");
|
||||
assertThat(captured.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDelete_publishFailureDuringFullReindex_recordsBothRows() throws QException
|
||||
{
|
||||
doThrow(new QException("cluster down")).when(publisher).publishDeleteEvents(anyList());
|
||||
insertIndexRowWithStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
delete(insert("widget", null));
|
||||
|
||||
assertThat(queryFailedEvents()).extracting(r -> r.getValueString("status"))
|
||||
.containsExactlyInAnyOrder(QuickSearchFailedEvent.STATUS_PENDING, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDelete_withoutFullReindex_isNotCaptured() throws QException
|
||||
{
|
||||
insertIndexRowWithStatus(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
delete(insert("widget", null));
|
||||
|
||||
verify(publisher, times(1)).publishDeleteEvents(anyList());
|
||||
assertThat(queryFailedEvents()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testCapture_rebuildFinishedBeforeInsert_handsRowsToBasepull() throws QException
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// the second read sees the rebuild finished, as if it ended between //
|
||||
// the first read and the insert //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
insertIndexRowWithStatus(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
List<IndexEvent> events = List.of(new IndexEvent().withAction(IndexEventAction.DELETE).withTableName(TEST_ENTITY_TABLE).withRecordId("7"));
|
||||
|
||||
List<QRecord> inserted = QuickSearchRecordChangeListener.insertFailedEventRows(runtime, events, "captured", QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
QuickSearchRecordChangeListener.handOverIfRebuildFinished(runtime, TEST_ENTITY_TABLE, inserted);
|
||||
|
||||
List<QRecord> rows = queryFailedEvents();
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadFails_recordsFailedEventsAfterCommit() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new QException("possible-value backend unavailable")))
|
||||
{
|
||||
assertThatCode(() -> new QuickSearchRecordChangeListener().onRecordsChanged(updateEvent(transaction, 7, 8))).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
assertThat(queryFailedEvents()).isEmpty();
|
||||
transaction.commit();
|
||||
|
||||
List<QRecord> failed = queryFailedEvents();
|
||||
assertThat(failed).extracting(r -> r.getValueString("recordId")).containsExactlyInAnyOrder("7", "8");
|
||||
assertThat(failed).allSatisfy(r ->
|
||||
{
|
||||
assertThat(r.getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE);
|
||||
assertThat(r.getValueString("action")).isEqualTo("INDEX");
|
||||
assertThat(r.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
|
||||
assertThat(r.getValueString("errorMessage")).contains("possible-value backend unavailable");
|
||||
});
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadFailsThenRollback_logsCauseAndRecordsNothing() throws QException
|
||||
{
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new QException("possible-value backend unavailable")))
|
||||
{
|
||||
new QuickSearchRecordChangeListener().onRecordsChanged(updateEvent(transaction, 7, 8));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
}
|
||||
transaction.rollback();
|
||||
|
||||
assertThat(queryFailedEvents()).isEmpty();
|
||||
assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains("Could not re-read updated records").contains(TEST_ENTITY_TABLE).contains("\"recordCount\":2").doesNotContain("changed");
|
||||
assertThat(m.getMessage()).contains("possible-value backend unavailable");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadFailsLinkage_recordsFailedEvents() throws QException
|
||||
{
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new NoClassDefFoundError("com/example/pvs/Provider")))
|
||||
{
|
||||
assertThatCode(() -> new QuickSearchRecordChangeListener().onRecordsChanged(updateEvent(null, 7))).doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
List<QRecord> failed = queryFailedEvents();
|
||||
assertThat(failed).extracting(r -> r.getValueString("recordId")).containsExactly("7");
|
||||
assertThat(failed.get(0).getValueString("errorMessage")).contains("com/example/pvs/Provider");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadFailsWithoutTransaction_recordsImmediately() throws QException
|
||||
{
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new QException("possible-value backend unavailable")))
|
||||
{
|
||||
new QuickSearchRecordChangeListener().onRecordsChanged(updateEvent(null, 7));
|
||||
}
|
||||
|
||||
assertThat(queryFailedEvents()).extracting(r -> r.getValueString("recordId")).containsExactly("7");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadAndRecordingBothFail_logsAndDoesNotThrow()
|
||||
{
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
try(MockedConstruction<QueryAction> ignoredQueries = failingQueries(new QException("possible-value backend unavailable"));
|
||||
MockedConstruction<InsertAction> ignoredInserts = mockConstruction(InsertAction.class, (mock, context) ->
|
||||
when(mock.execute(any(InsertInput.class))).thenThrow(new QException("failed-event table unavailable"))))
|
||||
{
|
||||
assertThatCode(() -> new QuickSearchRecordChangeListener().onRecordsChanged(updateEvent(null, 7))).doesNotThrowAnyException();
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
}
|
||||
|
||||
assertThat(collectingLogger.getCollectedMessages())
|
||||
.anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains("Could not re-read updated records").contains(TEST_ENTITY_TABLE).doesNotContain("changed");
|
||||
})
|
||||
.anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.ERROR);
|
||||
assertThat(m.getMessage()).contains("Could not record failed index events");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testClientBuildFailsLinkage_recordsFailedEvents() throws QException
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// no host publisher: the first write builds the client lazily, //
|
||||
// and a missing optional library fails linkage there //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
runtime.getConfig().setIndexEventPublisher(null);
|
||||
runtime.getConfig().setTransportCustomizer(new QCodeReference(MissingLibraryTransportCustomizer.class));
|
||||
|
||||
Integer id = insert("widget", null);
|
||||
|
||||
List<QRecord> failed = queryFailedEvents();
|
||||
assertThat(failed).hasSize(1);
|
||||
assertThat(failed.get(0).getValueString("recordId")).isEqualTo(String.valueOf(id));
|
||||
assertThat(failed.get(0).getValueString("action")).isEqualTo("INDEX");
|
||||
assertThat(failed.get(0).getValueString("errorMessage")).contains(MissingLibraryTransportCustomizer.MISSING_CLASS);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testPublishFailsLinkage_recordsFailedEvents() throws QException
|
||||
{
|
||||
doThrow(new NoClassDefFoundError("com/example/broker/Client")).when(publisher).publishDeleteEvents(anyList());
|
||||
Integer id = insert("widget", null);
|
||||
delete(id);
|
||||
|
||||
List<QRecord> failed = queryFailedEvents();
|
||||
assertThat(failed).hasSize(1);
|
||||
assertThat(failed.get(0).getValueString("recordId")).isEqualTo(String.valueOf(id));
|
||||
assertThat(failed.get(0).getValueString("action")).isEqualTo("DELETE");
|
||||
assertThat(failed.get(0).getValueString("errorMessage")).contains("com/example/broker/Client");
|
||||
}
|
||||
|
||||
|
||||
|
||||
private MockedConstruction<QueryAction> failingQueries(Throwable failure)
|
||||
{
|
||||
/////////////////////////////////////////////////////////
|
||||
// read the enabled flag first, so only the listener's //
|
||||
// re-read meets the failing QueryAction //
|
||||
/////////////////////////////////////////////////////////
|
||||
runtime.isTableEnabled(TEST_ENTITY_TABLE);
|
||||
return (mockConstruction(QueryAction.class, (mock, context) ->
|
||||
when(mock.execute(any(QueryInput.class))).thenThrow(failure)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
private RecordChangeEvent updateEvent(QBackendTransaction transaction, Integer... ids)
|
||||
{
|
||||
return (changeEvent(RecordChangeType.UPDATE, transaction, ids));
|
||||
}
|
||||
|
||||
|
||||
|
||||
private RecordChangeEvent changeEvent(RecordChangeType type, QBackendTransaction transaction, Integer... ids)
|
||||
{
|
||||
List<QRecord> records = new ArrayList<>();
|
||||
for(Integer id : ids)
|
||||
{
|
||||
records.add(new QRecord().withValue("id", id).withValue("description", "changed"));
|
||||
}
|
||||
return (new RecordChangeEvent().withTableName(TEST_ENTITY_TABLE).withType(type).withRecords(records).withTransaction(transaction));
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void delete(Integer id) throws QException
|
||||
{
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TEST_ENTITY_TABLE);
|
||||
deleteInput.setPrimaryKeys(List.of(id));
|
||||
new DeleteAction().execute(deleteInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void insertIndexRowWithStatus(String status) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("tableName", TEST_ENTITY_TABLE).withValue("enabled", true).withValue("status", status)));
|
||||
new InsertAction().execute(insertInput);
|
||||
runtime.invalidateEnabledCache();
|
||||
}
|
||||
|
||||
|
||||
|
||||
private List<QRecord> queryFailedEvents() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchFailedEvent.TABLE_NAME);
|
||||
return (new QueryAction().execute(queryInput).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDelete_rebuildEndsDuringPublish_isStillCaptured() throws QException
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// the reindex swaps the alias and clears REBUILDING while the delete is //
|
||||
// being published to the old index: the status read must come first //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
insertIndexRowWithStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
Integer id = insert("widget", null);
|
||||
doAnswer(invocation ->
|
||||
{
|
||||
setIndexRowStatus(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
return (null);
|
||||
}).when(publisher).publishDeleteEvents(anyList());
|
||||
|
||||
delete(id);
|
||||
|
||||
List<QRecord> rows = queryFailedEvents();
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).getValueString("recordId")).isEqualTo(String.valueOf(id));
|
||||
assertThat(rows.get(0).getValueString("action")).isEqualTo("DELETE");
|
||||
assertThat(rows.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void setIndexRowStatus(String status) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
QRecord row = new QueryAction().execute(queryInput).getRecords().get(0);
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id")).withValue("status", status)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
}
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.opensearch;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchTlsConfig;
|
||||
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.opensearch.client.transport.OpenSearchTransport;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit tests for OpenSearchTransportFactory: secret and path references that
|
||||
** resolve to nothing, and the hostname-verification warning.
|
||||
*******************************************************************************/
|
||||
class OpenSearchTransportFactoryTest
|
||||
{
|
||||
private static final String EMPTY_PROPERTY = "quickSearch.test.emptyValue";
|
||||
private static final String USER_PROPERTY = "quickSearch.test.userValue";
|
||||
private static final String PASSWORD_PROPERTY = "quickSearch.test.passwordValue";
|
||||
private static final String UNSET_ENV = "${env.QUICK_SEARCH_TEST_UNSET_VARIABLE_15}";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void afterEach()
|
||||
{
|
||||
System.clearProperty(EMPTY_PROPERTY);
|
||||
System.clearProperty(USER_PROPERTY);
|
||||
System.clearProperty(PASSWORD_PROPERTY);
|
||||
QLogger.deactivateCollectingLoggerForClass(OpenSearchTransportFactory.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QuickSearchQBitConfig baseConfig()
|
||||
{
|
||||
return (new QuickSearchQBitConfig()
|
||||
.withOpensearchUrl("https://localhost:9200")
|
||||
.withOpensearchIndexName("app-search"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testBasic_emptyPasswordReference_namesFieldNotSecrets()
|
||||
{
|
||||
System.setProperty(USER_PROPERTY, "secret-user-value");
|
||||
System.setProperty(EMPTY_PROPERTY, "");
|
||||
|
||||
QuickSearchQBitConfig config = baseConfig()
|
||||
.withAuthMode(QuickSearchAuthMode.BASIC)
|
||||
.withOpensearchUsername("${prop." + USER_PROPERTY + "}")
|
||||
.withOpensearchPassword("${prop." + EMPTY_PROPERTY + "}");
|
||||
|
||||
assertThatThrownBy(() -> OpenSearchTransportFactory.build(config))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("opensearchPassword")
|
||||
.hasMessageContaining("${prop." + EMPTY_PROPERTY + "}")
|
||||
.hasMessageNotContaining("secret-user-value");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testBasic_blankOrUnsetReference_isTreatedAsMissing()
|
||||
{
|
||||
System.setProperty(EMPTY_PROPERTY, " ");
|
||||
|
||||
QuickSearchQBitConfig blankPassword = baseConfig()
|
||||
.withAuthMode(QuickSearchAuthMode.BASIC)
|
||||
.withOpensearchUsername("admin")
|
||||
.withOpensearchPassword("${prop." + EMPTY_PROPERTY + "}");
|
||||
assertThatThrownBy(() -> OpenSearchTransportFactory.build(blankPassword))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("opensearchPassword");
|
||||
|
||||
QuickSearchQBitConfig unsetUsername = baseConfig()
|
||||
.withAuthMode(QuickSearchAuthMode.BASIC)
|
||||
.withOpensearchUsername(UNSET_ENV)
|
||||
.withOpensearchPassword("not-in-the-message");
|
||||
assertThatThrownBy(() -> OpenSearchTransportFactory.build(unsetUsername))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("opensearchUsername")
|
||||
.hasMessageContaining(UNSET_ENV)
|
||||
.hasMessageNotContaining("not-in-the-message");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testBasic_unresolvedFallbackChain_neverEchoesTheLiteral()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig()
|
||||
.withAuthMode(QuickSearchAuthMode.BASIC)
|
||||
.withOpensearchUsername("admin")
|
||||
.withOpensearchPassword(UNSET_ENV + "??fallback-secret-literal??${env.QUICK_SEARCH_TEST_UNSET_VARIABLE_15B}");
|
||||
|
||||
assertThatThrownBy(() -> OpenSearchTransportFactory.build(config))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("opensearchPassword")
|
||||
.hasMessageNotContaining("fallback-secret-literal");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testBasic_resolvedReferences_buildTransport() throws Exception
|
||||
{
|
||||
System.setProperty(USER_PROPERTY, "admin");
|
||||
System.setProperty(PASSWORD_PROPERTY, "admin-password");
|
||||
|
||||
QuickSearchQBitConfig config = baseConfig()
|
||||
.withAuthMode(QuickSearchAuthMode.BASIC)
|
||||
.withOpensearchUsername("${prop." + USER_PROPERTY + "}")
|
||||
.withOpensearchPassword("${prop." + PASSWORD_PROPERTY + "}");
|
||||
|
||||
try(OpenSearchTransport transport = OpenSearchTransportFactory.build(config))
|
||||
{
|
||||
assertThat(transport).isNotNull();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testTls_unresolvedPaths_nameTheField()
|
||||
{
|
||||
System.setProperty(EMPTY_PROPERTY, "");
|
||||
|
||||
assertPathError(new QuickSearchTlsConfig().withCaCertificatePath(UNSET_ENV), "tls.caCertificatePath", UNSET_ENV);
|
||||
assertPathError(new QuickSearchTlsConfig().withTrustStorePath(UNSET_ENV), "tls.trustStorePath", UNSET_ENV);
|
||||
assertPathError(new QuickSearchTlsConfig().withKeyStorePath("${prop." + EMPTY_PROPERTY + "}"), "tls.keyStorePath", "${prop." + EMPTY_PROPERTY + "}");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void assertPathError(QuickSearchTlsConfig tls, String fieldName, String reference)
|
||||
{
|
||||
assertThatThrownBy(() -> OpenSearchTransportFactory.build(baseConfig().withTls(tls)))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining(fieldName)
|
||||
.hasMessageContaining(reference)
|
||||
.hasMessageNotContaining(": null");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testHostnameVerificationDisabled_logsWarning()
|
||||
{
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(OpenSearchTransportFactory.class);
|
||||
|
||||
assertThat(OpenSearchTransportFactory.buildHostnameVerifier(new QuickSearchTlsConfig().withHostnameVerification(false))).isSameAs(NoopHostnameVerifier.INSTANCE);
|
||||
assertThat(collectingLogger.getCollectedMessages()).anyMatch(m -> Level.WARN.equals(m.getLevel()) && m.getMessage().contains("tls.hostnameVerification"));
|
||||
|
||||
collectingLogger.clear();
|
||||
assertThat(OpenSearchTransportFactory.buildHostnameVerifier(new QuickSearchTlsConfig())).isNull();
|
||||
assertThat(OpenSearchTransportFactory.buildHostnameVerifier(null)).isNull();
|
||||
assertThat(collectingLogger.getCollectedMessages()).isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
+48
-1
@@ -229,7 +229,7 @@ class QuickSearchOpenSearchClientTest
|
||||
|
||||
try
|
||||
{
|
||||
assertThatThrownBy(() -> client.search("hello", null, 10, 0, List.of()))
|
||||
assertThatThrownBy(() -> client.search("hello", List.of(), 10, 0, List.of()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("hello");
|
||||
}
|
||||
@@ -261,4 +261,51 @@ class QuickSearchOpenSearchClientTest
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** deleteDocuments with a null or empty list is a no-op, even with no server.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeleteDocuments_emptyList_returnsEmptyResult() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig());
|
||||
|
||||
try
|
||||
{
|
||||
BulkIndexResult emptyResult = client.deleteDocuments(List.of(), 100);
|
||||
BulkIndexResult nullResult = client.deleteDocuments(null, 100);
|
||||
|
||||
assertThat(emptyResult.getSuccessCount()).isEqualTo(0);
|
||||
assertThat(emptyResult.getFailureCount()).isEqualTo(0);
|
||||
assertThat(nullResult.getSuccessCount()).isEqualTo(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** deleteDocuments throws QException when the server is unavailable.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeleteDocuments_serverUnavailable_throwsQException() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig());
|
||||
|
||||
try
|
||||
{
|
||||
assertThatThrownBy(() -> client.deleteDocuments(List.of("orders:1", "orders:2"), 100))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("Bulk delete");
|
||||
}
|
||||
finally
|
||||
{
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+783
-177
File diff suppressed because it is too large.
Load diff
+570
-176
@@ -17,35 +17,54 @@
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.MockedConstruction;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockConstruction;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -55,53 +74,123 @@ import static org.mockito.Mockito.when;
|
||||
/*******************************************************************************
|
||||
** Tests for FullReindexStep.
|
||||
**
|
||||
** Verifies that:
|
||||
** - All tables are reindexed when no tableName input is provided
|
||||
** - Only the specified table is reindexed when tableName input is provided
|
||||
** - An empty table results in a run record with 0 records processed
|
||||
** - QuickSearchIndex rows are lazily created if they do not yet exist
|
||||
** - A failed indexDocuments() call results in a FAILED run record
|
||||
** - lastFullReindexTime is updated on the QuickSearchIndex row after reindex
|
||||
** Without a tableName input every enabled table is indexed into a new
|
||||
** physical index which then takes over the alias; with a tableName input the
|
||||
** table is rebuilt in place through the reconcile algorithm, never wiped.
|
||||
*******************************************************************************/
|
||||
class FullReindexStepTest extends BaseQuickSearchTest
|
||||
{
|
||||
private static final String PHYSICAL_INDEX = "test-index-v2-1700000000";
|
||||
|
||||
private QuickSearchOpenSearchClient mockClient;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper: insert source records into the testEntity table.
|
||||
** Install a mock client that reports every document as indexed, on both
|
||||
** the alias and the physical-index overloads.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUpClient() throws QException
|
||||
{
|
||||
mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.newPhysicalIndexName()).thenReturn(PHYSICAL_INDEX);
|
||||
when(mockClient.indexDocuments(anyString(), anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
List<?> docs = invocation.getArgument(1);
|
||||
return (new BulkIndexResult().withSuccessCount(docs.size()));
|
||||
});
|
||||
when(mockClient.indexDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
List<?> docs = invocation.getArgument(0);
|
||||
return (new BulkIndexResult().withSuccessCount(docs.size()));
|
||||
});
|
||||
when(mockClient.countDocumentsForTable(anyString())).thenReturn(0L);
|
||||
when(mockClient.deleteDocumentsIndexedBefore(anyString(), any())).thenReturn(0L);
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert count source records into the testEntity table.
|
||||
*******************************************************************************/
|
||||
private void insertTestEntities(int count) throws QException
|
||||
{
|
||||
List<QRecord> records = new java.util.ArrayList<>();
|
||||
List<QRecord> records = new ArrayList<>();
|
||||
for(int i = 1; i <= count; i++)
|
||||
{
|
||||
records.add(new QRecord()
|
||||
.withValue("name", "Entity " + i)
|
||||
.withValue("description", "Description " + i));
|
||||
records.add(new QRecord().withValue("name", "Entity " + i).withValue("description", "Description " + i));
|
||||
}
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TEST_ENTITY_TABLE);
|
||||
insertInput.setRecords(records);
|
||||
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper: build a RunBackendStepInput with no special values.
|
||||
** Insert a quickSearchIndex row for testEntity with the given enabled flag.
|
||||
*******************************************************************************/
|
||||
private RunBackendStepInput buildEmptyInput()
|
||||
private void insertIndexRow(Boolean enabled) throws QException
|
||||
{
|
||||
return (new RunBackendStepInput());
|
||||
insertIndexRow(enabled, null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper: build a RunBackendStepInput with a tableName filter.
|
||||
** Insert a quickSearchIndex row for testEntity with the given enabled flag
|
||||
** and documentCount.
|
||||
*******************************************************************************/
|
||||
private RunBackendStepInput buildInputWithTableName(String tableName)
|
||||
private void insertIndexRow(Boolean enabled, Integer documentCount) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord()
|
||||
.withValue("tableName", TEST_ENTITY_TABLE)
|
||||
.withValue("enabled", enabled)
|
||||
.withValue("basepullIntervalMinutes", 5)
|
||||
.withValue("documentCount", documentCount)
|
||||
.withValue("status", "ACTIVE")));
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Query all rows of a table.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryAll(String tableName) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
return (new QueryAction().execute(queryInput).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The quickSearchIndex row for testEntity.
|
||||
*******************************************************************************/
|
||||
private QRecord queryIndexRow() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, TEST_ENTITY_TABLE)));
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
return (rows.isEmpty() ? null : rows.get(0));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Input with a tableName filter.
|
||||
*******************************************************************************/
|
||||
private RunBackendStepInput inputForTable(String tableName)
|
||||
{
|
||||
RunBackendStepInput input = new RunBackendStepInput();
|
||||
input.addValue("tableName", tableName);
|
||||
@@ -111,250 +200,555 @@ class FullReindexStepTest extends BaseQuickSearchTest
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper: query all records from quickSearchIndexRun.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryAllRunRecords() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchIndexRun.TABLE_NAME);
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
return (queryOutput.getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper: query all records from quickSearchIndex.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryAllIndexRecords() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
return (queryOutput.getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper: query a quickSearchIndex record by tableName.
|
||||
*******************************************************************************/
|
||||
private QRecord queryIndexByTableName(String tableName) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
queryInput.setFilter(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
List<QRecord> records = queryOutput.getRecords();
|
||||
return (records.isEmpty() ? null : records.get(0));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: reindex all tables when no tableName input is given.
|
||||
**
|
||||
** With one discovered table (testEntity) and source records present,
|
||||
** deleteDocumentsForTable() and indexDocuments() should each be called once.
|
||||
** Test: without a tableName input, documents go into a fresh physical
|
||||
** index which is refreshed and then takes over the alias; the live index
|
||||
** is never wiped.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReindexAllTables_noTableNameInput_reindexesAllDiscoveredTables() throws QException
|
||||
void testAllTables_indexesIntoNewPhysicalIndexThenSwapsAlias() throws QException
|
||||
{
|
||||
insertTestEntities(3);
|
||||
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(3));
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildEmptyInput(), new RunBackendStepOutput());
|
||||
InOrder inOrder = inOrder(mockClient);
|
||||
inOrder.verify(mockClient).createPhysicalIndex(PHYSICAL_INDEX);
|
||||
inOrder.verify(mockClient).indexDocuments(eq(PHYSICAL_INDEX), anyList(), anyInt());
|
||||
inOrder.verify(mockClient).refreshIndex(PHYSICAL_INDEX);
|
||||
inOrder.verify(mockClient).swapAliasTo(PHYSICAL_INDEX);
|
||||
|
||||
verify(mockClient, times(1)).deleteDocumentsForTable(TEST_ENTITY_TABLE);
|
||||
verify(mockClient, times(1)).indexDocuments(any(), anyInt());
|
||||
verify(mockClient, never()).deleteDocumentsForTable(anyString());
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
verify(mockClient, never()).deletePhysicalIndex(anyString());
|
||||
|
||||
assertThat(output.getValueInteger("recordsIndexed")).isEqualTo(3);
|
||||
assertThat(output.getValueBoolean("aliasSwapped")).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: reindex only the specified table when tableName input is provided.
|
||||
**
|
||||
** Registers two discovered tables, passes tableName=testEntity.
|
||||
** Only testEntity should be reindexed; the other table should be skipped.
|
||||
** Test: the documents built from the source records are the ones sent to
|
||||
** the physical index, and the run record counts them.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReindexSingleTable_tableNameInputProvided_onlyThatTableReindexed() throws QException
|
||||
@SuppressWarnings("unchecked")
|
||||
void testAllTables_documentsBuiltFromSourceRecords() throws QException
|
||||
{
|
||||
insertTestEntities(5);
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
ArgumentCaptor<List<OpenSearchDocument>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mockClient).indexDocuments(eq(PHYSICAL_INDEX), captor.capture(), anyInt());
|
||||
assertThat(captor.getValue()).extracting(OpenSearchDocument::getRecordId).containsExactly("1", "2", "3", "4", "5");
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("runType")).isEqualTo("FULL_REINDEX");
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("COMPLETED");
|
||||
assertThat(runs.get(0).getValueInteger("recordsProcessed")).isEqualTo(5);
|
||||
assertThat(runs.get(0).getValueInteger("recordsIndexed")).isEqualTo(5);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the index row is updated after the swap.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_updatesIndexRowAfterSwap() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenReturn(2L);
|
||||
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(2));
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildInputWithTableName(TEST_ENTITY_TABLE), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, times(1)).deleteDocumentsForTable(TEST_ENTITY_TABLE);
|
||||
verify(mockClient, never()).deleteDocumentsForTable("someOtherTable");
|
||||
QRecord row = queryIndexRow();
|
||||
assertThat(row).isNotNull();
|
||||
assertThat(row.getValue("lastFullReindexTime")).isNotNull();
|
||||
assertThat(row.getValue("lastBasepullTime")).isNotNull();
|
||||
assertThat(row.getValueInteger("recordCount")).isEqualTo(2);
|
||||
assertThat(row.getValueInteger("documentCount")).isEqualTo(2);
|
||||
assertThat(row.getValueString("status")).isEqualTo("ACTIVE");
|
||||
assertThat(row.getValueString("lastRunStatus")).isEqualTo("FULL_REINDEX COMPLETED");
|
||||
assertThat(row.getValueString("searchableFieldsJson")).isEqualTo(AbstractIndexingStep.buildSearchableFieldsJson(QuickSearchQBitContext.getTableConfig(TEST_ENTITY_TABLE)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: empty source table produces a run record with 0 records processed.
|
||||
** Test: a failing document count after the swap is logged at WARN with the
|
||||
** table name, and the run still completes, keeping the previous
|
||||
** documentCount.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testEmptyTable_noSourceRecords_runRecordCreatedWithZeroProcessed() throws QException
|
||||
void testAllTables_documentCountFails_loggedAsWarning_previousCountKept() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult());
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(true, 99);
|
||||
when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenThrow(new QException("count failed"));
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildEmptyInput(), new RunBackendStepOutput());
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ReconcileIndexStep.class);
|
||||
try
|
||||
{
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(ReconcileIndexStep.class);
|
||||
}
|
||||
|
||||
List<QRecord> runRecords = queryAllRunRecords();
|
||||
assertThat(runRecords).hasSize(1);
|
||||
assertThat(collectingLogger.getCollectedMessages())
|
||||
.filteredOn(m -> Level.WARN.equals(m.getLevel()) && m.getMessage().contains("Could not count indexed documents"))
|
||||
.singleElement()
|
||||
.satisfies(m ->
|
||||
{
|
||||
assertThat(m.getMessage()).contains("\"tableName\":\"" + TEST_ENTITY_TABLE + "\"");
|
||||
assertThat(m.getMessage()).contains("\"stackTrace\"").contains("count failed");
|
||||
});
|
||||
|
||||
QRecord runRecord = runRecords.get(0);
|
||||
assertThat(runRecord.getValueString("status")).isEqualTo("COMPLETED");
|
||||
assertThat(runRecord.getValueInteger("recordsProcessed")).isEqualTo(0);
|
||||
assertThat(runRecord.getValueInteger("recordsIndexed")).isEqualTo(0);
|
||||
|
||||
verify(mockClient, times(1)).deleteDocumentsForTable(TEST_ENTITY_TABLE);
|
||||
verify(mockClient, never()).indexDocuments(any(), anyInt());
|
||||
QRecord row = queryIndexRow();
|
||||
assertThat(row.getValueString("lastRunStatus")).isEqualTo("FULL_REINDEX COMPLETED");
|
||||
assertThat(row.getValueInteger("recordCount")).isEqualTo(1);
|
||||
assertThat(row.getValueInteger("documentCount")).isEqualTo(99);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: QuickSearchIndex row is lazily created if it does not exist yet.
|
||||
**
|
||||
** No pre-existing index rows. After run(), a quickSearchIndex row for
|
||||
** testEntity should be present.
|
||||
** Test: an empty source table still produces a new physical index, a
|
||||
** swap, and a COMPLETED run with zero counts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_emptyTable_swapsWithZeroCounts() throws QException
|
||||
{
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient).createPhysicalIndex(PHYSICAL_INDEX);
|
||||
verify(mockClient).swapAliasTo(PHYSICAL_INDEX);
|
||||
verify(mockClient, never()).indexDocuments(anyString(), anyList(), anyInt());
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("COMPLETED");
|
||||
assertThat(runs.get(0).getValueInteger("recordsProcessed")).isEqualTo(0);
|
||||
assertThat(runs.get(0).getValueInteger("recordsIndexed")).isEqualTo(0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a missing quickSearchIndex row is created before the reindex.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLazyIndexRowCreation_noExistingIndexRow_createdBeforeReindex() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult());
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
assertThat(queryAll(QuickSearchIndex.TABLE_NAME)).isEmpty();
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Verify no rows exist before run //
|
||||
////////////////////////////////////////////////////
|
||||
assertThat(queryAllIndexRecords()).isEmpty();
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildEmptyInput(), new RunBackendStepOutput());
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// Verify index row was created //
|
||||
////////////////////////////////////////////////////
|
||||
List<QRecord> indexRecords = queryAllIndexRecords();
|
||||
assertThat(indexRecords).hasSize(1);
|
||||
assertThat(indexRecords.get(0).getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE);
|
||||
List<QRecord> rows = queryAll(QuickSearchIndex.TABLE_NAME);
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when indexDocuments() throws, the run record is marked FAILED.
|
||||
** Test: when the client throws while indexing, the new physical index is
|
||||
** deleted, the alias is never swapped, the run is FAILED with the error
|
||||
** message, and the step throws.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testErrorDuringIndexing_mockClientThrows_failedRunRecord() throws QException
|
||||
void testAllTables_clientThrows_deletesPhysicalIndexAndNeverSwaps() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
when(mockClient.indexDocuments(anyString(), anyList(), anyInt())).thenThrow(new QException("OpenSearch connection refused"));
|
||||
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt()))
|
||||
.thenThrow(new QException("OpenSearch connection refused"));
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
assertThatThrownBy(() -> step.run(buildEmptyInput(), new RunBackendStepOutput()))
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("Full reindex failed");
|
||||
.hasMessageContaining("OpenSearch connection refused");
|
||||
|
||||
List<QRecord> runRecords = queryAllRunRecords();
|
||||
assertThat(runRecords).hasSize(1);
|
||||
assertThat(runRecords.get(0).getValueString("status")).isEqualTo("FAILED");
|
||||
assertThat(runRecords.get(0).getValueString("errorMessage")).contains("OpenSearch connection refused");
|
||||
verify(mockClient).deletePhysicalIndex(PHYSICAL_INDEX);
|
||||
verify(mockClient, never()).swapAliasTo(anyString());
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("FAILED");
|
||||
assertThat(runs.get(0).getValueString("errorMessage")).contains("OpenSearch connection refused");
|
||||
|
||||
QRecord row = queryIndexRow();
|
||||
assertThat(row.getValueString("lastRunStatus")).isEqualTo("FULL_REINDEX FAILED");
|
||||
assertThat(row.getValueString("lastErrorMessage")).contains("OpenSearch connection refused");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: lastFullReindexTime is updated on the QuickSearchIndex row after reindex.
|
||||
** Test: per-document bulk failures also abort the rebuild, with the first
|
||||
** item error on the run record.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUpdatesLastFullReindexTime_afterReindex_fieldIsSet() throws QException
|
||||
void testAllTables_bulkItemFailures_runFailedWithErrorMessage() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
BulkIndexResult partial = new BulkIndexResult();
|
||||
partial.addSuccess();
|
||||
partial.addFailure("testEntity:2: mapper_parsing_exception");
|
||||
when(mockClient.indexDocuments(anyString(), anyList(), anyInt())).thenReturn(partial);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("mapper_parsing_exception");
|
||||
|
||||
verify(mockClient).deletePhysicalIndex(PHYSICAL_INDEX);
|
||||
verify(mockClient, never()).swapAliasTo(anyString());
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("FAILED");
|
||||
assertThat(runs.get(0).getValueInteger("errorCount")).isEqualTo(1);
|
||||
assertThat(runs.get(0).getValueString("errorMessage")).contains("mapper_parsing_exception");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a failure to delete the incomplete physical index does not hide
|
||||
** the original error.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_cleanupFails_originalErrorStillThrown() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
when(mockClient.indexDocuments(anyString(), anyList(), anyInt())).thenThrow(new QException("bulk failed"));
|
||||
doThrow(new QException("delete failed")).when(mockClient).deletePhysicalIndex(PHYSICAL_INDEX);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("bulk failed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with a tableName input the table is rebuilt in place through the
|
||||
** reconcile algorithm: alias writes, refresh, stale delete, no wipe, no
|
||||
** physical index, no swap.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSingleTable_reconcilesInPlace_neverWipes() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(2));
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep().run(inputForTable(TEST_ENTITY_TABLE), output);
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildEmptyInput(), new RunBackendStepOutput());
|
||||
InOrder inOrder = inOrder(mockClient);
|
||||
inOrder.verify(mockClient).indexDocuments(anyList(), anyInt());
|
||||
inOrder.verify(mockClient).refreshIndex();
|
||||
inOrder.verify(mockClient).deleteDocumentsIndexedBefore(eq(TEST_ENTITY_TABLE), any());
|
||||
|
||||
QRecord indexRecord = queryIndexByTableName(TEST_ENTITY_TABLE);
|
||||
assertThat(indexRecord).isNotNull();
|
||||
assertThat(indexRecord.getValue("lastFullReindexTime")).isNotNull();
|
||||
assertThat(indexRecord.getValueInteger("recordCount")).isEqualTo(2);
|
||||
verify(mockClient, never()).deleteDocumentsForTable(anyString());
|
||||
verify(mockClient, never()).createPhysicalIndex(anyString());
|
||||
verify(mockClient, never()).swapAliasTo(anyString());
|
||||
verify(mockClient, never()).indexDocuments(anyString(), anyList(), anyInt());
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("runType")).isEqualTo("FULL_REINDEX");
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("COMPLETED");
|
||||
assertThat(runs.get(0).getValueInteger("recordsProcessed")).isEqualTo(2);
|
||||
|
||||
assertThat(queryIndexRow().getValue("lastFullReindexTime")).isNotNull();
|
||||
assertThat(output.getValueInteger("recordsIndexed")).isEqualTo(2);
|
||||
assertThat(output.getValueBoolean("aliasSwapped")).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: run record has correct runType = "FULL_REINDEX".
|
||||
** Test: a tableName input naming an unknown table reindexes nothing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRunRecord_runTypeIsFullReindex() throws QException
|
||||
void testSingleTable_unknownTableName_nothingReindexed() throws QException
|
||||
{
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult());
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
insertTestEntities(1);
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildEmptyInput(), new RunBackendStepOutput());
|
||||
new FullReindexStep().run(inputForTable("someOtherTable"), new RunBackendStepOutput());
|
||||
|
||||
List<QRecord> runRecords = queryAllRunRecords();
|
||||
assertThat(runRecords).hasSize(1);
|
||||
assertThat(runRecords.get(0).getValueString("runType")).isEqualTo("FULL_REINDEX");
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
verify(mockClient, never()).createPhysicalIndex(anyString());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: documents are built and passed to indexDocuments with source records.
|
||||
**
|
||||
** Verifies that the correct number of documents are submitted to the client.
|
||||
** Test: a disabled index row is skipped by the all-tables rebuild; the
|
||||
** swap still happens for the (empty) set of enabled tables.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testIndexDocuments_documentsBuiltFromSourceRecords() throws QException
|
||||
void testDisabledRow_skippedByAllTablesReindex() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
insertIndexRow(false);
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, never()).indexDocuments(anyString(), anyList(), anyInt());
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a disabled index row is skipped by the single-table reindex too.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDisabledRow_skippedBySingleTableReindex() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
insertIndexRow(false);
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep().run(inputForTable(TEST_ENTITY_TABLE), output);
|
||||
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
verify(mockClient, never()).deleteDocumentsIndexedBefore(anyString(), any());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).isEmpty();
|
||||
assertThat(output.getValueInteger("recordsIndexed")).isEqualTo(0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: source records are paged by primary key into the physical index.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_pagesByPrimaryKey() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(2);
|
||||
insertTestEntities(5);
|
||||
|
||||
QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
ArgumentCaptor<List<OpenSearchDocument>> docsCaptor = ArgumentCaptor.forClass(List.class);
|
||||
when(mockClient.indexDocuments(docsCaptor.capture(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(5));
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
FullReindexStep step = new FullReindexStep();
|
||||
step.run(buildEmptyInput(), new RunBackendStepOutput());
|
||||
verify(mockClient, times(3)).indexDocuments(eq(PHYSICAL_INDEX), anyList(), anyInt());
|
||||
}
|
||||
|
||||
assertThat(docsCaptor.getValue()).hasSize(5);
|
||||
|
||||
List<QRecord> runRecords = queryAllRunRecords();
|
||||
assertThat(runRecords).hasSize(1);
|
||||
assertThat(runRecords.get(0).getValueInteger("recordsProcessed")).isEqualTo(5);
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the index row reads REBUILDING while tables are read, and a delete
|
||||
** captured during the rebuild is applied through the alias after the swap,
|
||||
** then its failed-event row is removed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_appliesCapturedDeletesAfterTheSwap() throws QException
|
||||
{
|
||||
insertTestEntities(3);
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(1));
|
||||
|
||||
List<String> statusesDuringRebuild = new ArrayList<>();
|
||||
new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex);
|
||||
statusesDuringRebuild.add(FullReindexStepTest.this.queryIndexRow().getValueString("status"));
|
||||
insertCapturedDelete("7");
|
||||
return (counts);
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
assertThat(statusesDuringRebuild).containsExactly(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
|
||||
InOrder inOrder = inOrder(mockClient);
|
||||
inOrder.verify(mockClient).swapAliasTo(PHYSICAL_INDEX);
|
||||
inOrder.verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":7")), anyInt());
|
||||
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a captured delete the cluster rejects is handed to basepull.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_capturedDeleteFailure_becomesPendingForBasepull() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("9");
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenThrow(new QException("cluster down"));
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
List<QRecord> rows = queryAll(QuickSearchFailedEvent.TABLE_NAME);
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
|
||||
assertThat(rows.get(0).getValueString("errorMessage")).contains("cluster down");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when the rebuild fails before the swap, the previous status comes
|
||||
** back and the captured deletes still leave the AWAITING_REINDEX state.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_failureBeforeSwap_restoresStatusAndDrainsCapturedDeletes() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(true);
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_NEEDS_REINDEX);
|
||||
insertCapturedDelete("4");
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(1));
|
||||
doThrow(new QException("swap refused")).when(mockClient).swapAliasTo(PHYSICAL_INDEX);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class);
|
||||
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_NEEDS_REINDEX);
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":4")), anyInt());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert an AWAITING_REINDEX delete for testEntity, as the listener does
|
||||
** during a full reindex.
|
||||
*******************************************************************************/
|
||||
private void insertCapturedDelete(String recordId) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchFailedEvent.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord()
|
||||
.withValue("tableName", TEST_ENTITY_TABLE)
|
||||
.withValue("recordId", recordId)
|
||||
.withValue("action", "DELETE")
|
||||
.withValue("attempts", 0)
|
||||
.withValue("status", QuickSearchFailedEvent.STATUS_AWAITING_REINDEX)));
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the status on the testEntity quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
private void updateIndexRowStatus(String status) throws QException
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", queryIndexRow().getValue("id")).withValue("status", status)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a captured delete for a record re-created with the same key since
|
||||
** is not applied (its document stays), and its row is removed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_capturedDeleteOfRecreatedRecord_isNotApplied() throws QException
|
||||
{
|
||||
insertTestEntities(3);
|
||||
insertCapturedDelete("2");
|
||||
insertCapturedDelete("9");
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(1));
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, times(1)).deleteDocuments(anyList(), anyInt());
|
||||
verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":9")), anyInt());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: rows the delete action reports as not removed (per-row errors, not
|
||||
** an exception) are not read again; the drain moves on and ends.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_capturedRowsNotRemoved_drainStillEnds() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(1);
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("8");
|
||||
insertCapturedDelete("9");
|
||||
|
||||
AtomicInteger deleteCalls = new AtomicInteger(0);
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// fail loudly instead of hanging if the same batch is re-read //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
if(deleteCalls.incrementAndGet() > 10)
|
||||
{
|
||||
throw (new IllegalStateException("drain re-read the same rows"));
|
||||
}
|
||||
return (new BulkIndexResult().withSuccessCount(1));
|
||||
});
|
||||
|
||||
try(MockedConstruction<DeleteAction> deleteActions = mockConstruction(DeleteAction.class, (deleteAction, context) ->
|
||||
{
|
||||
DeleteOutput output = new DeleteOutput();
|
||||
output.setRecordsWithErrors(List.of(new QRecord().withValue("id", 1).withError(new SystemErrorStatusMessage("row is locked"))));
|
||||
when(deleteAction.execute(any())).thenReturn(output);
|
||||
}))
|
||||
{
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
}
|
||||
|
||||
assertThat(deleteCalls.get()).isEqualTo(2);
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).extracting(row -> row.getValueString("status"))
|
||||
.containsOnly(QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a batch that fails before it is applied (here, the source lookup)
|
||||
** stays AWAITING_REINDEX, and the later batches are still applied.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_failedCapturedBatch_laterBatchesStillApplied() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(1);
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("8");
|
||||
insertCapturedDelete("9");
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(1));
|
||||
|
||||
new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
|
||||
{
|
||||
if("8".equals(rows.get(0).getValueString("recordId")))
|
||||
{
|
||||
throw (new QException("source table unavailable"));
|
||||
}
|
||||
return (super.queryExistingRecordIds(tableName, rows));
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, times(1)).deleteDocuments(anyList(), anyInt());
|
||||
verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":9")), anyInt());
|
||||
|
||||
List<QRecord> rows = queryAll(QuickSearchFailedEvent.TABLE_NAME);
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).getValueString("recordId")).isEqualTo("8");
|
||||
assertThat(rows.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,12 +17,24 @@
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -34,12 +46,13 @@ class IndexingUtilsTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that normal input is lowercased and trimmed.
|
||||
** Test that normal input is trimmed and its case is preserved (the analyzer
|
||||
** lower-cases at index and search time).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNormalizeSearchText_normalInput_lowercaseTrimmed()
|
||||
void testNormalizeSearchText_normalInput_trimmedCasePreserved()
|
||||
{
|
||||
assertThat(IndexingUtils.normalizeSearchText("Hello World")).isEqualTo("hello world");
|
||||
assertThat(IndexingUtils.normalizeSearchText(" Hello World ")).isEqualTo("Hello World");
|
||||
}
|
||||
|
||||
|
||||
@@ -84,12 +97,12 @@ class IndexingUtilsTest
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test that mixed case input is fully lowercased.
|
||||
** Test that mixed case input is not lower-cased.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNormalizeSearchText_mixedCase_lowercased()
|
||||
void testNormalizeSearchText_mixedCase_preserved()
|
||||
{
|
||||
assertThat(IndexingUtils.normalizeSearchText("HeLLo WoRLd")).isEqualTo("hello world");
|
||||
assertThat(IndexingUtils.normalizeSearchText("HeLLo WoRLd")).isEqualTo("HeLLo WoRLd");
|
||||
}
|
||||
|
||||
|
||||
@@ -270,7 +283,7 @@ class IndexingUtilsTest
|
||||
|
||||
/*******************************************************************************
|
||||
** Test the new buildDocument overload with QuickSearchableTableConfig that
|
||||
** has a recordLabelFormat -- verifies the formatted label is applied.
|
||||
** has a recordLabelFormat; verifies the formatted label is applied.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_tableConfig_withRecordLabelFormat_usesFormattedLabel()
|
||||
@@ -302,7 +315,7 @@ class IndexingUtilsTest
|
||||
|
||||
/*******************************************************************************
|
||||
** Test the new buildDocument overload with QuickSearchableTableConfig that
|
||||
** does NOT have a recordLabelFormat -- falls back to QRecord.getRecordLabel().
|
||||
** does NOT have a recordLabelFormat; falls back to QRecord.getRecordLabel().
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_tableConfig_withoutRecordLabelFormat_usesQRecordLabel()
|
||||
@@ -378,4 +391,239 @@ class IndexingUtilsTest
|
||||
assertThat(doc).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clear any QContext a test installed.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void clearContext()
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Install a QInstance holding one "person" table with a record label format
|
||||
** and a labelled "sku" field.
|
||||
*******************************************************************************/
|
||||
private void initContextWithPersonTable()
|
||||
{
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName("person")
|
||||
.withBackendName("memory")
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s %s")
|
||||
.withRecordLabelFields(List.of("firstName", "lastName"))
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("firstName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("lastName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("sku", QFieldType.STRING).withLabel("SKU Code"))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a display value (possible-value label, formatted date) wins over
|
||||
** the raw value, both in fieldValues and in the searchable text.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_displayValuePresent_usedInsteadOfRawValue()
|
||||
{
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("statusId", 3);
|
||||
record.setDisplayValue("statusId", "Shipped");
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
.withTableName("order")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("statusId"));
|
||||
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
|
||||
assertThat(doc.getFieldValues().get("statusId")).isEqualTo("Shipped");
|
||||
assertThat(doc.getSearchableText()).isEqualTo("Shipped");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: non-string values are stored as strings in fieldValues, since the
|
||||
** mapping types every fieldValues.* entry as text.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_integerValue_storedAsString()
|
||||
{
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("quantity", 42);
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
.withTableName("order")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("quantity"));
|
||||
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
|
||||
assertThat(doc.getFieldValues().get("quantity")).isEqualTo("42");
|
||||
assertThat(doc.getSearchableText()).isEqualTo("42");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: values longer than maxFieldLength are truncated.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_valueLongerThanMaxFieldLength_truncated()
|
||||
{
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("notes", "abcdefghijklmnop");
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
.withTableName("order")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("notes"))
|
||||
.withMaxFieldLength(5);
|
||||
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
|
||||
assertThat(doc.getFieldValues().get("notes")).isEqualTo("abcde");
|
||||
assertThat(doc.getSearchableText()).isEqualTo("abcde");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a record without a label gets one from the table's record label
|
||||
** format when the table is known to the QInstance in QContext.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_noRecordLabel_fallsBackToTableRecordLabelFormat()
|
||||
{
|
||||
initContextWithPersonTable();
|
||||
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("firstName", "Ada");
|
||||
record.setValue("lastName", "Lovelace");
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
.withTableName("person")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("firstName"));
|
||||
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
|
||||
assertThat(doc.getRecordLabel()).isEqualTo("Ada Lovelace");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a recordLabelFormat that cannot be applied to the values never
|
||||
** throws; the default label is kept.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_badRecordLabelFormat_fallsBackToDefaultLabel()
|
||||
{
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("firstName", "Ada");
|
||||
record.setRecordLabel("Default Label");
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
.withTableName("person")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("firstName"))
|
||||
.withRecordLabelFormat("%d-%s")
|
||||
.withRecordLabelFields(List.of("firstName", "lastName"));
|
||||
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
|
||||
assertThat(doc).isNotNull();
|
||||
assertThat(doc.getRecordLabel()).isEqualTo("Default Label");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the document version is the basepull timestamp in epoch millis,
|
||||
** and null when the record has no timestamp or the table has no field.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildDocument_version_fromBasepullTimestampField()
|
||||
{
|
||||
Instant modifyDate = Instant.parse("2026-10-04T12:34:56.789Z");
|
||||
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("name", "Widget");
|
||||
record.setValue("modifyDate", modifyDate);
|
||||
|
||||
QuickSearchableTableConfig withTimestamp = new QuickSearchableTableConfig()
|
||||
.withTableName("product")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("name"))
|
||||
.withBasepullTimestampField("modifyDate");
|
||||
|
||||
assertThat(IndexingUtils.buildDocument(record, withTimestamp).getVersion()).isEqualTo(modifyDate.toEpochMilli());
|
||||
|
||||
QRecord withoutValue = new QRecord().withValue("id", 2).withValue("name", "Gadget");
|
||||
assertThat(IndexingUtils.buildDocument(withoutValue, withTimestamp).getVersion()).isNull();
|
||||
|
||||
QuickSearchableTableConfig withoutTimestamp = new QuickSearchableTableConfig()
|
||||
.withTableName("product")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("name"));
|
||||
assertThat(IndexingUtils.buildDocument(record, withoutTimestamp).getVersion()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the label prefix uses the QQQ field label when the table is known,
|
||||
** and the field name otherwise.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildSearchableText_includeLabel_usesFieldLabelWhenTableKnown()
|
||||
{
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("id", 1);
|
||||
record.setValue("sku", "AB-1");
|
||||
|
||||
QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig()
|
||||
.withTableName("person")
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("sku"))
|
||||
.withFieldIncludeLabels(Map.of("sku", true));
|
||||
|
||||
assertThat(IndexingUtils.buildDocument(record, tableConfig).getSearchableText()).isEqualTo("sku: AB-1");
|
||||
|
||||
initContextWithPersonTable();
|
||||
assertThat(IndexingUtils.buildDocument(record, tableConfig).getSearchableText()).isEqualTo("SKU Code: AB-1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a null includeLabels map is tolerated.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBuildSearchableText_nullIncludeLabels_doesNotThrow()
|
||||
{
|
||||
QRecord record = new QRecord();
|
||||
record.setValue("name", "Widget");
|
||||
|
||||
assertThat(IndexingUtils.buildSearchableText(record, List.of("name"), null)).isEqualTo("Widget");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InOrder;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests for ReconcileIndexStep.
|
||||
*******************************************************************************/
|
||||
class ReconcileIndexStepTest extends BaseQuickSearchTest
|
||||
{
|
||||
private QuickSearchOpenSearchClient mockClient;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Install a mock client that reports every document as indexed.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUpClient() throws QException
|
||||
{
|
||||
mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
when(mockClient.indexDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
List<?> docs = invocation.getArgument(0);
|
||||
return (new BulkIndexResult().withSuccessCount(docs.size()));
|
||||
});
|
||||
when(mockClient.deleteDocumentsIndexedBefore(anyString(), any())).thenReturn(0L);
|
||||
QuickSearchQBitContext.setClient(mockClient);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert count source records into the testEntity table.
|
||||
*******************************************************************************/
|
||||
private void insertTestEntities(int count) throws QException
|
||||
{
|
||||
List<QRecord> records = new ArrayList<>();
|
||||
for(int i = 1; i <= count; i++)
|
||||
{
|
||||
records.add(new QRecord().withValue("name", "Entity " + i).withValue("description", "Description " + i));
|
||||
}
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TEST_ENTITY_TABLE);
|
||||
insertInput.setRecords(records);
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Query all rows of the given table.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryAll(String tableName) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter());
|
||||
return (new QueryAction().execute(queryInput).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Collect the record IDs of every document passed to indexDocuments.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private List<String> indexedRecordIds() throws QException
|
||||
{
|
||||
ArgumentCaptor<List<OpenSearchDocument>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(mockClient, atLeastOnce()).indexDocuments(captor.capture(), anyInt());
|
||||
|
||||
List<String> recordIds = new ArrayList<>();
|
||||
for(List<OpenSearchDocument> batch : captor.getAllValues())
|
||||
{
|
||||
batch.forEach(doc -> recordIds.add(doc.getRecordId()));
|
||||
}
|
||||
return (recordIds);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: every source record is re-indexed, then documents not re-indexed by
|
||||
** this run are removed, after a refresh. The table is never wiped first.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_reindexesSourceThenRemovesStaleDocuments() throws QException
|
||||
{
|
||||
insertTestEntities(3);
|
||||
when(mockClient.deleteDocumentsIndexedBefore(anyString(), any())).thenReturn(2L);
|
||||
|
||||
Instant beforeRun = Instant.now();
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), output);
|
||||
Instant afterRun = Instant.now();
|
||||
|
||||
assertThat(indexedRecordIds()).containsExactlyInAnyOrder("1", "2", "3");
|
||||
|
||||
ArgumentCaptor<Instant> cutoffCaptor = ArgumentCaptor.forClass(Instant.class);
|
||||
InOrder inOrder = inOrder(mockClient);
|
||||
inOrder.verify(mockClient).indexDocuments(anyList(), anyInt());
|
||||
inOrder.verify(mockClient).refreshIndex();
|
||||
inOrder.verify(mockClient).deleteDocumentsIndexedBefore(eq(TEST_ENTITY_TABLE), cutoffCaptor.capture());
|
||||
|
||||
assertThat(cutoffCaptor.getValue()).isBetween(beforeRun, afterRun);
|
||||
verify(mockClient, never()).deleteDocumentsForTable(anyString());
|
||||
|
||||
assertThat(output.getValueInteger("recordsIndexed")).isEqualTo(3);
|
||||
assertThat(output.getValue("documentsRemoved")).isEqualTo(2L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: source records are paged by primary key, so every record is read
|
||||
** exactly once across batches.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_pagesByPrimaryKey_eachRecordOnce() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(2);
|
||||
insertTestEntities(5);
|
||||
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, times(3)).indexDocuments(anyList(), anyInt());
|
||||
assertThat(indexedRecordIds()).containsExactly("1", "2", "3", "4", "5");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when any document fails to index, stale documents are not removed
|
||||
** (a document that failed to re-index would otherwise be purged), the run
|
||||
** is marked FAILED, and the step throws.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_indexingErrors_keepsDocumentsAndFailsRun() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
BulkIndexResult partial = new BulkIndexResult();
|
||||
partial.addSuccess();
|
||||
partial.addFailure("testEntity:2: mapper_parsing_exception");
|
||||
when(mockClient.indexDocuments(anyList(), anyInt())).thenReturn(partial);
|
||||
|
||||
assertThatThrownBy(() -> new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("mapper_parsing_exception");
|
||||
|
||||
verify(mockClient, never()).deleteDocumentsIndexedBefore(anyString(), any());
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("runType")).isEqualTo("RECONCILE");
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("FAILED");
|
||||
assertThat(runs.get(0).getValueInteger("errorCount")).isEqualTo(1);
|
||||
assertThat(runs.get(0).getValueString("errorMessage")).contains("mapper_parsing_exception");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a client exception fails the run record and is rethrown.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_clientThrows_failsRunAndThrows() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
when(mockClient.indexDocuments(anyList(), anyInt())).thenThrow(new QException("connection refused"));
|
||||
|
||||
assertThatThrownBy(() -> new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("connection refused");
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("FAILED");
|
||||
assertThat(runs.get(0).getValueString("errorMessage")).contains("connection refused");
|
||||
verify(mockClient, never()).deleteDocumentsIndexedBefore(anyString(), any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a successful run completes its run record and updates the index row.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_success_completesRunAndUpdatesIndexRow() throws QException
|
||||
{
|
||||
insertTestEntities(4);
|
||||
when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenReturn(4L);
|
||||
|
||||
Instant beforeRun = Instant.now();
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
List<QRecord> runs = queryAll(QuickSearchIndexRun.TABLE_NAME);
|
||||
assertThat(runs).hasSize(1);
|
||||
assertThat(runs.get(0).getValueString("runType")).isEqualTo("RECONCILE");
|
||||
assertThat(runs.get(0).getValueString("status")).isEqualTo("COMPLETED");
|
||||
assertThat(runs.get(0).getValueInteger("recordsProcessed")).isEqualTo(4);
|
||||
assertThat(runs.get(0).getValueInteger("recordsIndexed")).isEqualTo(4);
|
||||
|
||||
List<QRecord> indexRows = queryAll(QuickSearchIndex.TABLE_NAME);
|
||||
assertThat(indexRows).hasSize(1);
|
||||
assertThat(indexRows.get(0).getValueInteger("recordCount")).isEqualTo(4);
|
||||
assertThat(indexRows.get(0).getValueInstant("lastReconcileTime")).isAfterOrEqualTo(beforeRun);
|
||||
assertThat(indexRows.get(0).getValueInstant("lastBasepullTime")).isAfterOrEqualTo(beforeRun);
|
||||
assertThat(indexRows.get(0).getValue("lastFullReindexTime")).as("a reconcile is not a full reindex").isNull();
|
||||
assertThat(indexRows.get(0).getValueInteger("documentCount")).isEqualTo(4);
|
||||
assertThat(indexRows.get(0).getValueString("status")).isEqualTo("ACTIVE");
|
||||
assertThat(indexRows.get(0).getValueString("lastRunStatus")).isEqualTo("RECONCILE COMPLETED");
|
||||
assertThat(indexRows.get(0).getValueString("searchableFieldsJson")).isEqualTo(AbstractIndexingStep.buildSearchableFieldsJson(QuickSearchQBitContext.getTableConfig(TEST_ENTITY_TABLE)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a failing document count is logged at WARN with the table name, and
|
||||
** the run still completes, keeping the previous documentCount.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_documentCountFails_loggedAsWarning_previousCountKept() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(true, 99);
|
||||
when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenThrow(new QException("count failed"));
|
||||
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ReconcileIndexStep.class);
|
||||
try
|
||||
{
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(ReconcileIndexStep.class);
|
||||
}
|
||||
|
||||
assertThat(collectingLogger.getCollectedMessages())
|
||||
.filteredOn(m -> Level.WARN.equals(m.getLevel()) && m.getMessage().contains("Could not count indexed documents"))
|
||||
.singleElement()
|
||||
.satisfies(m ->
|
||||
{
|
||||
assertThat(m.getMessage()).contains("\"tableName\":\"" + TEST_ENTITY_TABLE + "\"");
|
||||
assertThat(m.getMessage()).contains("\"stackTrace\"").contains("count failed");
|
||||
});
|
||||
|
||||
QRecord row = queryAll(QuickSearchIndex.TABLE_NAME).get(0);
|
||||
assertThat(row.getValueString("lastRunStatus")).isEqualTo("RECONCILE COMPLETED");
|
||||
assertThat(row.getValueInteger("recordCount")).isEqualTo(1);
|
||||
assertThat(row.getValueInteger("documentCount")).isEqualTo(99);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the tableName input limits the run to that table.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_unknownTableNameInput_nothingReconciled() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
|
||||
RunBackendStepInput input = new RunBackendStepInput();
|
||||
input.addValue("tableName", "someOtherTable");
|
||||
new ReconcileIndexStep().run(input, new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
verify(mockClient, never()).deleteDocumentsIndexedBefore(anyString(), any());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a quickSearchIndex row for testEntity with the given enabled flag.
|
||||
*******************************************************************************/
|
||||
private void insertIndexRow(Boolean enabled) throws QException
|
||||
{
|
||||
insertIndexRow(enabled, null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a quickSearchIndex row for testEntity with the given enabled flag
|
||||
** and documentCount.
|
||||
*******************************************************************************/
|
||||
private void insertIndexRow(Boolean enabled, Integer documentCount) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord()
|
||||
.withValue("tableName", TEST_ENTITY_TABLE)
|
||||
.withValue("enabled", enabled)
|
||||
.withValue("basepullIntervalMinutes", 5)
|
||||
.withValue("documentCount", documentCount)
|
||||
.withValue("status", "ACTIVE")));
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a disabled index row is skipped and counted in tablesSkipped.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_disabledRow_skipped() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
insertIndexRow(false);
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
verify(mockClient, never()).deleteDocumentsIndexedBefore(anyString(), any());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).isEmpty();
|
||||
assertThat(output.getValueInteger("tablesSkipped")).isEqualTo(1);
|
||||
assertThat(output.getValueInteger("recordsIndexed")).isEqualTo(0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: without a table filter, documents of tables that are no longer
|
||||
** configured are swept, passing the configured table names to keep.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void testReconcile_allTables_sweepsOrphanDocumentsOfUnconfiguredTables() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
when(mockClient.deleteDocumentsForTablesNotIn(any())).thenReturn(7L);
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
ArgumentCaptor<Collection<String>> keepCaptor = ArgumentCaptor.forClass(Collection.class);
|
||||
verify(mockClient).deleteDocumentsForTablesNotIn(keepCaptor.capture());
|
||||
assertThat(keepCaptor.getValue()).containsExactly(TEST_ENTITY_TABLE);
|
||||
assertThat(output.getValue("orphanDocumentsRemoved")).isEqualTo(7L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with a table filter the orphan sweep does not run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_singleTable_noOrphanSweep() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
|
||||
RunBackendStepInput input = new RunBackendStepInput();
|
||||
input.addValue("tableName", TEST_ENTITY_TABLE);
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new ReconcileIndexStep().run(input, output);
|
||||
|
||||
verify(mockClient).indexDocuments(anyList(), anyInt());
|
||||
verify(mockClient, never()).deleteDocumentsForTablesNotIn(any());
|
||||
assertThat(output.getValue("orphanDocumentsRemoved")).isEqualTo(0L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a failed run carries the error message on the run record and on
|
||||
** the index row.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_failedRun_indexRowCarriesError() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
when(mockClient.indexDocuments(anyList(), anyInt())).thenThrow(new QException("connection refused"));
|
||||
|
||||
assertThatThrownBy(() -> new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput())).isInstanceOf(QException.class);
|
||||
|
||||
QRecord row = queryAll(QuickSearchIndex.TABLE_NAME).get(0);
|
||||
assertThat(row.getValueString("lastRunStatus")).isEqualTo("RECONCILE FAILED");
|
||||
assertThat(row.getValueString("lastErrorMessage")).contains("connection refused");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a reconcile finishing while a full reindex runs leaves the
|
||||
** REBUILDING status in place, so deletes are still captured.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReconcile_duringFullReindex_keepsRebuilding() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord()
|
||||
.withValue("tableName", TEST_ENTITY_TABLE)
|
||||
.withValue("enabled", true)
|
||||
.withValue("status", AbstractIndexingStep.STATUS_REBUILDING)));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
new ReconcileIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
assertThat(queryAll(QuickSearchIndex.TABLE_NAME).get(0).getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
assertThat(indexedRecordIds()).containsExactly("1");
|
||||
}
|
||||
|
||||
}
|
||||
+58
-5
@@ -34,6 +34,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -167,10 +168,11 @@ class SynchronousIndexEventPublisherTest
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test publishDeleteEvents calls client.deleteDocument for each event.
|
||||
** Test publishDeleteEvents removes all documents in one bulk call, so one
|
||||
** failing document cannot stop the others from being removed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPublishDeleteEvents_callsClientForEachEvent() throws QException
|
||||
void testPublishDeleteEvents_bulkDeletesAllDocuments() throws QException
|
||||
{
|
||||
List<IndexEvent> events = List.of(
|
||||
new IndexEvent()
|
||||
@@ -182,10 +184,61 @@ class SynchronousIndexEventPublisherTest
|
||||
.withRecordId("20")
|
||||
.withAction(IndexEventAction.DELETE));
|
||||
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult());
|
||||
|
||||
publisher.publishDeleteEvents(events);
|
||||
|
||||
verify(mockClient).deleteDocument("orders", "10");
|
||||
verify(mockClient).deleteDocument("orders", "20");
|
||||
verify(mockClient).deleteDocuments(List.of("orders:10", "orders:20"), BATCH_SIZE);
|
||||
verify(mockClient, never()).deleteDocument(anyString(), anyString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test publishDeleteEvents throws, naming the failed documents, when some
|
||||
** bulk items fail.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPublishDeleteEvents_itemFailures_throwsQException() throws QException
|
||||
{
|
||||
BulkIndexResult result = new BulkIndexResult();
|
||||
result.addSuccess();
|
||||
result.addFailure("orders:20: shard unavailable");
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(result);
|
||||
|
||||
List<IndexEvent> events = List.of(
|
||||
new IndexEvent().withTableName("orders").withRecordId("10").withAction(IndexEventAction.DELETE),
|
||||
new IndexEvent().withTableName("orders").withRecordId("20").withAction(IndexEventAction.DELETE));
|
||||
|
||||
assertThatThrownBy(() -> publisher.publishDeleteEvents(events))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("1 of 2")
|
||||
.hasMessageContaining("orders:20: shard unavailable");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test publishIndexEvents throws, naming the failed documents, when some
|
||||
** bulk items fail (instead of silently dropping the failures).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPublishIndexEvents_itemFailures_throwsQException() throws QException
|
||||
{
|
||||
BulkIndexResult result = new BulkIndexResult();
|
||||
result.addFailure("orders:1: mapper_parsing_exception");
|
||||
when(mockClient.indexDocuments(anyList(), anyInt())).thenReturn(result);
|
||||
|
||||
IndexEvent event = new IndexEvent()
|
||||
.withTableName("orders")
|
||||
.withRecordId("1")
|
||||
.withAction(IndexEventAction.INDEX)
|
||||
.withRecord(new QRecord().withValue("id", 1).withValue("orderNumber", "ORD-001"));
|
||||
|
||||
assertThatThrownBy(() -> publisher.publishIndexEvents(List.of(event)))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("1 of 1")
|
||||
.hasMessageContaining("orders:1: mapper_parsing_exception");
|
||||
}
|
||||
|
||||
|
||||
@@ -209,7 +262,7 @@ class SynchronousIndexEventPublisherTest
|
||||
void testPublishDeleteEvents_clientThrows_propagatesException() throws QException
|
||||
{
|
||||
doThrow(new QException("delete failed"))
|
||||
.when(mockClient).deleteDocument("orders", "1");
|
||||
.when(mockClient).deleteDocuments(anyList(), anyInt());
|
||||
|
||||
IndexEvent event = new IndexEvent()
|
||||
.withTableName("orders")
|
||||
|
||||
Reference in new issue
Block a user