Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 james.maes
|
56829ce997 |
No files matched your search
@@ -1,20 +0,0 @@
|
||||
# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 3
|
||||
tab_width = 3
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.{cmd,bat}]
|
||||
end_of_line = crlf
|
||||
@@ -1,3 +0,0 @@
|
||||
* text=auto eol=lf
|
||||
*.cmd text eol=crlf
|
||||
mvnw text eol=lf
|
||||
@@ -1,25 +0,0 @@
|
||||
# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
|
||||
# major updates stay separate so each gets its own review. At most three version-update PRs per
|
||||
# ecosystem are open at once. Security updates are enabled in the repository settings and are
|
||||
# not limited by this schedule.
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: maven
|
||||
directory: /
|
||||
target-branch: develop
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 3
|
||||
groups:
|
||||
maven-minor-and-patch:
|
||||
update-types: [minor, patch]
|
||||
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
target-branch: develop
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 3
|
||||
groups:
|
||||
github-actions:
|
||||
patterns: ['*']
|
||||
@@ -15,7 +15,7 @@ This file provides guidance to Codex when working with code in this repository.
|
||||
./mvnw package # Build JAR
|
||||
CI=true ./mvnw verify # Full verify: integration tests (Docker) and the JaCoCo gate
|
||||
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0 # other server line
|
||||
./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1 # other qqq 4.1 build (profile is active by default)
|
||||
./mvnw verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT # other qqq 4.1 build (profile is active by default)
|
||||
```
|
||||
|
||||
Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
|
||||
@@ -34,7 +34,6 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
|
||||
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
|
||||
- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
|
||||
- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
|
||||
- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
|
||||
- `metadata/*` - Component producers and the table/process helpers.
|
||||
|
||||
**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
|
||||
@@ -51,7 +50,7 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
|
||||
## Dependencies
|
||||
|
||||
- Java 21
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom:4.1.0-RC.1` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007)
|
||||
- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
|
||||
- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
|
||||
- JUnit 5 + AssertJ + Mockito for testing
|
||||
|
||||
@@ -2,36 +2,6 @@
|
||||
|
||||
All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
Requires QQQ 4.1.0-RC.4 or later (the developer panel contract, QRun-IO/qqq#1076).
|
||||
|
||||
### Added
|
||||
- Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`.
|
||||
|
||||
## [1.1.0-RC.1] - 2026-10-11
|
||||
|
||||
First release candidate of 1.1, published to Maven Central from `release/1.1`. It contains every change listed under 1.1.0 and 1.0.0 below (1.0.0 has not been released as GA). Requires QQQ 4.1.0-RC.2 or later. Upgrade notes: `docs/MIGRATION-1.1.md`.
|
||||
|
||||
## [1.1.0] - Unreleased
|
||||
|
||||
Requires QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI; built against 4.1.0-RC.2. See `docs/MIGRATION-1.1.md`.
|
||||
|
||||
### Added
|
||||
- `QuickSearchRecordSearchProvider`: serves QQQ core record search (`RecordSearchAction`, `POST /qqq/v1/search`, the Next UI global search) from OpenSearch for the tables Quick Search indexes. One `_msearch` request with a per-table top-N search (same query, AND semantics and field weights as `QuickSearchAction`); returns record ids only, which core re-reads through `QueryAction` (permissions, record locks, labels). Audit finding F-11, decision D1.
|
||||
- `serveCoreRecordSearch` (default `true`): registers the provider and gives indexed tables core `searchFields` (visible string and integer searchable fields) when they have none. Host `searchFields` and a host-registered provider are never replaced.
|
||||
- `recordSearchTimeoutMillis` (default 2500): wall-clock limit on the OpenSearch request; on timeout or any failure core falls back to its own search, and the provider claims no tables for 30 seconds.
|
||||
- `QuickSearchOpenSearchClient.searchRecordIdsPerTable`.
|
||||
- Diagnostic tools (#35), on by default (`enableDiagnosticTools`): Search Playground, Validate Search Index (schedulable with `validationCronExpression`; reports kept in the new `quickSearchValidationRun` table), Test Search Analyzer, and on every indexed table and on `quickSearchFailedEvent` rows the record actions Inspect Search Document (database vs index, and why a record does or does not match a term) and Reindex in Search. All are table or record actions, gated by the new `debugPermissionRules` (`quickSearchDebug.hasAccess`; Reindex in Search uses `adminPermissionRules`), rate-limited per user (`diagnosticRunsPerMinute`), and recorded in the new `quickSearchDiagnosticEvent` table. Settings: `enableRecordDiagnosticActions`, `validationCronExpression`, `validationCronTimeZoneId`, `validationSampleSize`.
|
||||
- Search failure diagnostics: a failed or timed-out OpenSearch request behind core record search or `QuickSearchAction` is recorded as a `SEARCH_FAILED` diagnostic event (tables, duration, error, server; never the term); the runtime counts failures and backoff fallbacks. The Search Diagnostics widget (`quickSearchDiagnostics`, on the admin app home) shows the provider's state, recent failures and the latest validation.
|
||||
- `QuickSearchDiagnosticsOperations`: read-only probe search (named clauses, field highlights, request and response JSON), explain, document reads, `_analyze`, document sampling, and raw cluster, mapping and settings reads.
|
||||
|
||||
### Changed
|
||||
- Run history purging also purges `quickSearchDiagnosticEvent` and `quickSearchValidationRun` rows older than `runHistoryRetentionDays`.
|
||||
|
||||
### Security
|
||||
- Fields of type `PASSWORD` are no longer indexed (as hidden fields already were). Reconcile or fully reindex once to remove values a 1.0 index may hold.
|
||||
|
||||
## [1.0.0-RC.2] - 2026-10-06
|
||||
|
||||
Second release candidate, published to Maven Central from `release/1.0`. Fixes the five pre-GA findings from the PR #10 review (#13 to #17); the changes are listed under 1.0.0 below. Requires QQQ 4.1. No schema change, but two new status values: `REBUILDING` on `quickSearchIndex.status` and `AWAITING_REINDEX` on `quickSearchFailedEvent.status`; a host that sized its own `quickSearchFailedEvent.status` column needs at least 16 characters. Known gaps before GA are tracked in #12.
|
||||
@@ -85,7 +55,6 @@ Requires QQQ 4.1. Closes every blocker of the 2026-10-04 production-readiness au
|
||||
- An update was dropped without a `quickSearchFailedEvent` row when the listener's re-read failed or the optional AWS SDK was missing (`LinkageError`); both are now recorded for basepull, and a `DEGRADED` start without the AWS SDK no longer aborts host startup (#13).
|
||||
- Search pages overlapped under record locks, per-table mode ignored `offset`, and a large `maxSearchLimit` could exceed the 10,000 result window (#16).
|
||||
- A failing document count was swallowed and cleared `documentCount`; it is now logged at warn and the previous count is kept (#17).
|
||||
- A real-time insert was indexed with raw values (a foreign key as its id, the record label as "2 - email") until the next basepull. Inserts are now re-read in the write's transaction with display values and possible-value labels, as updates are; a failed re-read is recorded in `quickSearchFailedEvent` for basepull and never fails the insert (#28).
|
||||
|
||||
## [0.2.0] - 2026-03-30
|
||||
- Config-driven table indexing (`SearchableTableConfig`).
|
||||
|
||||
@@ -15,7 +15,7 @@ This file provides guidance to Claude Code when working with code in this reposi
|
||||
./mvnw package # Build JAR
|
||||
CI=true ./mvnw verify # Full verify: integration tests (Docker) and the JaCoCo gate
|
||||
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0 # other server line
|
||||
./mvnw verify -Dqqq.snapshot.version=4.1.0-RC.1 # other qqq 4.1 build (profile is active by default)
|
||||
./mvnw verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT # other qqq 4.1 build (profile is active by default)
|
||||
```
|
||||
|
||||
Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin.
|
||||
@@ -34,7 +34,6 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
|
||||
- `listeners/QuickSearchRecordChangeListener` - Real-time path: builds events in the write's transaction, publishes after commit, records failures in `quickSearchFailedEvent`.
|
||||
- `processes/AbstractIndexingStep` - Keyset paging by primary key with display values, run records, drift detection, run purge. `BasepullIndexStep` (replays failed events, start-anchored watermark with overlap), `ReconcileIndexStep` (in place, orphan sweep), `FullReindexStep` (all tables: new physical index plus alias swap; one table: reconcile algorithm).
|
||||
- `actions/QuickSearchAction` - Permission-aware search (table READ filter, `QueryAction` post-filter), bounds, `tableNames`, `limitPerTable`, `tableLabel`.
|
||||
- `actions/QuickSearchRecordSearchProvider` - QQQ core `RecordSearchProviderInterface` (`POST /qqq/v1/search`): claims indexed, enabled tables; one `_msearch` per call returning ordered record ids per table; bounded by `recordSearchTimeoutMillis`; failures make core fall back and start a 30 s backoff (`QuickSearchRuntime.markRecordSearchFailed`). Registered, with default table `searchFields`, when `serveCoreRecordSearch` is true.
|
||||
- `metadata/*` - Component producers and the table/process helpers.
|
||||
|
||||
**Tables Produced:** `quickSearchIndex`, `quickSearchIndexRun`, `quickSearchFailedEvent` (plus a PVS over `quickSearchIndex`).
|
||||
@@ -51,7 +50,7 @@ Integration tests use `@ExtendWith(RequiresDockerCondition.class)`: skipped with
|
||||
## Dependencies
|
||||
|
||||
- Java 21
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007). 1.1 imports `4.1.0-RC.4`: the record-search provider SPI arrived in RC.2, the developer panel contract (`DeveloperPanelProviderInterface`) in RC.4
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). 1.0 needs qqq 4.1, so the `qqq-snapshot` profile is active by default and imports `qqq-bom-pom:4.1.0-RC.1` ahead of the parent's BOM; remove that activation when the parent moves to 4.1 (ADR-0007)
|
||||
- OpenSearch Java Client 3.10.0; optional AWS SDK v2 `apache-client` and `sts` for `AWS_SIGV4`
|
||||
- Apache HttpClient5 and jackson-datatype-jsr310, versions managed by the qqq BOM (keep them unpinned so they match its httpcore5 and jackson)
|
||||
- JUnit 5 + AssertJ + Mockito for testing
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# QBit: Quick Search
|
||||
|
||||
[](https://github.com/QRun-IO/qbit-quick-search)
|
||||
[](https://github.com/QRun-IO/qbit-quick-search)
|
||||
[](https://www.apache.org/licenses/LICENSE-2.0)
|
||||
[](https://adoptium.net/)
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
Annotate entity classes or declare tables in config, point the QBit at an OpenSearch cluster, produce it into your `QInstance`, and you get real-time indexing of committed writes, a scheduled catch-up and reconcile, a blackout-free full rebuild, and a permission-aware search API with relevance scoring, typeahead, highlights and pagination.
|
||||
|
||||
Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services). Serving core search (`POST /qqq/v1/search`, below) needs QQQ 4.1.0-RC.2 or later, the first release that carries the record-search provider SPI. The record developer panel (below) needs QQQ 4.1.0-RC.4 or later, which carries the developer panel contract; the 1.1 line builds against 4.1.0-RC.4 from the next release candidate on.
|
||||
Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit callbacks and runtime services).
|
||||
|
||||
## What it does
|
||||
|
||||
@@ -18,7 +18,6 @@ Requires QQQ 4.1 (the QBit uses 4.1's record-change listeners, after-commit call
|
||||
| Basepull catch-up | Scheduled process (`schedulerName` plus `basepullRepeatSeconds`) that re-indexes rows changed since the previous run's start, with a configurable overlap, paging by primary key. |
|
||||
| Reconcile | Scheduled or on-demand process that re-indexes everything in place, then removes documents whose source row is gone, including documents of tables no longer configured. No search blackout. |
|
||||
| Full rebuild | Indexes every table into a fresh physical index and atomically swaps the alias; the old index keeps serving until the new one is complete. Also upgrades an index created by an older release. |
|
||||
| Core search | `QuickSearchRecordSearchProvider` serves QQQ's `RecordSearchAction` (`POST /qqq/v1/search`, the Next UI global search box) from OpenSearch for the indexed tables, with a timeout and an automatic fallback to core's own search. |
|
||||
| Search | `QuickSearchAction`: AND semantics, edge-ngram typeahead with a plain search analyzer, per-field weights, highlights, `tableNames` and `limitPerTable`, bounded paging, `tableLabel` on results. Only tables the session may read are searched; hits are re-read through `QueryAction` so record security locks apply. |
|
||||
| Connections | HTTP or HTTPS, basic auth, AWS IAM SigV4, custom CA or trust store, mTLS, timeouts, pool size, a transport customizer SPI for anything else. Secrets by `${env.X}` reference. |
|
||||
| Operations | `quickSearchIndex` (per-table status, counts, last run, last error), `quickSearchIndexRun` (history, purged after `runHistoryRetentionDays`), `quickSearchFailedEvent`, a `quickSearchAdmin` app, explicit permission rules, fail-fast or degraded startup. |
|
||||
@@ -49,11 +48,11 @@ For `AWS_SIGV4` add these optional dependencies to your host (the QBit declares
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-quick-search</artifactId>
|
||||
<version>1.1.0-RC.1</version>
|
||||
<version>1.0.0-RC.1</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
`1.1.0-RC.1` is the current release candidate (the 1.1 line, published from `release/1.1`; it needs QQQ 4.1.0-RC.2 or later, see [Upgrading from 1.0](#upgrading-from-10)). Hosts staying on the 1.0 line use `1.0.0-RC.2`; `1.0.0` follows once host applications have adopted it (#12).
|
||||
`1.0.0-RC.1` is the current release candidate; `1.0.0` follows once host applications have adopted it (#12).
|
||||
|
||||
Annotate entity classes:
|
||||
|
||||
@@ -100,7 +99,7 @@ config.withSearchableTable(new SearchableTableConfig("crmContact", List.of(
|
||||
.withRecordLabelFormat("%s %s", "firstName", "lastName"));
|
||||
```
|
||||
|
||||
Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden and password fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull, and the QBit logs a warning for it at startup. Set `basepullTimestampField` to null on a config-driven table to opt out without the warning.
|
||||
Annotation and config-driven tables can be mixed; a table may appear in only one of them. Hidden fields are never indexed. A table without the change-detection field (`modifyDate` by default) is covered by reconcile instead of incremental basepull.
|
||||
|
||||
### Searching
|
||||
|
||||
@@ -125,73 +124,16 @@ output.getTotalHits(); output.getTotalHitsIsLowerBound(); output.getHasMore();
|
||||
|
||||
Search runs as the current `QContext` session: tables the session cannot read are skipped, and with `applyRecordSecurityLocks` (default) hits are re-read through `QueryAction` so record security locks apply.
|
||||
|
||||
### Core search (`POST /qqq/v1/search`)
|
||||
|
||||
QQQ core has its own record search (`RecordSearchAction`, served at `POST /qqq/v1/search` and used by the Next UI's global search box): a case-insensitive contains-query over each table's `searchFields`. With `serveCoreRecordSearch` (default `true`) the producer registers `QuickSearchRecordSearchProvider` on the instance, and core asks it to search every table Quick Search indexes. The API and the UI do not change.
|
||||
|
||||
- **What is served from OpenSearch.** Tables this QBit indexes and that are enabled in `quickSearchIndex`. Core offers a table only when it has `searchFields`, the session may read it, and it supports query, so the producer gives each indexed table that has no `searchFields` its visible string and integer searchable fields (never hidden or password fields). `searchFields` you set yourself are kept. A table must be in the `QInstance` before the QBit is produced to get them.
|
||||
- **How.** One `_msearch` request with one search per table (Quick Search's query, AND semantics and field weights); each table returns up to 4 x `limitPerTable` record ids (at most 100) in relevance order. Core re-reads them by primary key through `QueryAction`, so table permissions, record security locks and record labels are core's, and records deleted since they were indexed drop out.
|
||||
- **Failure.** The OpenSearch request is cancelled after `recordSearchTimeoutMillis` (default 2500). Any failure (timeout, cluster down, index missing) makes core search those tables with its own contains-query instead, so search degrades but never fails. After a failure, Quick Search claims no tables for 30 seconds, so a down cluster does not add a timeout to every search.
|
||||
- **Opting out.** `withServeCoreRecordSearch(false)` leaves core search and `searchFields` untouched. If the host already registered another record-search provider, the QBit keeps it and logs a warning (an instance has one provider).
|
||||
|
||||
`QuickSearchAction` remains the richer API (scores, highlights, pagination, totals).
|
||||
|
||||
### Processes
|
||||
|
||||
| Process | Trigger | What it does |
|
||||
|---|---|---|
|
||||
| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Recovers a full reindex that stopped without finishing, replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
|
||||
| Quick Search Basepull Index | Scheduled every `basepullRepeatSeconds` when `schedulerName` is set; also on demand | Replays failed real-time events, re-indexes rows changed since the last run start minus `basepullOverlapSeconds`, purges old run history |
|
||||
| Quick Search Reconcile Index | Cron via `reconcileCronExpression` when `schedulerName` is set; also on demand (optional `tableName` input) | Re-indexes everything in place, removes documents without a source row and documents of unconfigured tables |
|
||||
| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place. Fails to start while another full reindex is running |
|
||||
| Quick Search Full Reindex | On demand (optional `tableName` input) | All tables: build a fresh physical index and swap the alias. One table: reconcile algorithm in place |
|
||||
|
||||
Without `schedulerName` the processes exist but nothing runs them; the QBit logs a warning at startup.
|
||||
|
||||
### Scheduling
|
||||
|
||||
The schedules run only when the host starts through QQQ's `QApplicationLauncher` (`com.kingsrook.qqq.middleware.javalin.QApplicationLauncher`, in `qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` (because the QBit's processes are scheduled), then the instance's runtime services. A host that starts only a server, or builds the `QInstance` itself, gets the processes but nothing runs them on a schedule, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
|
||||
|
||||
Basepull's fixed interval runs on any QQQ scheduler. `reconcileCronExpression` needs a scheduler that supports cron, which in QQQ 4.1 means a `QuartzSchedulerMetaData`; a `SimpleSchedulerMetaData` runs fixed intervals only. The expression uses Quartz syntax, seconds first (`0 0 3 * * ?` is 03:00 every day), and needs `reconcileCronTimeZoneId`, a Java time zone ID such as `UTC` or `America/Chicago`; QQQ has no default zone. `produce()` rejects a cron without a zone, an invalid expression or zone, and a cron on a scheduler in the instance that does not support cron.
|
||||
|
||||
```java
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName("quartz").withProperties(quartzProperties));
|
||||
|
||||
config.withSchedulerName("quartz")
|
||||
.withReconcileCronExpression("0 0 3 * * ?")
|
||||
.withReconcileCronTimeZoneId("America/Chicago");
|
||||
```
|
||||
|
||||
### Diagnostic tools
|
||||
|
||||
With `enableDiagnosticTools` (default `true`) the QBit adds tools to validate the search and debug it from the admin UI. They are table and record actions (never left-nav entries), gated by permissions, recorded in `quickSearchDiagnosticEvent`, limited to `diagnosticRunsPerMinute` runs per user and tool, and read-only except Reindex in Search.
|
||||
|
||||
| Tool | Where | What it does |
|
||||
|---|---|---|
|
||||
| Search Playground | Quick Search Index, Actions (select rows to pre-fill their tables) | Runs a term the way the global search does, as you: the path each table takes (OpenSearch provider, core database search, or not searched) and why; the provider's own request, timed; the global search results you would see; per indexed table, the production query with scores, highlights, matched fields, which hits you can read and which the global search shows, the query DSL and the raw response (hits you cannot read are redacted); the term's tokens; and `explain` for a hit (`table:id`). The form stays open, so you can change the term and run again. |
|
||||
| Validate Search Index | Quick Search Index, Actions; also scheduled by `validationCronExpression` | Cluster and index health, the alias, leftover physical indexes from interrupted rebuilds, the live mapping (version, field types, analyzers, the `fieldValues` template) and analysis settings, then per table: database rows vs documents, a sample of records checked present and current in the index, a random sample of documents checked to still have a record, the index row's status and last run, and waiting failed events. Each finding is OK, INFO, WARN or FAIL with what to do. The report is saved as a `quickSearchValidationRun` row (HTML for the record view, JSON for tools). Database reads run as the system user so counts compare like with like; the report holds counts, ids and field names, never record values. One validation runs at a time per server. |
|
||||
| Test Search Analyzer | Quick Search Index, Actions | `_analyze` on the live index: the tokens of a text as indexed (edge n-grams of 2 to 20 characters) and as a query, and, given a term, whether every term token is among the text's indexed tokens. |
|
||||
| Inspect Search Document | Every indexed table's records (Actions), and Quick Search Failed Event rows | The record's document in the index next to the document indexing would write from the database now, field by field (current, stale or missing), with versions and the raw document. Enter a term to see why the record does or does not match it: term length, table enabled, read permission, document present and current, each term token found or not (and why: one character, longer than 20, not in the text, or only in the database's newer values), `explain`, and the record's rank against the number of ids the global search takes. |
|
||||
| Reindex in Search | Every indexed table's records, and Quick Search Failed Event rows | Rebuild the selected records' documents from the database now (versioned writes, so a newer document is never overwritten); a record that no longer exists has its document removed; a record you cannot read is skipped. Asks to confirm. Needs the admin permission. |
|
||||
| Search Diagnostics widget | The Quick Search Admin app home (hosts with their own home add `quickSearchDiagnostics`) | Whether the global search is served by OpenSearch on this server or backing off (and why), failure and fallback counts, the latest search failures from every server, and the latest validation run. |
|
||||
|
||||
Search failures are recorded too: when the provider's OpenSearch request fails or times out (and core falls back to its own search), or `QuickSearchAction`'s search fails, a `SEARCH_FAILED` event keeps the tables, duration, error class and message, and the server. The user's term is never stored, only its length. At most 120 events are written per server per minute.
|
||||
|
||||
Permissions: the read-only tools and the widget use `debugPermissionRules` (default `quickSearchDebug.hasAccess`); Reindex in Search uses `adminPermissionRules` (`quickSearchAdmin.hasAccess`); the two history tables use `tablePermissionRules` (grant `.read` only: rows are written by the QBit). Impersonation ("search as user X") is deliberately not offered: a fabricated session would not carry the real user's security key values, so its results would mislead.
|
||||
|
||||
### Record developer panel
|
||||
|
||||
With QQQ 4.1.0-RC.4 or later the QBit registers a record developer panel provider (`QuickSearchDeveloperPanelProvider`, id `quickSearch`, in QQQ's `DeveloperPanelsMetaData`). Frontends that render developer panels (the Next UI, `/app/{table}/{id}/dev`) show a "Quick Search" panel on every record of an indexed table, to users who may read Quick Search Index. Every part is permission-checked:
|
||||
|
||||
| Part | Shown when |
|
||||
|---|---|
|
||||
| Index: enabled, status, record count, last basepull, last full reindex, searchable fields, link to the index row | always (read on Quick Search Index) |
|
||||
| Latest index run: type, status, start and end, processed, indexed, errors | the user may read Quick Search Index Run |
|
||||
| Search document: verdict (current, stale, missing) and a Field / Database / Index / Match table | `enableDiagnosticTools` and the debug permission (`debugPermissionRules`); otherwise a note says why it is missing |
|
||||
| Failed index events of the record | the user may read Quick Search Failed Event and the record has any |
|
||||
| Actions: Inspect search document, Reindex this record | the record actions exist and the user may run them (debug and admin permission) |
|
||||
| Link to Quick Search Admin | the user may open the app |
|
||||
|
||||
The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned off in Quick Search Index), or OpenSearch unavailable. When OpenSearch cannot be reached the panel still shows the database-side state, with a warning in place of the document comparison. Hosts without a developer-panel frontend are unaffected.
|
||||
|
||||
## Configuration reference
|
||||
|
||||
| Field | Default | Description |
|
||||
@@ -209,24 +151,15 @@ The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned
|
||||
| `startupMode` | `FAIL_FAST` | `FAIL_FAST` fails `produce()` when the cluster is unreachable; `DEGRADED` boots and retries on first use |
|
||||
| `enableRealTimeIndexing` | `true` | Register the record-change listener |
|
||||
| `enableBasepullProcess`, `enableMaintenanceProcesses` | `true` | Register basepull; register full reindex and reconcile. `enableScheduledProcesses` is a deprecated alias for both |
|
||||
| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none, none | Scheduling; a cron needs the time zone ID and a Quartz scheduler, and nothing runs without `QApplicationLauncher` (see [Scheduling](#scheduling)) |
|
||||
| `schedulerName`, `basepullRepeatSeconds`, `reconcileCronExpression`, `reconcileCronTimeZoneId` | none, 300, none | Scheduling |
|
||||
| `basepullOverlapSeconds` | 300 | Re-read window before the previous run start |
|
||||
| `defaultBasepullIntervalMinutes` | 5 | Per-table due interval for config-driven tables |
|
||||
| `runHistoryRetentionDays` | 30 | Purge run records older than this; null keeps them |
|
||||
| `fullReindexStaleMinutes` | 30 | A running full reindex that shows no sign of life for this long is treated as stopped: it no longer blocks a new one, and basepull recovers its tables. Keep it well above the time one page of `sourceBatchSize` records takes; null means the default |
|
||||
| `bulkBatchSize`, `maxBulkRequestBytes`, `sourceBatchSize` | 500, 5 MiB, 1000 | Batch sizes |
|
||||
| `maxFieldLength` | 10000 | Truncate each indexed value |
|
||||
| `maxSearchLimit` | 100 | Largest page size |
|
||||
| `applyRecordSecurityLocks` | `true` | Re-read hits through `QueryAction` |
|
||||
| `serveCoreRecordSearch` | `true` | Register `QuickSearchRecordSearchProvider` so core `POST /qqq/v1/search` uses OpenSearch for indexed tables, and give those tables `searchFields` when they have none |
|
||||
| `recordSearchTimeoutMillis` | 2500 | Wall-clock limit on the OpenSearch request behind a core search; past it, core falls back to its own search |
|
||||
| `enableDiagnosticTools` | `true` | Add the diagnostic tools, the `quickSearchDiagnosticEvent` and `quickSearchValidationRun` tables and the diagnostics widget |
|
||||
| `enableRecordDiagnosticActions` | `true` | With the tools on, add Inspect Search Document and Reindex in Search to every indexed table |
|
||||
| `validationCronExpression`, `validationCronTimeZoneId` | none | Schedule Validate Search Index; same rules as the reconcile schedule (a cron-capable scheduler such as Quartz, a valid Quartz expression, a time zone ID) |
|
||||
| `validationSampleSize` | 50 | Records and documents sampled per table by validation (at most 1000) |
|
||||
| `diagnosticRunsPerMinute` | 30 | Per-user, per-tool limit on diagnostic runs |
|
||||
| `debugPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchDebug` | Rules for the read-only diagnostic tools and the widget |
|
||||
| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchAdmin` | Rules for the app, the maintenance processes and Reindex in Search |
|
||||
| `adminPermissionRules` | `HAS_ACCESS_PERMISSION`, base name `quickSearchAdmin` | Rules for the app and processes |
|
||||
| `tablePermissionRules` | `READ_WRITE_PERMISSIONS` | Rules for the operational tables |
|
||||
| `tableMetaDataCustomizer` | | Applied to every produced table (backend details, for example) |
|
||||
| `searchableEntityClasses`, `searchableTables` | at least one | What to index |
|
||||
@@ -238,9 +171,7 @@ The panel's status is Indexed, Stale, Not indexed, Disabled (the table is turned
|
||||
|
||||
- `quickSearchIndex`: one row per table with `enabled` (honoured by indexing and search, cached for one minute), `basepullIntervalMinutes`, `lastBasepullTime`, `lastReconcileTime`, `lastFullReindexTime`, `lastRunStatus`, `lastErrorMessage`, `recordCount`, `documentCount`, `realTimeErrorCount`, `status` (`ACTIVE`, `NEEDS_REINDEX` when the configured fields changed since the row was created, or `REBUILDING` while a full reindex of all tables runs). Unique on `tableName`.
|
||||
- `quickSearchIndexRun`: `runType` (`BASEPULL`, `RECONCILE`, `FULL_REINDEX`), `status`, counts, `errorMessage`.
|
||||
- `quickSearchDiagnosticEvent` (with `enableDiagnosticTools`): `eventType` (`SEARCH_FAILED`, `TOOL_RUN`, `REINDEX`, `RATE_LIMITED`), `source`, `outcome`, `userName`, `tableNames`, `searchTerm` (tool runs only), `durationMillis`, `errorClass`, `errorMessage` (text), `detail` (text), `hostName`, `createDate`. Purged after `runHistoryRetentionDays`.
|
||||
- `quickSearchValidationRun` (with `enableDiagnosticTools`): `status` (`PASS`, `WARN`, `FAIL`), `triggerType` (`MANUAL`, `SCHEDULED`), `userName`, `tableNames`, `sampleSize`, `tablesChecked`, `failCount`, `warnCount`, `summary`, `reportHtml` (HTML), `reportJson` (text), `startTime`, `endTime`, `createDate`. Purged after `runHistoryRetentionDays`.
|
||||
- `quickSearchFailedEvent`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. Only one full reindex runs at a time: starting one while another is running fails. A full reindex shows it is running through its `FULL_REINDEX` run records, which stay `RUNNING` with `modifyDate` refreshed after every page. If it is killed mid-run (pod restart, OOM, deploy), recovery is automatic once its run records have shown no sign of life for `fullReindexStaleMinutes` (30 by default): the next basepull, or the next full reindex, marks those run records `FAILED`, returns the tables to `ACTIVE` with `lastBasepullTime` moved back to the run's start, and turns the `AWAITING_REINDEX` rows `PENDING` so basepull replays the deletes. No data is lost. A run that was only slow and is judged stopped notices before its alias swap and fails without swapping, or, after its swap, stops applying its captured deletes; basepull skips a replayed delete whose record exists again.
|
||||
- `quickSearchFailedEvent`: real-time events that could not be applied; `status` `PENDING` or `EXHAUSTED` after 10 attempts. Deletes made while a full reindex of all tables runs are also recorded here with status `AWAITING_REINDEX`: the reindex applies them to the new index after the alias swap (basepull does not replay them). While a table is `REBUILDING`, drift detection is suspended for it. If a full reindex is killed mid-run, its tables stay `REBUILDING` and deletes keep collecting as `AWAITING_REINDEX`; no data is lost, and the recovery is to run a full reindex of all tables again, which applies the collected deletes and returns the tables to `ACTIVE`.
|
||||
|
||||
## Extending
|
||||
|
||||
@@ -254,21 +185,11 @@ CI=true ./mvnw verify # plus Testcontainers integration te
|
||||
./mvnw verify -Dopensearch.test.image=opensearchproject/opensearch:3.9.0
|
||||
```
|
||||
|
||||
Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.4 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
|
||||
|
||||
## Upgrading from 1.0
|
||||
|
||||
See [docs/MIGRATION-1.1.md](docs/MIGRATION-1.1.md). In short: 1.1 needs the QQQ release with the record-search provider SPI; core search starts using OpenSearch for indexed tables by default, and indexed tables gain `searchFields` (set `serveCoreRecordSearch(false)` to keep 1.0 behaviour).
|
||||
Use the wrapper: Maven 3.10 cannot read the published parent POM. The build imports qqq 4.1.0-RC.1 through the default-active `qqq-snapshot` profile until `qbit-build-parent` 2.1.0 ships (override with `-Dqqq.snapshot.version=...`). Without Docker the integration tests are skipped locally; with `CI=true` they fail instead.
|
||||
|
||||
## Upgrading from 0.x
|
||||
|
||||
See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, grant `quickSearchAdmin.hasAccess` and the table permissions, add the new operational table columns and `quickSearchFailedEvent`, and run a full reindex once so the index gets the new mapping.
|
||||
|
||||
## Adopting in another app
|
||||
|
||||
Follow [Getting started](#getting-started), the [Configuration reference](#configuration-reference) and [Operational tables](#operational-tables) (the host application creates the three tables; this QBit ships no DDL), then run a full reindex. Hosts on 1.0 or 0.x should read the upgrade sections above.
|
||||
|
||||
See also the [QQQ adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) for the full per-feature list (QQQ upgrade, Next dashboard, QBits, migrations, permissions).
|
||||
See [docs/MIGRATION-1.0.md](docs/MIGRATION-1.0.md). In short: QQQ 4.1 is required, remove any manual `addQBit` for this QBit, set `schedulerName`, prefer `opensearchUrl` and `${env.}` secrets, and run a full reindex once so the index gets the new mapping.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+4
-68
@@ -12,84 +12,20 @@
|
||||
1. **Remove any manual `QBitMetaData` registration.** The producer now registers `com.kingsrook.qbits:quick-search` itself; a second `addQBit` with that name throws at startup. (qqq-all's `DemoQBits` did this in 0.x.)
|
||||
2. **Decide the startup mode.** `startupMode` defaults to `FAIL_FAST`: `produce()` fails when OpenSearch is unreachable. Demo profiles that may run without a cluster should set `withStartupMode(QuickSearchStartupMode.DEGRADED)`.
|
||||
3. **Run a full reindex once after upgrading.** The index mapping changed (version 2: search analyzer, text-typed `fieldValues`). The QBit logs a warning when it finds an index with the old mapping; "Quick Search Full Reindex" with no table name builds a new physical index and swaps the configured name to an alias. Search keeps working during the rebuild. The identity the QBit uses needs index create, delete and alias privileges for this (see the README role).
|
||||
4. **Set `schedulerName` and start the host through `QApplicationLauncher`** (and optionally `basepullRepeatSeconds`, or `reconcileCronExpression` with `reconcileCronTimeZoneId` on a Quartz scheduler) so basepull and reconcile actually run. In 0.x they were never scheduled. See [Scheduling](#scheduling).
|
||||
5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `withAllowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
|
||||
6. **Grant the permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant the names listed under [Permissions](#permissions) to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
|
||||
7. **Update the operational tables before starting 1.0.** `quickSearchIndex` gains five columns, `searchableFieldsJson` changes from `STRING` to `TEXT`, and `tableName` becomes required with a unique key. `quickSearchIndexRun.errorMessage` changes from `STRING` to `TEXT`. There is a new `quickSearchFailedEvent` table. A host that checks its columns at startup rejects the 0.x tables. See [Schema changes](#schema-changes).
|
||||
|
||||
## Permissions
|
||||
|
||||
QQQ names a permission `{baseName}.{type}`. The admin rules use the base name `quickSearchAdmin`; the table rules have no base name, so each table's name is its base name. With the default rules, grant these to operators:
|
||||
|
||||
| Permission | Covers |
|
||||
|---|---|
|
||||
| `quickSearchAdmin.hasAccess` | The `quickSearchAdmin` app and the three processes (`quickSearchBasepullIndex`, `quickSearchReconcileIndex`, `quickSearchFullReindex`) |
|
||||
| `quickSearchIndex.read`, `quickSearchIndex.write` | The `quickSearchIndex` table |
|
||||
| `quickSearchIndexRun.read`, `quickSearchIndexRun.write` | The `quickSearchIndexRun` table |
|
||||
| `quickSearchFailedEvent.read`, `quickSearchFailedEvent.write` | The `quickSearchFailedEvent` table |
|
||||
|
||||
The tables use `READ_WRITE_PERMISSIONS`: `.read` covers query, get and count, and `.write` covers insert, edit and delete. If you set `tablePermissionRules` to `READ_INSERT_EDIT_DELETE_PERMISSIONS`, grant `.read`, `.insert`, `.edit` and `.delete` on each table instead. With `tableNamePrefix` set, the table permissions carry the prefix (`{prefix}quickSearchIndex.read`); the admin permission stays `quickSearchAdmin.hasAccess`.
|
||||
|
||||
Users who only search need none of these. Search checks the read permission of each searched table as that table's own rules name it (for example `customer.read`). The QBit's own reads and writes of its tables go through QQQ's table actions, which do not check permissions, so real-time indexing does not depend on these grants.
|
||||
|
||||
For a worked example, qqq-all's RC.2 adoption (QRun-IO/qqq-all#30) adds `infra/postgres/06-quick-search-permissions.sql`, which grants the seven names above to its admin role.
|
||||
|
||||
## Schema changes
|
||||
|
||||
The QBit declares its tables in metadata and ships no DDL. A backend that creates tables from metadata gets the 1.0 shape on a new database. A host that manages its own schema, or checks columns at startup, must change the 0.x tables before starting 1.0. Names below are QQQ field names and types; your backend's naming decides the physical table and column names.
|
||||
|
||||
`quickSearchIndex`:
|
||||
|
||||
| Field | Type | Change from 0.x |
|
||||
|---|---|---|
|
||||
| `lastReconcileTime` | `DATE_TIME` | New |
|
||||
| `lastRunStatus` | `STRING` | New. Run type and outcome, for example `FULL_REINDEX COMPLETED` (at most 22 characters) |
|
||||
| `lastErrorMessage` | `TEXT` | New. Up to 4,000 characters |
|
||||
| `documentCount` | `INTEGER` | New |
|
||||
| `realTimeErrorCount` | `INTEGER` | New |
|
||||
| `searchableFieldsJson` | `TEXT` | Was `STRING` |
|
||||
| `tableName` | `STRING` | Now required, with a unique key |
|
||||
|
||||
`status` holds `ACTIVE`, `NEEDS_REINDEX` (new in 1.0) or `REBUILDING` (new in RC.2), so it needs at least 13 characters. QQQ checks the `tableName` unique key on insert; a unique index in the database also stops two concurrent first runs from both inserting. 0.x could leave duplicate rows, so keep one row per table before adding the index.
|
||||
|
||||
`quickSearchIndexRun` has no new columns. `errorMessage` changed from `STRING` to `TEXT` and holds up to 4,000 characters.
|
||||
|
||||
`quickSearchFailedEvent` is new:
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `id` | `INTEGER` | Primary key, generated by the backend like the other two tables |
|
||||
| `tableName` | `STRING` | Source table |
|
||||
| `recordId` | `STRING` | Source record's primary key as text |
|
||||
| `action` | `STRING` | `INDEX` or `DELETE` |
|
||||
| `errorMessage` | `TEXT` | Up to 4,000 characters |
|
||||
| `attempts` | `INTEGER` | Replay attempts |
|
||||
| `status` | `STRING` | `PENDING`, `EXHAUSTED` or `AWAITING_REINDEX` (new in RC.2); size it for at least 16 characters |
|
||||
| `createDate`, `modifyDate` | `DATE_TIME` | |
|
||||
|
||||
The QBit sets no maximum length on its `STRING` fields. Map `STRING` to a character column sized for the values above, `TEXT` to a type that holds at least 4,000 characters, and `DATE_TIME` to a timestamp, the same way your other QQQ tables do.
|
||||
|
||||
If your backend creates tables from metadata and you do not need the 0.x run history, you can instead drop the 0.x `quickSearchIndex` and `quickSearchIndexRun` tables and let startup recreate them, as qqq-all#30 documents. The next basepull, reconcile or full reindex recreates each `quickSearchIndex` row, with `enabled` and `basepullIntervalMinutes` back at their configured defaults.
|
||||
4. **Set `schedulerName`** (and optionally `basepullRepeatSeconds`, `reconcileCronExpression`) so basepull and reconcile actually run. In 0.x they were never scheduled.
|
||||
5. **Credentials over HTTP are refused.** Use `opensearchUrl` with `https://`, or set `allowPlaintextCredentials(true)` for local development. Prefer `${env.OPENSEARCH_PASSWORD}` references over literal secrets; never name an OpenSearch secret `QQQ_ENV_*` (those are sent to the frontend).
|
||||
6. **Permissions.** Processes and the admin app now require `quickSearchAdmin.hasAccess`; the operational tables require read and write permissions. Grant them to operators, or override `adminPermissionRules` and `tablePermissionRules`. Search now honours table read permissions and record security locks, so users see fewer results than in 0.x if they lacked access.
|
||||
|
||||
## Behaviour changes to be aware of
|
||||
|
||||
- Real-time indexing happens after commit. Code that searched for a record inside the same transaction that inserted it will not find it until commit.
|
||||
- Real-time failures no longer vanish: they land in `quickSearchFailedEvent` and are replayed by the next basepull. Review that table after an OpenSearch outage.
|
||||
- Inserted and updated records are re-read inside the write's transaction, with display values and possible-value labels, before they are indexed. That is one extra query per write batch on an indexed table, and it runs the host's possible-value providers. If the re-read fails, the write still succeeds and the records are replayed by basepull.
|
||||
- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile. The QBit logs a warning for each such table at startup, for annotated and config-driven tables alike; a config-driven table with `basepullTimestampField` set to null has opted out and gets no warning.
|
||||
- A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile.
|
||||
- `limit` is capped (default 100), `offset + limit` at 10,000, and search terms must be 2 to 100 characters.
|
||||
- Query semantics changed from OR over n-grams to AND over whole terms with prefix matching: "blue widget" now requires both words.
|
||||
- `enableScheduledProcesses` still works and sets both `enableBasepullProcess` and `enableMaintenanceProcesses`.
|
||||
- `QuickSearchQBitContext` is deprecated and removed in 1.1; use `QuickSearchRuntime.get()`.
|
||||
|
||||
## Scheduling
|
||||
|
||||
Setting `schedulerName` puts schedules on the basepull and reconcile processes; QQQ runs them only once something starts its `QScheduleManager`.
|
||||
|
||||
- **Start through `QApplicationLauncher`.** In QQQ 4.1 that is `com.kingsrook.qqq.middleware.javalin.QApplicationLauncher` (`qqq-middleware-javalin`). `QApplicationLauncher.run(new MyApplication(), new QApplicationLauncherConfig())` starts the Javalin server, then the `QScheduleManager` when any process is scheduled, then the instance's runtime services, including this QBit's. A host that starts only a `QApplicationJavalinServer`, or builds its `QInstance` without the launcher, gets the processes but no scheduled runs, unless it calls `QScheduleManager.initInstance(qInstance, systemSessionSupplier).start()` itself. `qqq.scheduleManager.enabled=false` (or `QQQ_SCHEDULE_MANAGER_ENABLED=false`) turns schedules off on a node.
|
||||
- **A reconcile cron needs a Quartz scheduler.** Basepull's fixed interval runs on any QQQ scheduler, but only `QuartzSchedulerMetaData` supports cron; QQQ's `SimpleSchedulerMetaData` runs fixed intervals only, and a cron schedule on it fails when the schedule manager starts.
|
||||
- **A reconcile cron needs a time zone.** Set `reconcileCronTimeZoneId` to a Java time zone ID (`UTC`, `America/Chicago`). QQQ has no default zone: its instance validation rejects a cron schedule without one at startup. `produce()` now catches this first, along with an invalid Quartz expression (seconds first, for example `0 0 3 * * ?`), an unknown zone ID, and a cron on a scheduler in the instance that does not support cron.
|
||||
|
||||
## Removed
|
||||
|
||||
- `QuickSearchPostInsertCustomizer`, `QuickSearchPostUpdateCustomizer`, `QuickSearchPostDeleteCustomizer`. Hosts that referenced them (for example to compose with their own customizers) should remove those references; the record-change listener coexists with host customizers.
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# Migrating from 1.0 to 1.1
|
||||
|
||||
1.1 lets QQQ core's record search (`POST /qqq/v1/search`, the Next UI global search box) use the Quick Search index. Hosts that do nothing get it by default.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- The QQQ release that carries the record-search provider SPI (`RecordSearchProviderInterface`, `QInstance.withRecordSearchProvider`), first released in QQQ `4.1.0-RC.2`. 1.1 builds against `4.1.0-RC.2` through the `qqq-snapshot` profile.
|
||||
|
||||
## What changes at startup
|
||||
|
||||
1. **The QBit registers a record-search provider.** `QuickSearchRecordSearchProvider` becomes the instance's `recordSearchProvider`. If the host already registered a different provider, the QBit keeps it and logs a warning; core search then does not use OpenSearch.
|
||||
2. **Indexed tables get `searchFields`.** Core only searches tables with `searchFields`. Each indexed table that has none gets its visible string and integer searchable fields. Tables that already have `searchFields` keep them. A table added to the instance after the QBit is produced gets none, so add tables first (as for field validation) or set `searchFields` yourself.
|
||||
3. **Password fields are no longer indexed.** Fields of type `PASSWORD` are now dropped from the searchable fields like hidden fields. If a 1.0 index holds such values, run "Quick Search Reconcile Index" (or a full reindex) to rewrite the documents without them.
|
||||
|
||||
## Behaviour to be aware of
|
||||
|
||||
- Next UI search results for indexed tables now come from OpenSearch: all words must match (in any searchable field), with prefix matching and field weights, instead of a contains-match on one field. Tables not indexed by Quick Search keep core's search.
|
||||
- Results still pass core's security: core re-reads every hit through `QueryAction`, so table permissions and record security locks apply, and records deleted since they were indexed do not appear.
|
||||
- Records are only as fresh as the index: a record updated in a way that is not yet indexed (for example while real-time indexing is off) is found by its indexed text.
|
||||
- When OpenSearch is slow or down, core waits at most `recordSearchTimeoutMillis` (default 2500) and then searches with its own contains-query; for the next 30 seconds it skips OpenSearch entirely.
|
||||
|
||||
## Opting out
|
||||
|
||||
```java
|
||||
new QuickSearchQBitConfig()
|
||||
// ...
|
||||
.withServeCoreRecordSearch(false);
|
||||
```
|
||||
|
||||
restores 1.0 behaviour: no provider, no `searchFields` changes.
|
||||
|
||||
## Diagnostic tools (1.1.0-RC.2)
|
||||
|
||||
The QBit now adds admin tools to validate and debug search (see the README, "Diagnostic tools"). They are on by default.
|
||||
|
||||
- **Two new operational tables**, on `backendName` like the others: `quickSearchDiagnosticEvent` and `quickSearchValidationRun` (columns in the README, "Operational tables"). Create them in the host's schema, with the naming the host applies to the other operational tables (`tableMetaDataCustomizer` applies to them too). Until they exist, writing events and saving reports fails quietly (logged at warn) and the tools still run.
|
||||
- **Permissions.** Grant `quickSearchDebug.hasAccess` to the people who may use the read-only tools and see the widget, `quickSearchAdmin.hasAccess` for Reindex in Search, and `.read` on the two new tables (the QBit writes them; users never do).
|
||||
- **New processes** have a `tableName` (Quick Search Index, Quick Search Failed Event, and each indexed table), so they appear as table and record actions. They are not added to the admin app's children; a host that strips processes from its nav needs no change.
|
||||
- **Opting out.** `withEnableDiagnosticTools(false)` adds none of it; `withEnableRecordDiagnosticActions(false)` keeps the tools on Quick Search Index but adds nothing to the indexed tables.
|
||||
+1
-6
@@ -9,7 +9,7 @@
|
||||
- [ ] qqq-all follow-up PR: remove the manual `instance.addQBit(...)` for quick-search in `DemoQBits`, pass `withSchedulerName("demoScheduler")`, set `withStartupMode(QuickSearchStartupMode.DEGRADED)` for the full profile test, pin 1.0.0.
|
||||
- [ ] Voyage: pin 1.0.0, set `schedulerName`, switch to `opensearchUrl` and `${env.}` credentials.
|
||||
- [ ] CI: add a second test job with `-Dopensearch.test.image=opensearchproject/opensearch:3.9.0` (needs a qqq-orb parameter for extra Maven args, or a plain job).
|
||||
- [x] Core SPI proposed as QRun-IO/qqq#1042 (D1); the 1.1 adapter `QuickSearchRecordSearchProvider` is on `feature/record-search-provider`.
|
||||
- [ ] Open the qqq issue proposing `RecordSearchProviderInterface` (D1); quick-search adapter lands in 1.1.
|
||||
- [x] Tracking issue #12 created and recorded in the PRD (2026-10-06).
|
||||
- [x] PR #8 merged 2026-10-05; Dependabot PR #1 closed as superseded.
|
||||
- [x] PR #10 merged and `1.0.0-RC.1` published to Maven Central from `release/1.0` (2026-10-05).
|
||||
@@ -26,11 +26,6 @@
|
||||
- [ ] Stuck `REBUILDING` after a killed or overlapping full reindex (from the PR #20 review): only another all-tables reindex clears it, `AWAITING_REINDEX` rows accumulate and drift detection stays off for those tables. No data is lost. Fix needs a liveness check on the `FULL_REINDEX` run record before reconcile may reset the status, plus refusing a second concurrent full reindex.
|
||||
- [ ] Drift detection still writes `NEEDS_REINDEX` into `status`, so a few-millisecond window can overwrite `REBUILDING`; the durable fix is a separate drift field (schema change, so 1.1 at the earliest).
|
||||
|
||||
## Before releasing 1.1.0
|
||||
|
||||
- [x] Re-pointed the `qqq-snapshot` profile from `4.1.0-SNAPSHOT` to `4.1.0-RC.2`, the first QQQ release with the record-search provider SPI (2026-10-11).
|
||||
- [ ] qqq-all / Website-Backend: confirm the Next UI search box is served from OpenSearch (provider registered, indexed tables have `searchFields`).
|
||||
|
||||
## Later (1.x)
|
||||
|
||||
- [ ] Health widget for the admin app (data exists: `isHealthy`, `documentCount`, `lastRunStatus`).
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
tracker_key: "QRun-IO/qbit-quick-search#12"
|
||||
project: qbit-quick-search
|
||||
status: released-rc
|
||||
updated: 2026-10-09
|
||||
updated: 2026-10-06
|
||||
---
|
||||
|
||||
# Initiative: QBit Quick Search 1.0 on QQQ 4.1
|
||||
@@ -11,7 +11,7 @@ updated: 2026-10-09
|
||||
|
||||
## Status
|
||||
|
||||
`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). `1.0.0-RC.2` followed on 2026-10-06 with the review follow-ups #13 to #17. Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
|
||||
`1.0.0-RC.1` was released to Maven Central on 2026-10-05 from `release/1.0`, after the work merged to `develop` in #10 and #8. All six epics are coded and tested (307 unit, 18 integration, coverage gate met). Decisions D1 to D11 were applied with the recommended answers, with two recorded adjustments: D4 (missing `schedulerName` warns rather than fails) and D5 (single-table full reindex uses the reconcile algorithm). Tracking issue: #12. Open before GA: host adoption in qqq-all and Voyage, the review follow-ups #13 to #17, the `qbit-build-parent` 2.1.0 re-pin, the CI image matrix, and the qqq core SPI issue (1.1).
|
||||
|
||||
## Executive Summary
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>1.1.0-SNAPSHOT</revision>
|
||||
<revision>1.0.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<opensearch.version>3.10.0</opensearch.version>
|
||||
@@ -264,17 +264,13 @@
|
||||
<!-- qqq-bom-pom 4.1.0-RC.1 ahead of the parent's 4.0.0 BOM. Override with -->
|
||||
<!-- -Dqqq.snapshot.version=... ; deactivate with -P!qqq-snapshot. Remove the -->
|
||||
<!-- activation when the parent moves to 4.1 (ADR-0007). -->
|
||||
<!-- 1.1: the record-search provider needs the core SPI -->
|
||||
<!-- (RecordSearchProviderInterface), first released in qqq 4.1.0-RC.2. -->
|
||||
<!-- The record developer panel provider needs the developer panel -->
|
||||
<!-- contract (DeveloperPanelProviderInterface), first in qqq 4.1.0-RC.4. -->
|
||||
<profile>
|
||||
<id>qqq-snapshot</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.1</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -18,13 +18,10 @@ package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.net.URI;
|
||||
import java.text.ParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.TimeZone;
|
||||
import java.util.regex.Pattern;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
@@ -33,14 +30,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import org.quartz.CronExpression;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -57,33 +52,14 @@ import org.quartz.CronExpression;
|
||||
** enableMaintenanceProcesses, schedulerName plus basepull/reconcile schedules,
|
||||
** permission rules for the admin surface, search bounds, and indexing limits.
|
||||
**
|
||||
** Core search: serveCoreRecordSearch (default true) registers
|
||||
** QuickSearchRecordSearchProvider, so QQQ's POST /qqq/v1/search (the Next UI
|
||||
** global search) is served from OpenSearch for indexed tables, bounded by
|
||||
** recordSearchTimeoutMillis.
|
||||
**
|
||||
** Tables: searchableEntityClasses (annotated entities) and/or searchableTables
|
||||
** (config-driven). validate() is list-based and never throws; the producer
|
||||
** fails production when it reports errors.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchQBitConfig implements QBitConfig
|
||||
{
|
||||
public static final String DEFAULT_ADMIN_PERMISSION_BASE_NAME = "quickSearchAdmin";
|
||||
public static final String DEFAULT_BASEPULL_TIMESTAMP_FIELD = "modifyDate";
|
||||
public static final int DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS = 2500;
|
||||
public static final String DEFAULT_DEBUG_PERMISSION_BASE_NAME = "quickSearchDebug";
|
||||
public static final int DEFAULT_VALIDATION_SAMPLE_SIZE = 50;
|
||||
public static final int MAX_VALIDATION_SAMPLE_SIZE = 1000;
|
||||
public static final int DEFAULT_DIAGNOSTIC_RUNS_PER_MINUTE = 30;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// a live full reindex refreshes its run record after every page; a page is //
|
||||
// one source query plus sourceBatchSize / bulkBatchSize bulk requests, each //
|
||||
// bounded by the transport timeouts (a few minutes at most by default), //
|
||||
// so 30 minutes leaves a wide margin before a slow run is judged dead, and //
|
||||
// basepull (every 5 minutes by default) recovers a dead one soon after //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
public static final Integer DEFAULT_FULL_REINDEX_STALE_MINUTES = 30;
|
||||
public static final String DEFAULT_ADMIN_PERMISSION_BASE_NAME = "quickSearchAdmin";
|
||||
public static final String DEFAULT_BASEPULL_TIMESTAMP_FIELD = "modifyDate";
|
||||
|
||||
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^[a-z0-9][a-z0-9_+.-]*$");
|
||||
private static final Set<String> AWS_SERVICE_NAMES = Set.of("es", "aoss");
|
||||
@@ -119,27 +95,17 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
private Integer basepullOverlapSeconds = 300;
|
||||
private Integer defaultBasepullIntervalMinutes = 5;
|
||||
private Integer runHistoryRetentionDays = 30;
|
||||
private Integer fullReindexStaleMinutes = DEFAULT_FULL_REINDEX_STALE_MINUTES;
|
||||
private Integer bulkBatchSize = 500;
|
||||
private Integer sourceBatchSize = 1000;
|
||||
private Integer maxFieldLength = 10000;
|
||||
private Integer maxSearchLimit = 100;
|
||||
private Boolean applyRecordSecurityLocks = true;
|
||||
private Boolean serveCoreRecordSearch = true;
|
||||
private Integer recordSearchTimeoutMillis = DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS;
|
||||
private Boolean enableDiagnosticTools = true;
|
||||
private Boolean enableRecordDiagnosticActions = true;
|
||||
private String validationCronExpression;
|
||||
private String validationCronTimeZoneId;
|
||||
private Integer validationSampleSize = DEFAULT_VALIDATION_SAMPLE_SIZE;
|
||||
private Integer diagnosticRunsPerMinute = DEFAULT_DIAGNOSTIC_RUNS_PER_MINUTE;
|
||||
private List<Class<?>> searchableEntityClasses;
|
||||
private List<SearchableTableConfig> searchableTables;
|
||||
|
||||
private transient IndexEventPublisher indexEventPublisher;
|
||||
private transient QPermissionRules adminPermissionRules = new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(DEFAULT_ADMIN_PERMISSION_BASE_NAME);
|
||||
private transient QPermissionRules tablePermissionRules = new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS);
|
||||
private transient QPermissionRules debugPermissionRules = new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(DEFAULT_DEBUG_PERMISSION_BASE_NAME);
|
||||
private transient MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer;
|
||||
private transient QuickSearchRuntime runtime;
|
||||
|
||||
@@ -421,17 +387,8 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
requirePositive(maxBulkRequestBytes, "maxBulkRequestBytes", errors);
|
||||
requirePositive(basepullRepeatSeconds, "basepullRepeatSeconds", errors);
|
||||
requirePositive(maxSearchLimit, "maxSearchLimit", errors);
|
||||
requirePositive(recordSearchTimeoutMillis, "recordSearchTimeoutMillis", errors);
|
||||
requirePositive(maxFieldLength, "maxFieldLength", errors);
|
||||
requirePositive(runHistoryRetentionDays, "runHistoryRetentionDays", errors);
|
||||
requirePositive(fullReindexStaleMinutes, "fullReindexStaleMinutes", errors);
|
||||
requirePositive(validationSampleSize, "validationSampleSize", errors);
|
||||
requirePositive(diagnosticRunsPerMinute, "diagnosticRunsPerMinute", errors);
|
||||
|
||||
if(validationSampleSize != null && validationSampleSize > MAX_VALIDATION_SAMPLE_SIZE)
|
||||
{
|
||||
errors.add("validationSampleSize must be at most " + MAX_VALIDATION_SAMPLE_SIZE + " for QuickSearchQBit");
|
||||
}
|
||||
|
||||
if(basepullOverlapSeconds != null && basepullOverlapSeconds < 0)
|
||||
{
|
||||
@@ -455,71 +412,18 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The scheduler must exist when the instance already has schedulers. A
|
||||
** reconcile cron gets the checks QQQ's instance validator applies to every
|
||||
** cron schedule (Quartz syntax, a required and known time zone, which QQQ
|
||||
** never defaults), plus a scheduler that supports cron, so a bad schedule
|
||||
** fails produce() naming these fields instead of failing at startup.
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void validateScheduling(QInstance qInstance, List<String> errors)
|
||||
{
|
||||
QSchedulerMetaData scheduler = null;
|
||||
if(StringUtils.hasContent(schedulerName) && qInstance != null && !CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulers()))
|
||||
if(StringUtils.hasContent(schedulerName) && qInstance != null && !CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulers()) && qInstance.getScheduler(schedulerName) == null)
|
||||
{
|
||||
scheduler = qInstance.getScheduler(schedulerName);
|
||||
if(scheduler == null)
|
||||
{
|
||||
errors.add("Scheduler not found: " + schedulerName);
|
||||
}
|
||||
errors.add("Scheduler not found: " + schedulerName);
|
||||
}
|
||||
|
||||
validateCron("reconcile", reconcileCronExpression, reconcileCronTimeZoneId, scheduler, errors);
|
||||
validateCron("validation", validationCronExpression, validationCronTimeZoneId, scheduler, errors);
|
||||
|
||||
if(StringUtils.hasContent(validationCronExpression) && !Boolean.TRUE.equals(enableDiagnosticTools))
|
||||
if(StringUtils.hasContent(reconcileCronExpression) && !StringUtils.hasContent(schedulerName))
|
||||
{
|
||||
errors.add("validationCronExpression requires enableDiagnosticTools");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A cron schedule (prefix "reconcile" or "validation"): needs schedulerName
|
||||
** on a scheduler that supports cron, a valid Quartz expression and a
|
||||
** recognized time zone ID.
|
||||
*******************************************************************************/
|
||||
private void validateCron(String prefix, String cronExpression, String cronTimeZoneId, QSchedulerMetaData scheduler, List<String> errors)
|
||||
{
|
||||
if(StringUtils.hasContent(cronExpression))
|
||||
{
|
||||
if(!StringUtils.hasContent(schedulerName))
|
||||
{
|
||||
errors.add(prefix + "CronExpression requires schedulerName");
|
||||
}
|
||||
else if(scheduler != null && !scheduler.supportsCronSchedules())
|
||||
{
|
||||
errors.add(prefix + "CronExpression requires a scheduler that supports cron schedules, such as a QuartzSchedulerMetaData; scheduler [" + schedulerName + "] is of type " + scheduler.getType());
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
CronExpression.validateExpression(cronExpression);
|
||||
}
|
||||
catch(ParseException e)
|
||||
{
|
||||
errors.add(prefix + "CronExpression [" + cronExpression + "] is not a valid Quartz cron expression: " + e.getMessage());
|
||||
}
|
||||
|
||||
if(!StringUtils.hasContent(cronTimeZoneId))
|
||||
{
|
||||
errors.add(prefix + "CronExpression requires " + prefix + "CronTimeZoneId (a time zone ID such as UTC or America/Chicago)");
|
||||
}
|
||||
}
|
||||
|
||||
if(StringUtils.hasContent(cronTimeZoneId) && !Arrays.asList(TimeZone.getAvailableIDs()).contains(cronTimeZoneId))
|
||||
{
|
||||
errors.add(prefix + "CronTimeZoneId [" + cronTimeZoneId + "] is not a recognized time zone ID");
|
||||
errors.add("reconcileCronExpression requires schedulerName");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -654,17 +558,6 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** fullReindexStaleMinutes, defaulting to DEFAULT_FULL_REINDEX_STALE_MINUTES.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Integer getEffectiveFullReindexStaleMinutes()
|
||||
{
|
||||
return (fullReindexStaleMinutes == null ? DEFAULT_FULL_REINDEX_STALE_MINUTES : fullReindexStaleMinutes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@@ -1686,9 +1579,7 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for reconcileCronExpression: optional Quartz cron schedule for
|
||||
** reconcile; needs reconcileCronTimeZoneId and a scheduler that supports
|
||||
** cron (Quartz)
|
||||
** Getter for reconcileCronExpression: optional cron schedule for reconcile (needs a scheduler that supports cron)
|
||||
*******************************************************************************/
|
||||
public String getReconcileCronExpression()
|
||||
{
|
||||
@@ -1720,9 +1611,7 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for reconcileCronTimeZoneId: time zone ID for
|
||||
** reconcileCronExpression (UTC, America/Chicago); required with it, since
|
||||
** QQQ has no default zone for cron schedules
|
||||
** Getter for reconcileCronTimeZoneId: time zone for reconcileCronExpression
|
||||
*******************************************************************************/
|
||||
public String getReconcileCronTimeZoneId()
|
||||
{
|
||||
@@ -1848,36 +1737,6 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fullReindexStaleMinutes: a RUNNING full reindex whose run record shows no sign of life (start or heartbeat) for this many minutes is treated as stopped; null means the default
|
||||
*******************************************************************************/
|
||||
public Integer getFullReindexStaleMinutes()
|
||||
{
|
||||
return (this.fullReindexStaleMinutes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for fullReindexStaleMinutes
|
||||
*******************************************************************************/
|
||||
public void setFullReindexStaleMinutes(Integer fullReindexStaleMinutes)
|
||||
{
|
||||
this.fullReindexStaleMinutes = fullReindexStaleMinutes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fullReindexStaleMinutes
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withFullReindexStaleMinutes(Integer fullReindexStaleMinutes)
|
||||
{
|
||||
this.fullReindexStaleMinutes = fullReindexStaleMinutes;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for bulkBatchSize: documents per OpenSearch bulk request
|
||||
@@ -2038,72 +1897,6 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for serveCoreRecordSearch: register QuickSearchRecordSearchProvider
|
||||
** so core RecordSearchAction (POST /qqq/v1/search) searches indexed tables in
|
||||
** OpenSearch, and give indexed tables core searchFields when they have none
|
||||
*******************************************************************************/
|
||||
public Boolean getServeCoreRecordSearch()
|
||||
{
|
||||
return (this.serveCoreRecordSearch);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for serveCoreRecordSearch
|
||||
*******************************************************************************/
|
||||
public void setServeCoreRecordSearch(Boolean serveCoreRecordSearch)
|
||||
{
|
||||
this.serveCoreRecordSearch = serveCoreRecordSearch;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for serveCoreRecordSearch
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withServeCoreRecordSearch(Boolean serveCoreRecordSearch)
|
||||
{
|
||||
this.serveCoreRecordSearch = serveCoreRecordSearch;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for recordSearchTimeoutMillis: wall-clock limit on the OpenSearch
|
||||
** request behind a core record search; past it the provider fails and core
|
||||
** falls back to its own search (default 2500)
|
||||
*******************************************************************************/
|
||||
public Integer getRecordSearchTimeoutMillis()
|
||||
{
|
||||
return (this.recordSearchTimeoutMillis);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for recordSearchTimeoutMillis
|
||||
*******************************************************************************/
|
||||
public void setRecordSearchTimeoutMillis(Integer recordSearchTimeoutMillis)
|
||||
{
|
||||
this.recordSearchTimeoutMillis = recordSearchTimeoutMillis;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for recordSearchTimeoutMillis
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withRecordSearchTimeoutMillis(Integer recordSearchTimeoutMillis)
|
||||
{
|
||||
this.recordSearchTimeoutMillis = recordSearchTimeoutMillis;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for searchableEntityClasses: entity classes annotated with @QuickSearchable
|
||||
@@ -2265,229 +2058,6 @@ public class QuickSearchQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for enableDiagnosticTools: add the admin diagnostic tools (search playground, index validation, analyzer tester, diagnostic event and validation run tables, diagnostics widget); default true
|
||||
*******************************************************************************/
|
||||
public Boolean getEnableDiagnosticTools()
|
||||
{
|
||||
return (this.enableDiagnosticTools);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for enableDiagnosticTools
|
||||
*******************************************************************************/
|
||||
public void setEnableDiagnosticTools(Boolean enableDiagnosticTools)
|
||||
{
|
||||
this.enableDiagnosticTools = enableDiagnosticTools;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for enableDiagnosticTools
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withEnableDiagnosticTools(Boolean enableDiagnosticTools)
|
||||
{
|
||||
this.enableDiagnosticTools = enableDiagnosticTools;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for enableRecordDiagnosticActions: with enableDiagnosticTools, also add the per-table record actions (inspect search document, reindex records) to every indexed table; default true
|
||||
*******************************************************************************/
|
||||
public Boolean getEnableRecordDiagnosticActions()
|
||||
{
|
||||
return (this.enableRecordDiagnosticActions);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for enableRecordDiagnosticActions
|
||||
*******************************************************************************/
|
||||
public void setEnableRecordDiagnosticActions(Boolean enableRecordDiagnosticActions)
|
||||
{
|
||||
this.enableRecordDiagnosticActions = enableRecordDiagnosticActions;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for enableRecordDiagnosticActions
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withEnableRecordDiagnosticActions(Boolean enableRecordDiagnosticActions)
|
||||
{
|
||||
this.enableRecordDiagnosticActions = enableRecordDiagnosticActions;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for validationCronExpression: cron schedule for the index validation process (needs schedulerName); none by default
|
||||
*******************************************************************************/
|
||||
public String getValidationCronExpression()
|
||||
{
|
||||
return (this.validationCronExpression);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for validationCronExpression
|
||||
*******************************************************************************/
|
||||
public void setValidationCronExpression(String validationCronExpression)
|
||||
{
|
||||
this.validationCronExpression = validationCronExpression;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for validationCronExpression
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withValidationCronExpression(String validationCronExpression)
|
||||
{
|
||||
this.validationCronExpression = validationCronExpression;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for validationCronTimeZoneId: time zone for validationCronExpression
|
||||
*******************************************************************************/
|
||||
public String getValidationCronTimeZoneId()
|
||||
{
|
||||
return (this.validationCronTimeZoneId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for validationCronTimeZoneId
|
||||
*******************************************************************************/
|
||||
public void setValidationCronTimeZoneId(String validationCronTimeZoneId)
|
||||
{
|
||||
this.validationCronTimeZoneId = validationCronTimeZoneId;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for validationCronTimeZoneId
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withValidationCronTimeZoneId(String validationCronTimeZoneId)
|
||||
{
|
||||
this.validationCronTimeZoneId = validationCronTimeZoneId;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for validationSampleSize: records sampled per table by index validation (default 50, at most 1000)
|
||||
*******************************************************************************/
|
||||
public Integer getValidationSampleSize()
|
||||
{
|
||||
return (this.validationSampleSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for validationSampleSize
|
||||
*******************************************************************************/
|
||||
public void setValidationSampleSize(Integer validationSampleSize)
|
||||
{
|
||||
this.validationSampleSize = validationSampleSize;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for validationSampleSize
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withValidationSampleSize(Integer validationSampleSize)
|
||||
{
|
||||
this.validationSampleSize = validationSampleSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for diagnosticRunsPerMinute: per-user limit on diagnostic tool runs per minute (default 30)
|
||||
*******************************************************************************/
|
||||
public Integer getDiagnosticRunsPerMinute()
|
||||
{
|
||||
return (this.diagnosticRunsPerMinute);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for diagnosticRunsPerMinute
|
||||
*******************************************************************************/
|
||||
public void setDiagnosticRunsPerMinute(Integer diagnosticRunsPerMinute)
|
||||
{
|
||||
this.diagnosticRunsPerMinute = diagnosticRunsPerMinute;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for diagnosticRunsPerMinute
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withDiagnosticRunsPerMinute(Integer diagnosticRunsPerMinute)
|
||||
{
|
||||
this.diagnosticRunsPerMinute = diagnosticRunsPerMinute;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for debugPermissionRules: permission rules for the read-only diagnostic tools and the diagnostics widget (default: quickSearchDebug.hasAccess)
|
||||
*******************************************************************************/
|
||||
public QPermissionRules getDebugPermissionRules()
|
||||
{
|
||||
return (this.debugPermissionRules);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for debugPermissionRules
|
||||
*******************************************************************************/
|
||||
public void setDebugPermissionRules(QPermissionRules debugPermissionRules)
|
||||
{
|
||||
this.debugPermissionRules = debugPermissionRules;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for debugPermissionRules
|
||||
*******************************************************************************/
|
||||
public QuickSearchQBitConfig withDebugPermissionRules(QPermissionRules debugPermissionRules)
|
||||
{
|
||||
this.debugPermissionRules = debugPermissionRules;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for runtime: live state once produced (client, publisher, discovered tables)
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -28,22 +28,16 @@ import java.util.Properties;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchProcessMetaDataHelper;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
@@ -56,9 +50,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** processes, admin app), applies the config's table customizer and default
|
||||
** backend, stamps sourceQBitName, then calls {@link #postProduceActions}, which
|
||||
** discovers searchable tables, builds the runtime, registers the record-change
|
||||
** listener and runtime service, registers the core record-search provider
|
||||
** (serveCoreRecordSearch) and gives indexed tables core searchFields, and (in
|
||||
** FAIL_FAST mode) prepares the index.
|
||||
** listener and runtime service, and (in FAIL_FAST mode) prepares the index.
|
||||
**
|
||||
** Host usage, unchanged from 0.x:
|
||||
** new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
@@ -262,14 +254,6 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
|
||||
|
||||
qInstance.withRuntimeService(new QCodeReference(QuickSearchRuntimeService.class));
|
||||
|
||||
addRecordDiagnosticActions(output, qInstance, discoveredTables);
|
||||
|
||||
if(Boolean.TRUE.equals(qBitConfig.getServeCoreRecordSearch()))
|
||||
{
|
||||
registerRecordSearchProvider(qInstance);
|
||||
defaultCoreSearchFields(qInstance, discoveredTables);
|
||||
}
|
||||
|
||||
if(!StringUtils.hasContent(qBitConfig.getSchedulerName()) && Boolean.TRUE.equals(qBitConfig.getEnableBasepullProcess()))
|
||||
{
|
||||
LOG.warn("Quick Search has no schedulerName; the basepull and reconcile processes are registered but will not run on a schedule");
|
||||
@@ -286,116 +270,6 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Give every indexed table in the instance the Inspect Search Document and
|
||||
** Reindex in Search record actions (when enableDiagnosticTools and
|
||||
** enableRecordDiagnosticActions are on). A process the host already
|
||||
** defined with the same name is kept.
|
||||
*******************************************************************************/
|
||||
void addRecordDiagnosticActions(MetaDataProducerMultiOutput output, QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
|
||||
{
|
||||
if(!Boolean.TRUE.equals(qBitConfig.getEnableDiagnosticTools()) || !Boolean.TRUE.equals(qBitConfig.getEnableRecordDiagnosticActions()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tableConfigs)
|
||||
{
|
||||
if(qInstance.getTable(tableConfig.getTableName()) == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for(QProcessMetaData process : List.of(
|
||||
QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(qBitConfig, tableConfig.getTableName()),
|
||||
QuickSearchDiagnosticsProcessMetaDataHelper.buildReindexRecordsProcess(qBitConfig, tableConfig.getTableName())))
|
||||
{
|
||||
if(qInstance.getProcess(process.getName()) == null && output.get(QProcessMetaData.class, process.getName()) == null)
|
||||
{
|
||||
process.setSourceQBitName(getQBitMetaData().getName());
|
||||
output.add(process);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register QuickSearchRecordSearchProvider as the instance's core record
|
||||
** search provider. An instance holds one provider; a different one the host
|
||||
** already registered is kept (with a warning).
|
||||
*******************************************************************************/
|
||||
static void registerRecordSearchProvider(QInstance qInstance)
|
||||
{
|
||||
QCodeReference existing = qInstance.getRecordSearchProvider();
|
||||
if(existing != null && !QuickSearchRecordSearchProvider.class.getName().equals(existing.getName()))
|
||||
{
|
||||
LOG.warn("The instance already has a record search provider; Quick Search does not replace it, so core search does not use OpenSearch",
|
||||
logPair("provider", existing.getName()));
|
||||
return;
|
||||
}
|
||||
|
||||
qInstance.withRecordSearchProvider(new QCodeReference(QuickSearchRecordSearchProvider.class));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core record search only searches tables with searchFields, so give each
|
||||
** indexed table that has none the searchable fields core accepts (visible
|
||||
** string or integer fields; never password fields). Fields the host set are
|
||||
** never replaced. A table that is not in the instance yet, uses backend
|
||||
** variants, or has no such field is skipped with a warning.
|
||||
*******************************************************************************/
|
||||
static void defaultCoreSearchFields(QInstance qInstance, List<QuickSearchableTableConfig> tableConfigs)
|
||||
{
|
||||
for(QuickSearchableTableConfig tableConfig : tableConfigs)
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(tableConfig.getTableName());
|
||||
if(table == null)
|
||||
{
|
||||
LOG.warn("Indexed table is not in the instance when Quick Search is produced, so it gets no core searchFields; set them on the table for core search to use it",
|
||||
logPair("tableName", tableConfig.getTableName()));
|
||||
continue;
|
||||
}
|
||||
|
||||
if(table.getSearchFields() != null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
|
||||
if(backend != null && Boolean.TRUE.equals(backend.getUsesVariants()))
|
||||
{
|
||||
LOG.warn("Indexed table uses backend variants, which core record search does not support; it gets no core searchFields", logPair("tableName", table.getName()));
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> searchFields = new ArrayList<>();
|
||||
for(String fieldName : CollectionUtils.nonNullList(tableConfig.getSearchableFields()))
|
||||
{
|
||||
QFieldMetaData field = table.getFields() == null ? null : table.getFields().get(fieldName);
|
||||
if(field != null && !Boolean.TRUE.equals(field.getIsHidden()) && field.getType() != null
|
||||
&& (field.getType().isStringLike() || field.getType().isIntegral()) && !field.getType().needsMasked()
|
||||
&& !searchFields.contains(fieldName))
|
||||
{
|
||||
searchFields.add(fieldName);
|
||||
}
|
||||
}
|
||||
|
||||
if(searchFields.isEmpty())
|
||||
{
|
||||
LOG.warn("Indexed table has no string or integer searchable field, so it gets no core searchFields; set them on the table for core search to use it",
|
||||
logPair("tableName", table.getName()));
|
||||
continue;
|
||||
}
|
||||
|
||||
table.setSearchFields(searchFields);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Discover searchable tables from @QuickSearchable entity classes. Fails when
|
||||
** a configured field does not exist on a table that is in the instance.
|
||||
@@ -512,12 +386,8 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
|
||||
QTableMetaData table = qInstance.getTable(stc.getTableName());
|
||||
if(table != null)
|
||||
{
|
||||
primaryKeyField = table.getPrimaryKeyField();
|
||||
if(basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField))
|
||||
{
|
||||
warnIncrementalBasepullDisabled(table, basepullTimestampField);
|
||||
basepullTimestampField = null;
|
||||
}
|
||||
primaryKeyField = table.getPrimaryKeyField();
|
||||
basepullTimestampField = (basepullTimestampField != null && !table.getFields().containsKey(basepullTimestampField)) ? null : basepullTimestampField;
|
||||
removeHiddenFields(table, searchableFields, fieldWeights, fieldIncludeLabels);
|
||||
}
|
||||
|
||||
@@ -574,7 +444,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
|
||||
{
|
||||
if(QuickSearchQBitConfig.DEFAULT_BASEPULL_TIMESTAMP_FIELD.equals(basepullTimestampField))
|
||||
{
|
||||
warnIncrementalBasepullDisabled(table, basepullTimestampField);
|
||||
LOG.warn("Table has no modifyDate field; incremental basepull is disabled for it (reconcile still covers it)", logPair("tableName", table.getName()));
|
||||
return (null);
|
||||
}
|
||||
throw (new QException("basepullTimestampField [" + basepullTimestampField + "] does not exist on table [" + table.getName() + "]"));
|
||||
@@ -585,36 +455,22 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer<QuickSearch
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The startup warning, once per table, for annotated and config-driven
|
||||
** tables alike, when the basepull timestamp field is missing. A field
|
||||
** explicitly set to null is an opt-out and is not warned about.
|
||||
*******************************************************************************/
|
||||
private static void warnIncrementalBasepullDisabled(QTableMetaData table, String basepullTimestampField)
|
||||
{
|
||||
LOG.warn("Table has no basepull timestamp field; incremental basepull is disabled for it (reconcile still covers it)",
|
||||
logPair("tableName", table.getName()), logPair("timestampField", basepullTimestampField));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hidden and password fields are never indexed: their values would leak
|
||||
** through searchable text and highlights.
|
||||
** Hidden fields are never indexed: their values would leak through
|
||||
** searchable text and highlights.
|
||||
*******************************************************************************/
|
||||
private static void removeHiddenFields(QTableMetaData table, List<String> searchableFields, Map<String, Integer> fieldWeights, Map<String, Boolean> fieldIncludeLabels)
|
||||
{
|
||||
List<String> hidden = new ArrayList<>();
|
||||
for(String fieldName : searchableFields)
|
||||
{
|
||||
QFieldMetaData field = table.getFields().get(fieldName);
|
||||
if(field != null && (Boolean.TRUE.equals(field.getIsHidden()) || (field.getType() != null && field.getType().needsMasked())))
|
||||
if(table.getFields().containsKey(fieldName) && Boolean.TRUE.equals(table.getField(fieldName).getIsHidden()))
|
||||
{
|
||||
hidden.add(fieldName);
|
||||
}
|
||||
}
|
||||
if(!hidden.isEmpty())
|
||||
{
|
||||
LOG.warn("Hidden and password fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
|
||||
LOG.warn("Hidden fields are excluded from Quick Search indexing", logPair("tableName", table.getName()), logPair("fields", hidden));
|
||||
searchableFields.removeAll(hidden);
|
||||
hidden.forEach(fieldWeights::remove);
|
||||
hidden.forEach(fieldIncludeLabels::remove);
|
||||
|
||||
@@ -17,15 +17,12 @@
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@@ -37,8 +34,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticEventRecorder;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsRateLimiter;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher;
|
||||
@@ -62,8 +57,6 @@ public class QuickSearchRuntime
|
||||
|
||||
private static final long ENABLED_CACHE_TTL_MILLIS = 60_000L;
|
||||
|
||||
public static final long RECORD_SEARCH_BACKOFF_MILLIS = 30_000L;
|
||||
|
||||
private final QuickSearchQBitConfig config;
|
||||
|
||||
private List<QuickSearchableTableConfig> discoveredTables = new ArrayList<>();
|
||||
@@ -71,21 +64,11 @@ public class QuickSearchRuntime
|
||||
|
||||
private volatile QuickSearchOpenSearchClient client;
|
||||
private volatile IndexEventPublisher publisher;
|
||||
private volatile boolean indexReady = false;
|
||||
private volatile boolean closed = false;
|
||||
private volatile long recordSearchBackoffUntilMillis = 0L;
|
||||
private volatile boolean indexReady = false;
|
||||
private volatile boolean closed = false;
|
||||
|
||||
private final Map<String, CachedFlag> enabledCache = new ConcurrentHashMap<>();
|
||||
|
||||
private volatile DiagnosticEventRecorder diagnosticEventRecorder;
|
||||
private final DiagnosticsRateLimiter diagnosticsRateLimiter = new DiagnosticsRateLimiter();
|
||||
private final AtomicBoolean validationRunning = new AtomicBoolean(false);
|
||||
private final AtomicLong recordSearchFailures = new AtomicLong();
|
||||
private final AtomicLong backoffDeclinedClaims = new AtomicLong();
|
||||
private final Instant startedAt = Instant.now();
|
||||
private volatile Instant lastRecordSearchFailureTime;
|
||||
private volatile String lastRecordSearchFailureMessage;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -319,179 +302,6 @@ public class QuickSearchRuntime
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record that a core record search through OpenSearch failed. For the next
|
||||
** RECORD_SEARCH_BACKOFF_MILLIS the record-search provider claims no tables,
|
||||
** so core searches them itself at once instead of waiting for another
|
||||
** timeout on every search while OpenSearch is down.
|
||||
*******************************************************************************/
|
||||
public void markRecordSearchFailed()
|
||||
{
|
||||
markRecordSearchFailed(null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** As markRecordSearchFailed(), also counting the failure and keeping its
|
||||
** message and time for the diagnostics.
|
||||
*******************************************************************************/
|
||||
public void markRecordSearchFailed(String message)
|
||||
{
|
||||
recordSearchBackoffUntilMillis = System.currentTimeMillis() + RECORD_SEARCH_BACKOFF_MILLIS;
|
||||
recordSearchFailures.incrementAndGet();
|
||||
lastRecordSearchFailureTime = Instant.now();
|
||||
lastRecordSearchFailureMessage = message;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Milliseconds left in the current record-search backoff (0 when none).
|
||||
*******************************************************************************/
|
||||
public long getRecordSearchBackoffRemainingMillis()
|
||||
{
|
||||
return (Math.max(0L, recordSearchBackoffUntilMillis - System.currentTimeMillis()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count a table the provider declined because of a backoff (core searched
|
||||
** it with its own search instead).
|
||||
*******************************************************************************/
|
||||
public void countBackoffDeclinedClaim()
|
||||
{
|
||||
backoffDeclinedClaims.incrementAndGet();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record searches through OpenSearch that failed since this runtime started.
|
||||
*******************************************************************************/
|
||||
public long getRecordSearchFailureCount()
|
||||
{
|
||||
return (recordSearchFailures.get());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Table searches core ran itself because the provider was backing off,
|
||||
** since this runtime started.
|
||||
*******************************************************************************/
|
||||
public long getBackoffDeclinedClaimCount()
|
||||
{
|
||||
return (backoffDeclinedClaims.get());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** When the last record search through OpenSearch failed (null when none).
|
||||
*******************************************************************************/
|
||||
public Instant getLastRecordSearchFailureTime()
|
||||
{
|
||||
return (lastRecordSearchFailureTime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The last record-search failure's message (null when none).
|
||||
*******************************************************************************/
|
||||
public String getLastRecordSearchFailureMessage()
|
||||
{
|
||||
return (lastRecordSearchFailureMessage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** When this runtime was created.
|
||||
*******************************************************************************/
|
||||
public Instant getStartedAt()
|
||||
{
|
||||
return (startedAt);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The diagnostic event recorder, built on first use.
|
||||
*******************************************************************************/
|
||||
public DiagnosticEventRecorder getDiagnosticEventRecorder()
|
||||
{
|
||||
DiagnosticEventRecorder existing = diagnosticEventRecorder;
|
||||
if(existing != null)
|
||||
{
|
||||
return (existing);
|
||||
}
|
||||
synchronized(this)
|
||||
{
|
||||
if(diagnosticEventRecorder == null)
|
||||
{
|
||||
diagnosticEventRecorder = new DiagnosticEventRecorder(config);
|
||||
}
|
||||
return (diagnosticEventRecorder);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The per-user rate limiter of the diagnostic tools.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsRateLimiter getDiagnosticsRateLimiter()
|
||||
{
|
||||
return (diagnosticsRateLimiter);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Claim the single validation slot of this JVM; false when a validation is
|
||||
** already running. Release with endValidation().
|
||||
*******************************************************************************/
|
||||
public boolean beginValidation()
|
||||
{
|
||||
return (validationRunning.compareAndSet(false, true));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Release the validation slot.
|
||||
*******************************************************************************/
|
||||
public void endValidation()
|
||||
{
|
||||
validationRunning.set(false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether core record search is backing off after a recent failure.
|
||||
*******************************************************************************/
|
||||
public boolean isRecordSearchBackingOff()
|
||||
{
|
||||
return (System.currentTimeMillis() < recordSearchBackoffUntilMillis);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** End a record-search backoff early (an operator or test knows OpenSearch is
|
||||
** back).
|
||||
*******************************************************************************/
|
||||
public void clearRecordSearchBackoff()
|
||||
{
|
||||
recordSearchBackoffUntilMillis = 0L;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the enabled flag from the quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
@@ -590,11 +400,6 @@ public class QuickSearchRuntime
|
||||
}
|
||||
}
|
||||
|
||||
if(diagnosticEventRecorder != null)
|
||||
{
|
||||
diagnosticEventRecorder.close();
|
||||
}
|
||||
|
||||
if(client != null)
|
||||
{
|
||||
client.close();
|
||||
|
||||
@@ -46,8 +46,6 @@ import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
@@ -455,34 +453,14 @@ public class QuickSearchAction
|
||||
*******************************************************************************/
|
||||
private static SearchResponse<OpenSearchDocument> runSearch(QuickSearchOpenSearchClient client, String term, List<String> tables, int fetchSize, int offset, List<QuickSearchableTableConfig> tableConfigs, String preference) throws QException
|
||||
{
|
||||
long start = System.currentTimeMillis();
|
||||
try
|
||||
{
|
||||
return (client.search(term, tables, fetchSize, offset, tableConfigs, preference));
|
||||
}
|
||||
catch(OpenSearchException e)
|
||||
{
|
||||
QException wrapped = new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e);
|
||||
recordFailure(tables, term, start, wrapped);
|
||||
throw (wrapped);
|
||||
throw (new QException("Quick search failed: " + (e.error() != null && e.error().reason() != null ? e.error().reason() : e.getMessage()), e));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
recordFailure(tables, term, start, e);
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record a failed search as a diagnostic event (the term's length only,
|
||||
** never the term).
|
||||
*******************************************************************************/
|
||||
private static void recordFailure(List<String> tables, String term, long start, Exception e)
|
||||
{
|
||||
DiagnosticsSupport.recordSearchFailure(QuickSearchRuntime.get(), QuickSearchDiagnosticEvent.SOURCE_QUICK_SEARCH_ACTION, tables, term,
|
||||
System.currentTimeMillis() - start, e, "term length " + (term == null ? 0 : term.length()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
-169
@@ -1,169 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.actions;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serves QQQ core record search (RecordSearchAction, POST /qqq/v1/search,
|
||||
** the Next UI global search) from the Quick Search index, for the tables
|
||||
** this QBit indexes. Registered on the QInstance by QuickSearchQBitProducer
|
||||
** when serveCoreRecordSearch is on (the default).
|
||||
**
|
||||
** Core creates a new instance for every search, so this class holds no
|
||||
** state: it resolves the QBit's QuickSearchRuntime (client, tables, config)
|
||||
** from QContext.
|
||||
**
|
||||
** A table is claimed when it is indexed by this QBit and enabled, and core
|
||||
** record search is not backing off after a recent failure. search() sends
|
||||
** one _msearch request with a top-N search per table and returns record ids
|
||||
** only, best first; core re-reads them by primary key through QueryAction,
|
||||
** so table permissions, record security locks and record labels are core's
|
||||
** job, and ids of deleted records drop out. Each table gets
|
||||
** IDS_PER_LIMIT_MULTIPLIER x limitPerTable ids (at most
|
||||
** RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE) so dropouts do not leave
|
||||
** the table short.
|
||||
**
|
||||
** The request is bounded by recordSearchTimeoutMillis. Any failure throws,
|
||||
** and core then searches the claimed tables with its own LIKE search; the
|
||||
** failure also starts a short backoff (QuickSearchRuntime
|
||||
** RECORD_SEARCH_BACKOFF_MILLIS) during which no tables are claimed.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchRecordSearchProvider implements RecordSearchProviderInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchRecordSearchProvider.class);
|
||||
|
||||
public static final int IDS_PER_LIMIT_MULTIPLIER = 4;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Claim tables this QBit indexes and that are enabled, unless core record
|
||||
** search is off or backing off. Cheap: an in-memory lookup plus the
|
||||
** runtime's cached enabled flag.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean claimsTable(QTableMetaData table)
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null || table == null || !Boolean.TRUE.equals(runtime.getConfig().getServeCoreRecordSearch()))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
if(runtime.getTableConfig(table.getName()) == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
if(runtime.isRecordSearchBackingOff())
|
||||
{
|
||||
runtime.countBackoffDeclinedClaim();
|
||||
return (false);
|
||||
}
|
||||
|
||||
return (runtime.isTableEnabled(table.getName()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ids of the best matches in each requested table, in relevance order.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RecordSearchOutput search(RecordSearchInput input) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.getOrThrow();
|
||||
QuickSearchQBitConfig config = runtime.getConfig();
|
||||
|
||||
List<RecordSearchResult> results = new ArrayList<>();
|
||||
String term = IndexingUtils.normalizeSearchText(input.getSearchTerm());
|
||||
if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
|
||||
{
|
||||
return (new RecordSearchOutput().withResults(results));
|
||||
}
|
||||
|
||||
List<String> tableNames = new ArrayList<>();
|
||||
for(String tableName : new LinkedHashSet<>(CollectionUtils.nonNullList(input.getTableNames())))
|
||||
{
|
||||
if(runtime.getTableConfig(tableName) != null)
|
||||
{
|
||||
tableNames.add(tableName);
|
||||
}
|
||||
}
|
||||
if(tableNames.isEmpty())
|
||||
{
|
||||
return (new RecordSearchOutput().withResults(results));
|
||||
}
|
||||
|
||||
int limitPerTable = input.getLimitPerTable() == null || input.getLimitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : input.getLimitPerTable();
|
||||
int idsPerTable = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * IDS_PER_LIMIT_MULTIPLIER);
|
||||
int timeoutMillis = config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis();
|
||||
|
||||
Map<String, List<String>> idsByTable;
|
||||
long start = System.currentTimeMillis();
|
||||
try
|
||||
{
|
||||
idsByTable = runtime.getClient().searchRecordIdsPerTable(term, tableNames, idsPerTable, runtime.getDiscoveredTables(), timeoutMillis);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
runtime.markRecordSearchFailed(e.getMessage());
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// the user's term is not recorded (it may hold personal data); its //
|
||||
// length is, so a length-related failure is still visible //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
DiagnosticsSupport.recordSearchFailure(runtime, QuickSearchDiagnosticEvent.SOURCE_CORE_RECORD_SEARCH, tableNames, term, System.currentTimeMillis() - start, e,
|
||||
"term length " + term.length() + ", " + idsPerTable + " ids per table, timeout " + timeoutMillis + " ms; core searched these tables itself");
|
||||
LOG.info("Quick Search record search failed; core searches these tables itself, and Quick Search claims none for a while", logPair("tables", tableNames),
|
||||
logPair("backoffMillis", QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS), logPair("error", e.getMessage()));
|
||||
throw (e instanceof QException qe ? qe : new QException("Quick Search record search failed: " + e.getMessage(), e));
|
||||
}
|
||||
|
||||
for(String tableName : tableNames)
|
||||
{
|
||||
for(String recordId : CollectionUtils.nonNullList(idsByTable.get(tableName)))
|
||||
{
|
||||
results.add(new RecordSearchResult().withTableName(tableName).withRecordId(recordId));
|
||||
}
|
||||
}
|
||||
|
||||
return (new RecordSearchOutput().withResults(results));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The analyzer tester: run _analyze on the live index with its two
|
||||
** analyzers. The text is analyzed as it would be indexed (standard
|
||||
** tokenizer, lowercase, edge n-grams of 2 to 20 characters) and as a query
|
||||
** (standard tokenizer, lowercase); with a query, each query token is marked
|
||||
** found or not among the text's indexed tokens, which is how matching works
|
||||
** (every query token must be found). Read-only.
|
||||
*******************************************************************************/
|
||||
public class AnalyzerTester
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** The tokens of the text (both analyzers) and of the query.
|
||||
*******************************************************************************/
|
||||
public record Result(String text, String query, List<AnalyzedToken> indexTokens, List<AnalyzedToken> textAsQueryTokens, List<AnalyzedToken> queryTokens)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Analyze text (and an optional query) with the index's analyzers.
|
||||
*******************************************************************************/
|
||||
public Result analyze(QuickSearchRuntime runtime, String text, String query) throws QException
|
||||
{
|
||||
QuickSearchDiagnosticsOperations ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
|
||||
String safeText = text == null ? "" : text;
|
||||
return (new Result(safeText, query,
|
||||
ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, safeText),
|
||||
ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, safeText),
|
||||
StringUtils.hasContent(query) ? ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, query) : null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render a result as HTML.
|
||||
*******************************************************************************/
|
||||
public static String render(Result result)
|
||||
{
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open();
|
||||
html.heading("Analyzer tokens");
|
||||
if(result.text().length() > QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH)
|
||||
{
|
||||
html.status(Level.WARN, "Only the first " + QuickSearchDiagnosticsOperations.MAX_ANALYZE_TEXT_LENGTH + " characters were analyzed.");
|
||||
}
|
||||
|
||||
Set<String> indexed = new LinkedHashSet<>();
|
||||
result.indexTokens().forEach(t -> indexed.add(t.token()));
|
||||
|
||||
if(result.queryTokens() != null)
|
||||
{
|
||||
html.subheading("Does the query match this text?");
|
||||
List<DiagnosticsHtml.Check> checks = new ArrayList<>();
|
||||
boolean all = !result.queryTokens().isEmpty();
|
||||
for(AnalyzedToken token : result.queryTokens())
|
||||
{
|
||||
boolean found = indexed.contains(token.token());
|
||||
all &= found;
|
||||
String why = found ? "found among the indexed tokens"
|
||||
: token.token().length() < 2 ? "one character: never indexed (n-grams start at 2)"
|
||||
: token.token().length() > 20 ? "longer than 20 characters: never indexed (n-grams stop at 20)"
|
||||
: "no word in the text starts with it";
|
||||
checks.add(new DiagnosticsHtml.Check("\"" + token.token() + "\"", found ? Level.OK : Level.FAIL, why));
|
||||
}
|
||||
html.status(all ? Level.OK : Level.FAIL, all ? "Every query token is found: the searchable-text clause matches."
|
||||
: "At least one query token is missing: with AND semantics, a document with only this text does not match.");
|
||||
html.checks(checks);
|
||||
}
|
||||
|
||||
html.subheading("As indexed (" + QuickSearchOpenSearchClient.INDEX_ANALYZER + "): " + result.indexTokens().size() + " token(s)");
|
||||
html.paragraph(join(result.indexTokens()));
|
||||
html.subheading("As a query (" + QuickSearchOpenSearchClient.SEARCH_ANALYZER + "): " + result.textAsQueryTokens().size() + " token(s)");
|
||||
html.paragraph(join(result.textAsQueryTokens()));
|
||||
if(result.queryTokens() != null)
|
||||
{
|
||||
html.subheading("Query tokens: " + result.queryTokens().size());
|
||||
html.paragraph(join(result.queryTokens()));
|
||||
}
|
||||
return (html.close().toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tokens as "token (position)" separated by commas.
|
||||
*******************************************************************************/
|
||||
static String join(List<AnalyzedToken> tokens)
|
||||
{
|
||||
if(tokens == null || tokens.isEmpty())
|
||||
{
|
||||
return ("(none)");
|
||||
}
|
||||
List<String> parts = new ArrayList<>();
|
||||
for(AnalyzedToken token : tokens)
|
||||
{
|
||||
parts.add(token.token() + " @" + token.position());
|
||||
}
|
||||
return (String.join(", ", parts));
|
||||
}
|
||||
|
||||
}
|
||||
-259
@@ -1,259 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.time.Instant;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Writes quickSearchDiagnosticEvent rows: failed searches, diagnostic tool
|
||||
** runs (the audit trail) and refused runs.
|
||||
** Best effort and off the caller's thread: events are queued (at most
|
||||
** QUEUE_CAPACITY) and inserted by one daemon thread as the system user, so
|
||||
** a slow or failing database never slows or fails a search. At most
|
||||
** MAX_EVENTS_PER_MINUTE events are written per JVM per minute; the rest are
|
||||
** counted as dropped (see getDroppedCount), so a search outage cannot flood
|
||||
** the table. A no-op when the diagnostic tables are not configured.
|
||||
*******************************************************************************/
|
||||
public class DiagnosticEventRecorder implements AutoCloseable
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(DiagnosticEventRecorder.class);
|
||||
|
||||
public static final int MAX_EVENTS_PER_MINUTE = 120;
|
||||
public static final int QUEUE_CAPACITY = 500;
|
||||
|
||||
private final QuickSearchQBitConfig config;
|
||||
private final ThreadPoolExecutor executor;
|
||||
private final AtomicLong droppedCount = new AtomicLong();
|
||||
private final AtomicLong failedCount = new AtomicLong();
|
||||
private final String hostName;
|
||||
|
||||
private long windowStartMillis = 0L;
|
||||
private int windowCount = 0;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
*******************************************************************************/
|
||||
public DiagnosticEventRecorder(QuickSearchQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
this.hostName = resolveHostName();
|
||||
this.executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(QUEUE_CAPACITY), runnable ->
|
||||
{
|
||||
Thread thread = new Thread(runnable, "quick-search-diagnostic-events");
|
||||
thread.setDaemon(true);
|
||||
return (thread);
|
||||
}, (runnable, pool) -> droppedCount.incrementAndGet());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Queue an event for insert. Fills in host name and create date, bounds
|
||||
** string lengths, and uses the current session's user when none is set.
|
||||
*******************************************************************************/
|
||||
public void record(QuickSearchDiagnosticEvent event)
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
String tableName = config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
|
||||
if(event == null || qInstance == null || !Boolean.TRUE.equals(config.getEnableDiagnosticTools()) || qInstance.getTable(tableName) == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if(!admit())
|
||||
{
|
||||
droppedCount.incrementAndGet();
|
||||
return;
|
||||
}
|
||||
|
||||
if(event.getUserName() == null)
|
||||
{
|
||||
event.setUserName(DiagnosticsSupport.currentUserName());
|
||||
}
|
||||
|
||||
event.withHostName(truncate(hostName, 100))
|
||||
.withCreateDate(Instant.now())
|
||||
.withSource(truncate(event.getSource(), 50))
|
||||
.withUserName(truncate(event.getUserName(), 250))
|
||||
.withTableNames(truncate(event.getTableNames(), 1000))
|
||||
.withSearchTerm(truncate(event.getSearchTerm(), 100))
|
||||
.withErrorClass(truncate(event.getErrorClass(), 250))
|
||||
.withErrorMessage(truncate(event.getErrorMessage(), 4000))
|
||||
.withDetail(truncate(event.getDetail(), 4000));
|
||||
|
||||
executor.execute(() -> insert(qInstance, tableName, event));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert one event as the system user. Failures are logged and counted.
|
||||
*******************************************************************************/
|
||||
private void insert(QInstance qInstance, String tableName, QuickSearchDiagnosticEvent event)
|
||||
{
|
||||
QSession session = new QSystemUserSession();
|
||||
try
|
||||
{
|
||||
QContext.init(qInstance, session);
|
||||
new InsertAction().execute(new InsertInput(tableName).withRecordEntity(event));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failedCount.incrementAndGet();
|
||||
LOG.warn("Could not record a Quick Search diagnostic event", e, logPair("eventType", event.getEventType()), logPair("source", event.getSource()));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether one more event fits in the current minute's budget.
|
||||
*******************************************************************************/
|
||||
private synchronized boolean admit()
|
||||
{
|
||||
long now = System.currentTimeMillis();
|
||||
if(now - windowStartMillis >= 60_000L)
|
||||
{
|
||||
windowStartMillis = now;
|
||||
windowCount = 0;
|
||||
}
|
||||
if(windowCount >= MAX_EVENTS_PER_MINUTE)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
windowCount++;
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Wait (up to timeoutMillis) until every queued event has been written.
|
||||
** For tests and orderly shutdown.
|
||||
*******************************************************************************/
|
||||
public void flush(long timeoutMillis)
|
||||
{
|
||||
try
|
||||
{
|
||||
executor.submit(() ->
|
||||
{
|
||||
}).get(timeoutMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.debug("Diagnostic event flush did not complete", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Events not written because the queue or the per-minute budget was full.
|
||||
*******************************************************************************/
|
||||
public long getDroppedCount()
|
||||
{
|
||||
return (droppedCount.get());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Events whose insert failed.
|
||||
*******************************************************************************/
|
||||
public long getFailedCount()
|
||||
{
|
||||
return (failedCount.get());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stop the writer after the queued events (up to 2 seconds).
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
executor.shutdown();
|
||||
try
|
||||
{
|
||||
if(!executor.awaitTermination(2, TimeUnit.SECONDS))
|
||||
{
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
catch(InterruptedException e)
|
||||
{
|
||||
executor.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** This JVM's host name (the pod name in Kubernetes).
|
||||
*******************************************************************************/
|
||||
static String resolveHostName()
|
||||
{
|
||||
String fromEnv = System.getenv("HOSTNAME");
|
||||
if(fromEnv != null && !fromEnv.isBlank())
|
||||
{
|
||||
return (fromEnv);
|
||||
}
|
||||
try
|
||||
{
|
||||
return (InetAddress.getLocalHost().getHostName());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
return ("unknown");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
static String truncate(String value, int max)
|
||||
{
|
||||
return (value == null || value.length() <= max ? value : value.substring(0, max));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,192 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Source-table reads for the diagnostics, in the current session (so table
|
||||
** permissions and record security locks apply, as for any query the user
|
||||
** runs). Records are read the way indexing reads them: display values
|
||||
** generated and possible values translated, so a document built from them
|
||||
** is exactly the document indexing would write.
|
||||
*******************************************************************************/
|
||||
public final class DiagnosticRecordReads
|
||||
{
|
||||
public static final int MAX_IDS_PER_QUERY = 1_000;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class - no instances.
|
||||
*******************************************************************************/
|
||||
private DiagnosticRecordReads()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Records of a table by primary key (as strings), keyed by primary key
|
||||
** string, read as indexing reads them. Ids that do not exist or that the
|
||||
** session cannot read are absent. Ids that do not convert to the key's
|
||||
** type are skipped.
|
||||
*******************************************************************************/
|
||||
public static Map<String, QRecord> loadForIndexing(String tableName, Collection<String> recordIds) throws QException
|
||||
{
|
||||
QTableMetaData table = requireTable(tableName);
|
||||
String pk = table.getPrimaryKeyField();
|
||||
Map<String, QRecord> records = new LinkedHashMap<>();
|
||||
List<Serializable> ids = typedIds(table, recordIds);
|
||||
|
||||
for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
|
||||
queryInput.setShouldGenerateDisplayValues(true);
|
||||
queryInput.setShouldTranslatePossibleValues(true);
|
||||
for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
records.put(record.getValueString(pk), record);
|
||||
}
|
||||
}
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Up to limit records of a table ordered by primary key (ascending or
|
||||
** descending), read as indexing reads them.
|
||||
*******************************************************************************/
|
||||
public static List<QRecord> loadEdge(String tableName, boolean ascending, int limit) throws QException
|
||||
{
|
||||
QTableMetaData table = requireTable(tableName);
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(table.getPrimaryKeyField(), ascending)).withLimit(limit));
|
||||
queryInput.setShouldGenerateDisplayValues(true);
|
||||
queryInput.setShouldTranslatePossibleValues(true);
|
||||
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The primary keys (as strings) among recordIds that exist and that the
|
||||
** session can read.
|
||||
*******************************************************************************/
|
||||
public static List<String> existingIds(String tableName, Collection<String> recordIds) throws QException
|
||||
{
|
||||
QTableMetaData table = requireTable(tableName);
|
||||
String pk = table.getPrimaryKeyField();
|
||||
List<String> found = new ArrayList<>();
|
||||
List<Serializable> ids = typedIds(table, recordIds);
|
||||
|
||||
for(int start = 0; start < ids.size(); start += MAX_IDS_PER_QUERY)
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(pk, QCriteriaOperator.IN, ids.subList(start, Math.min(ids.size(), start + MAX_IDS_PER_QUERY)))));
|
||||
for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
found.add(record.getValueString(pk));
|
||||
}
|
||||
}
|
||||
return (found);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Row count of a table in the current session.
|
||||
*******************************************************************************/
|
||||
public static Integer count(String tableName) throws QException
|
||||
{
|
||||
CountInput countInput = new CountInput();
|
||||
countInput.setTableName(tableName);
|
||||
return (new CountAction().execute(countInput).getCount());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ids converted to the primary key's type; ids that do not convert are
|
||||
** skipped (they cannot exist).
|
||||
*******************************************************************************/
|
||||
static List<Serializable> typedIds(QTableMetaData table, Collection<String> recordIds)
|
||||
{
|
||||
QFieldMetaData pkField = table.getField(table.getPrimaryKeyField());
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
for(String recordId : CollectionUtils.nonNullCollection(recordIds))
|
||||
{
|
||||
try
|
||||
{
|
||||
Serializable id = ValueUtils.getValueAsFieldType(pkField.getType(), recordId);
|
||||
if(id != null && !ids.contains(id))
|
||||
{
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
//////////////////////////////////////////////////////////
|
||||
// e.g. "abc" for an integer key: no such record exists //
|
||||
//////////////////////////////////////////////////////////
|
||||
}
|
||||
}
|
||||
return (ids);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData requireTable(String tableName) throws QException
|
||||
{
|
||||
QTableMetaData table = QContext.getQInstance() == null ? null : QContext.getQInstance().getTable(tableName);
|
||||
if(table == null)
|
||||
{
|
||||
throw (new QException("Table [" + tableName + "] is not in the instance"));
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,362 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A small HTML builder for the diagnostic screens (process HTML blocks, the
|
||||
** validation report field and the diagnostics widget). Every text value is
|
||||
** escaped. Layout rules: one column of blocks capped at a readable width,
|
||||
** no fixed widths, and long values (JSON, ids, text) wrap instead of
|
||||
** scrolling, so a screen reads from 360px to 3200px without an inner or
|
||||
** horizontal scrollbar. Colours are mid-tone so they read in light and dark
|
||||
** themes; status is always also spelled out in text.
|
||||
*******************************************************************************/
|
||||
public class DiagnosticsHtml
|
||||
{
|
||||
public static final int MAX_JSON_CHARS = 60_000;
|
||||
|
||||
private static final ObjectMapper PRETTY = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
|
||||
|
||||
private static final String WRAP = "overflow-wrap:anywhere;word-break:break-word;";
|
||||
|
||||
private final StringBuilder html = new StringBuilder();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Severity of a check or a line, rendered as a coloured label.
|
||||
*******************************************************************************/
|
||||
public enum Level
|
||||
{
|
||||
OK("OK", "#16a34a"),
|
||||
INFO("INFO", "#2563eb"),
|
||||
WARN("WARN", "#d97706"),
|
||||
FAIL("FAIL", "#dc2626");
|
||||
|
||||
private final String label;
|
||||
private final String color;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
*******************************************************************************/
|
||||
Level(String label, String color)
|
||||
{
|
||||
this.label = label;
|
||||
this.color = color;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for label
|
||||
*******************************************************************************/
|
||||
public String getLabel()
|
||||
{
|
||||
return (label);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape text for HTML element content and attribute values.
|
||||
*******************************************************************************/
|
||||
public static String escape(Object value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
String text = String.valueOf(value);
|
||||
StringBuilder out = new StringBuilder(text.length() + 16);
|
||||
for(int i = 0; i < text.length(); i++)
|
||||
{
|
||||
char c = text.charAt(i);
|
||||
switch(c)
|
||||
{
|
||||
case '<' -> out.append("<");
|
||||
case '>' -> out.append(">");
|
||||
case '&' -> out.append("&");
|
||||
case '"' -> out.append(""");
|
||||
case '\'' -> out.append("'");
|
||||
default -> out.append(c);
|
||||
}
|
||||
}
|
||||
return (out.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape text but keep OpenSearch highlight tags (em) as mark elements.
|
||||
*******************************************************************************/
|
||||
public static String highlight(String fragment)
|
||||
{
|
||||
return (escape(fragment).replace("<em>", "<mark>").replace("</em>", "</mark>"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A coloured status label.
|
||||
*******************************************************************************/
|
||||
public static String badge(Level level)
|
||||
{
|
||||
return ("<strong style=\"color:" + level.color + "\">" + level.label + "</strong>");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Open the root block.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml open()
|
||||
{
|
||||
html.append("<div style=\"max-width:72rem;").append(WRAP).append("\">");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Close the root block.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml close()
|
||||
{
|
||||
html.append("</div>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A section heading.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml heading(String text)
|
||||
{
|
||||
html.append("<h3>").append(escape(text)).append("</h3>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A sub-heading.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml subheading(String text)
|
||||
{
|
||||
html.append("<h4>").append(escape(text)).append("</h4>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A paragraph of escaped text.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml paragraph(String text)
|
||||
{
|
||||
html.append("<p>").append(escape(text)).append("</p>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A paragraph starting with a status label.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml status(Level level, String text)
|
||||
{
|
||||
html.append("<p>").append(badge(level)).append(" ").append(escape(text)).append("</p>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A list of checks: each line a status label and text.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml checks(List<Check> checks)
|
||||
{
|
||||
if(checks == null || checks.isEmpty())
|
||||
{
|
||||
return (this);
|
||||
}
|
||||
html.append("<ul>");
|
||||
for(Check check : checks)
|
||||
{
|
||||
html.append("<li>").append(badge(check.level())).append(" <strong>").append(escape(check.name())).append("</strong>");
|
||||
if(check.message() != null)
|
||||
{
|
||||
html.append(": ").append(escape(check.message()));
|
||||
}
|
||||
html.append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A list of label: value lines (values escaped). Null values show as
|
||||
** "none".
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml facts(List<String[]> facts)
|
||||
{
|
||||
html.append("<ul>");
|
||||
for(String[] fact : facts)
|
||||
{
|
||||
html.append("<li><strong>").append(escape(fact[0])).append(":</strong> ")
|
||||
.append(fact.length > 1 && fact[1] != null ? escape(fact[1]) : "<em>none</em>").append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A bulleted list of pre-built HTML items (callers escape their parts).
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml rawList(List<String> itemsHtml)
|
||||
{
|
||||
if(itemsHtml == null || itemsHtml.isEmpty())
|
||||
{
|
||||
return (this);
|
||||
}
|
||||
html.append("<ul>");
|
||||
for(String item : itemsHtml)
|
||||
{
|
||||
html.append("<li>").append(item).append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A preformatted block that wraps (no horizontal scrolling).
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml pre(String text)
|
||||
{
|
||||
html.append("<pre style=\"white-space:pre-wrap;").append(WRAP).append("\">").append(escape(text)).append("</pre>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A collapsed block with a summary line and pretty-printed JSON inside.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml jsonDetails(String summary, String json)
|
||||
{
|
||||
html.append("<details><summary>").append(escape(summary)).append("</summary>");
|
||||
pre(prettyJson(json));
|
||||
html.append("</details>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A collapsed block with a summary line and preformatted text inside.
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml textDetails(String summary, String text)
|
||||
{
|
||||
html.append("<details><summary>").append(escape(summary)).append("</summary>");
|
||||
pre(text);
|
||||
html.append("</details>");
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append pre-built HTML (callers escape their parts).
|
||||
*******************************************************************************/
|
||||
public DiagnosticsHtml raw(String rawHtml)
|
||||
{
|
||||
html.append(rawHtml);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Pretty-print JSON (or return the text as-is when it is not JSON),
|
||||
** truncated to MAX_JSON_CHARS.
|
||||
*******************************************************************************/
|
||||
public static String prettyJson(String json)
|
||||
{
|
||||
if(json == null)
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
String pretty;
|
||||
try
|
||||
{
|
||||
pretty = PRETTY.writeValueAsString(PRETTY.readTree(json));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
pretty = json;
|
||||
}
|
||||
return (pretty.length() > MAX_JSON_CHARS ? pretty.substring(0, MAX_JSON_CHARS) + "\n... (truncated at " + MAX_JSON_CHARS + " characters)" : pretty);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Pretty-print a JSON node.
|
||||
*******************************************************************************/
|
||||
public static String prettyJson(JsonNode node)
|
||||
{
|
||||
return (node == null ? "" : prettyJson(node.toString()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The built HTML.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return (html.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One check: name, level and message.
|
||||
*******************************************************************************/
|
||||
public record Check(String name, Level level, String message)
|
||||
{
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.LongSupplier;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sliding one-minute window of diagnostic tool runs per key (user and
|
||||
** tool), per JVM. tryAcquire admits a run when fewer than the limit ran in
|
||||
** the last 60 seconds.
|
||||
*******************************************************************************/
|
||||
public class DiagnosticsRateLimiter
|
||||
{
|
||||
private static final long WINDOW_MILLIS = 60_000L;
|
||||
|
||||
private final Map<String, Deque<Long>> runsByKey = new ConcurrentHashMap<>();
|
||||
private final LongSupplier clock;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor (system clock)
|
||||
*******************************************************************************/
|
||||
public DiagnosticsRateLimiter()
|
||||
{
|
||||
this(System::currentTimeMillis);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor with a clock (tests)
|
||||
*******************************************************************************/
|
||||
public DiagnosticsRateLimiter(LongSupplier clock)
|
||||
{
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Admit one run for the key when fewer than perMinute ran in the last
|
||||
** minute; a null or non-positive limit admits everything.
|
||||
*******************************************************************************/
|
||||
public boolean tryAcquire(String key, Integer perMinute)
|
||||
{
|
||||
if(perMinute == null || perMinute <= 0)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
long now = clock.getAsLong();
|
||||
Deque<Long> runs = runsByKey.computeIfAbsent(key, k -> new ArrayDeque<>());
|
||||
synchronized(runs)
|
||||
{
|
||||
while(!runs.isEmpty() && now - runs.peekFirst() >= WINDOW_MILLIS)
|
||||
{
|
||||
runs.pollFirst();
|
||||
}
|
||||
if(runs.size() >= perMinute)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
runs.addLast(now);
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.Collection;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Shared pieces of the diagnostic tools: tool names, the current user, the
|
||||
** per-user rate limit, and recording tool runs and search failures as
|
||||
** quickSearchDiagnosticEvent rows (the audit trail).
|
||||
*******************************************************************************/
|
||||
public final class DiagnosticsSupport
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(DiagnosticsSupport.class);
|
||||
|
||||
public static final String TOOL_PLAYGROUND = "SEARCH_PLAYGROUND";
|
||||
public static final String TOOL_INSPECT_RECORD = "INSPECT_RECORD";
|
||||
public static final String TOOL_REINDEX_RECORDS = "REINDEX_RECORDS";
|
||||
public static final String TOOL_VALIDATE_INDEX = "VALIDATE_INDEX";
|
||||
public static final String TOOL_ANALYZE_TEXT = "ANALYZE_TEXT";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class - no instances.
|
||||
*******************************************************************************/
|
||||
private DiagnosticsSupport()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The session's user id reference (else full name), or null.
|
||||
*******************************************************************************/
|
||||
public static String currentUserName()
|
||||
{
|
||||
QSession session = QContext.getQSession();
|
||||
if(session == null || session.getUser() == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
return (session.getUser().getIdReference() != null ? session.getUser().getIdReference() : session.getUser().getFullName());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The runtime of the Quick Search QBit, or a user-facing error saying the
|
||||
** tool is unavailable.
|
||||
*******************************************************************************/
|
||||
public static QuickSearchRuntime requireRuntime() throws QUserFacingException
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null)
|
||||
{
|
||||
throw (new QUserFacingException("Quick Search is not configured in this application, so its diagnostic tools are unavailable."));
|
||||
}
|
||||
return (runtime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Admit one run of a tool for the current user, or refuse it (recording
|
||||
** the refusal) when the user ran diagnosticRunsPerMinute tools in the last
|
||||
** minute.
|
||||
*******************************************************************************/
|
||||
public static void checkRateLimit(QuickSearchRuntime runtime, String tool) throws QUserFacingException
|
||||
{
|
||||
String user = currentUserName();
|
||||
if(runtime.getDiagnosticsRateLimiter().tryAcquire((user == null ? "anonymous" : user) + "|" + tool, runtime.getConfig().getDiagnosticRunsPerMinute()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
|
||||
.withEventType(QuickSearchDiagnosticEvent.TYPE_RATE_LIMITED)
|
||||
.withSource(tool)
|
||||
.withOutcome(QuickSearchDiagnosticEvent.OUTCOME_REFUSED));
|
||||
throw (new QUserFacingException("Too many Quick Search diagnostic runs in the last minute (limit " + runtime.getConfig().getDiagnosticRunsPerMinute()
|
||||
+ " per tool). Wait a minute and try again."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record a tool run (audit): who ran which tool on which tables with which
|
||||
** term, how long it took, and its outcome.
|
||||
*******************************************************************************/
|
||||
public static void recordToolRun(QuickSearchRuntime runtime, String tool, String eventType, Collection<String> tableNames, String term, long startMillis, Throwable error, String detail)
|
||||
{
|
||||
try
|
||||
{
|
||||
runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
|
||||
.withEventType(eventType)
|
||||
.withSource(tool)
|
||||
.withOutcome(error == null ? QuickSearchDiagnosticEvent.OUTCOME_OK : QuickSearchDiagnosticEvent.OUTCOME_ERROR)
|
||||
.withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
|
||||
.withSearchTerm(term)
|
||||
.withDurationMillis((int) Math.min(Integer.MAX_VALUE, System.currentTimeMillis() - startMillis))
|
||||
.withErrorClass(error == null ? null : error.getClass().getName())
|
||||
.withErrorMessage(error == null ? null : error.getMessage())
|
||||
.withDetail(detail));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.debug("Could not queue a diagnostic event", e);
|
||||
}
|
||||
|
||||
LOG.info("Quick Search diagnostic tool run", logPair("tool", tool), logPair("user", currentUserName()), logPair("tables", tableNames),
|
||||
logPair("millis", System.currentTimeMillis() - startMillis), logPair("error", error == null ? null : error.getMessage()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record a failed search (OpenSearch error or timeout). userTerm is the
|
||||
** user's term: never stored, and redacted from the message and detail.
|
||||
*******************************************************************************/
|
||||
public static void recordSearchFailure(QuickSearchRuntime runtime, String source, Collection<String> tableNames, String userTerm, long durationMillis, Throwable error, String detail)
|
||||
{
|
||||
if(runtime == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
Throwable cause = rootCause(error);
|
||||
runtime.getDiagnosticEventRecorder().record(new QuickSearchDiagnosticEvent()
|
||||
.withEventType(QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED)
|
||||
.withSource(source)
|
||||
.withOutcome(QuickSearchDiagnosticEvent.OUTCOME_ERROR)
|
||||
.withTableNames(tableNames == null || tableNames.isEmpty() ? null : String.join(",", tableNames))
|
||||
.withDurationMillis((int) Math.min(Integer.MAX_VALUE, durationMillis))
|
||||
.withErrorClass(cause == null ? null : cause.getClass().getName())
|
||||
.withErrorMessage(redactTerm(error == null ? null : error.getMessage(), userTerm))
|
||||
.withDetail(redactTerm(detail, userTerm)));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.debug("Could not queue a search failure event", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The text with the user's search term, and each of its words of three or
|
||||
** more characters, replaced by a marker (case-insensitive): error messages
|
||||
** (ours and OpenSearch's) can echo the term, which may hold personal data.
|
||||
*******************************************************************************/
|
||||
public static String redactTerm(String text, String userTerm)
|
||||
{
|
||||
if(text == null || userTerm == null || userTerm.isBlank())
|
||||
{
|
||||
return (text);
|
||||
}
|
||||
String redacted = text.replaceAll("(?i)" + java.util.regex.Pattern.quote(userTerm.trim()), "[search term]");
|
||||
for(String word : userTerm.trim().split("\\s+"))
|
||||
{
|
||||
if(word.length() >= 3)
|
||||
{
|
||||
redacted = redacted.replaceAll("(?i)" + java.util.regex.Pattern.quote(word), "[search term]");
|
||||
}
|
||||
}
|
||||
return (redacted);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The innermost cause of a throwable (the one that names the real error
|
||||
** class: timeout, connection refused, OpenSearch error).
|
||||
*******************************************************************************/
|
||||
public static Throwable rootCause(Throwable error)
|
||||
{
|
||||
Throwable cause = error;
|
||||
int depth = 0;
|
||||
while(cause != null && cause.getCause() != null && cause.getCause() != cause && depth++ < 20)
|
||||
{
|
||||
cause = cause.getCause();
|
||||
}
|
||||
return (cause);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A user-facing error for a tool whose OpenSearch call failed: the tool
|
||||
** reports the failure honestly instead of showing partial data.
|
||||
*******************************************************************************/
|
||||
public static QUserFacingException unavailable(String what, QException e)
|
||||
{
|
||||
return (new QUserFacingException(what + " is unavailable: " + e.getMessage(), e));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,782 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.CapturedContext;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.RawResponse;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate the Quick Search index against the database and its expected
|
||||
** shape, and report every finding with a level (OK, INFO, WARN, FAIL):
|
||||
** - cluster: reachable, cluster and index health, the alias pointing at one
|
||||
** physical index, leftover physical indexes from interrupted rebuilds;
|
||||
** - mapping and settings: mapping version, field types and analyzers of the
|
||||
** fixed fields, the fieldValues dynamic template and every fieldValues.*
|
||||
** field mapped as text, the two analyzers and the edge n-gram filter;
|
||||
** - per table: database rows vs indexed documents, a sample of records
|
||||
** (the lowest and highest primary keys) checked present and current in
|
||||
** the index, a random sample of documents checked to still have a record,
|
||||
** the quickSearchIndex row's status and last run, failed events waiting.
|
||||
** Database reads run as the system user, so counts compare like with like
|
||||
** (the index holds every row whatever the record locks). The report holds
|
||||
** counts, ids and field names, never record values. Read-only, except that
|
||||
** persist() saves the report as a quickSearchValidationRun row.
|
||||
*******************************************************************************/
|
||||
public class IndexValidator
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(IndexValidator.class);
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
|
||||
|
||||
public static final String SCOPE_CLUSTER = "cluster";
|
||||
public static final String SCOPE_MAPPING = "mapping";
|
||||
|
||||
private static final int MAX_LISTED_IDS = 25;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One finding.
|
||||
*******************************************************************************/
|
||||
public record Finding(String scope, String check, Level level, String message)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Per-table numbers.
|
||||
*******************************************************************************/
|
||||
public record TableStats(String tableName, Integer databaseCount, Long indexCount, Integer sampled, List<String> missingIds, Map<String, List<String>> staleFields,
|
||||
Integer documentsSampled, List<String> extraIds)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A validation report.
|
||||
*******************************************************************************/
|
||||
public record Report(String status, Instant startTime, Instant endTime, Integer sampleSize, List<String> tableNames, List<Finding> findings, List<TableStats> tableStats)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Count of findings at a level.
|
||||
*******************************************************************************/
|
||||
public int count(Level level)
|
||||
{
|
||||
return ((int) findings.stream().filter(f -> f.level() == level).count());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate the named tables (empty for every indexed table) with the given
|
||||
** sample size per table.
|
||||
*******************************************************************************/
|
||||
public Report validate(QuickSearchRuntime runtime, List<String> requestedTables, int sampleSize) throws QException
|
||||
{
|
||||
Instant start = Instant.now();
|
||||
List<Finding> findings = new ArrayList<>();
|
||||
List<TableStats> stats = new ArrayList<>();
|
||||
List<String> tables = resolveTables(runtime, requestedTables, findings);
|
||||
int boundedN = Math.max(1, Math.min(sampleSize, QuickSearchQBitConfig.MAX_VALIDATION_SAMPLE_SIZE));
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
|
||||
QContext.withTemporaryContext(new CapturedContext(qInstance, new QSystemUserSession()), () ->
|
||||
{
|
||||
QuickSearchDiagnosticsOperations ops;
|
||||
try
|
||||
{
|
||||
ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Client", Level.FAIL, "The OpenSearch client could not be built: " + e.getMessage()));
|
||||
return;
|
||||
}
|
||||
|
||||
if(!checkCluster(runtime, ops, findings))
|
||||
{
|
||||
return;
|
||||
}
|
||||
checkMapping(ops, findings);
|
||||
checkSettings(ops, findings);
|
||||
|
||||
for(String tableName : tables)
|
||||
{
|
||||
stats.add(checkTable(runtime, ops, runtime.getTableConfig(tableName), boundedN, findings));
|
||||
}
|
||||
});
|
||||
|
||||
String status = findings.stream().anyMatch(f -> f.level() == Level.FAIL) ? QuickSearchValidationRun.STATUS_FAIL
|
||||
: findings.stream().anyMatch(f -> f.level() == Level.WARN) ? QuickSearchValidationRun.STATUS_WARN : QuickSearchValidationRun.STATUS_PASS;
|
||||
return (new Report(status, start, Instant.now(), boundedN, tables, findings, stats));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The requested indexed tables (unknown ones reported), else every indexed
|
||||
** table.
|
||||
*******************************************************************************/
|
||||
private static List<String> resolveTables(QuickSearchRuntime runtime, List<String> requested, List<Finding> findings)
|
||||
{
|
||||
List<String> tables = new ArrayList<>();
|
||||
if(CollectionUtils.nullSafeIsEmpty(requested))
|
||||
{
|
||||
runtime.getDiscoveredTables().forEach(t -> tables.add(t.getTableName()));
|
||||
return (tables);
|
||||
}
|
||||
for(String tableName : requested)
|
||||
{
|
||||
if(runtime.getTableConfig(tableName) == null)
|
||||
{
|
||||
findings.add(new Finding(tableName, "Configured", Level.WARN, "Quick Search does not index this table; skipped."));
|
||||
}
|
||||
else if(!tables.contains(tableName))
|
||||
{
|
||||
tables.add(tableName);
|
||||
}
|
||||
}
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cluster health, alias and physical indexes. Returns false when the index
|
||||
** cannot be reached (the other checks are then skipped).
|
||||
*******************************************************************************/
|
||||
boolean checkCluster(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, List<Finding> findings)
|
||||
{
|
||||
RawResponse health;
|
||||
try
|
||||
{
|
||||
health = ops.clusterHealth();
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "OpenSearch did not answer: " + e.getMessage()));
|
||||
return (false);
|
||||
}
|
||||
|
||||
if(health.status() == 404)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Index", Level.FAIL, "The index alias [" + ops.getIndexName() + "] does not exist. Run the full reindex to create it."));
|
||||
return (false);
|
||||
}
|
||||
if(!health.isOk())
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.FAIL, "Cluster health returned status " + health.status() + ": " + QuickSearchDiagnosticsOperations.errorReason(health.body())));
|
||||
return (false);
|
||||
}
|
||||
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Reachable", Level.OK, "OpenSearch answered (" + health.body().path("cluster_name").asText("cluster") + ", "
|
||||
+ health.body().path("number_of_nodes").asInt() + " node(s))."));
|
||||
findings.add(healthFinding("Cluster health", health.body().path("status").asText("unknown")));
|
||||
|
||||
Iterator<Map.Entry<String, JsonNode>> indexes = health.body().path("indices").fields();
|
||||
while(indexes.hasNext())
|
||||
{
|
||||
Map.Entry<String, JsonNode> index = indexes.next();
|
||||
findings.add(healthFinding("Index health " + index.getKey(), index.getValue().path("status").asText("unknown")));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Set<String> physical = runtime.getClient().getPhysicalIndexes();
|
||||
if(physical.isEmpty())
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.WARN, "[" + ops.getIndexName() + "] is a concrete index, not an alias (created before 1.0). The next full reindex replaces it with an alias."));
|
||||
}
|
||||
else if(physical.size() > 1)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "The alias points at " + physical.size() + " physical indexes " + physical + "; searches and counts see documents twice. Run the full reindex."));
|
||||
}
|
||||
else
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.OK, "The alias points at " + physical.iterator().next() + "."));
|
||||
}
|
||||
|
||||
RawResponse cat = ops.physicalIndexes();
|
||||
if(cat.isOk() && cat.body() != null)
|
||||
{
|
||||
for(JsonNode row : cat.body())
|
||||
{
|
||||
String name = row.path("index").asText();
|
||||
if(!physical.contains(name))
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Leftover index", Level.WARN, "Physical index " + name + " (" + row.path("docs.count").asText("?") + " docs, "
|
||||
+ row.path("store.size").asText("?") + ") is not behind the alias: left by an interrupted rebuild. Delete it to free space."));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_CLUSTER, "Alias", Level.FAIL, "Could not read the alias: " + e.getMessage()));
|
||||
}
|
||||
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static Finding healthFinding(String check, String status)
|
||||
{
|
||||
return (switch(status)
|
||||
{
|
||||
case "green" -> new Finding(SCOPE_CLUSTER, check, Level.OK, "green.");
|
||||
case "yellow" -> new Finding(SCOPE_CLUSTER, check, Level.WARN, "yellow: replica shards are unassigned (expected on a single-node cluster when the index asks for replicas).");
|
||||
default -> new Finding(SCOPE_CLUSTER, check, Level.FAIL, status + ": primary shards are unassigned; searches and writes may fail.");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live mapping against the mapping this version creates.
|
||||
*******************************************************************************/
|
||||
void checkMapping(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
|
||||
{
|
||||
RawResponse response;
|
||||
try
|
||||
{
|
||||
response = ops.mapping();
|
||||
QuickSearchDiagnosticsOperations.requireOk(response, "Mapping read failed");
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_MAPPING, "Mapping", Level.FAIL, e.getMessage()));
|
||||
return;
|
||||
}
|
||||
|
||||
Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
|
||||
while(indexes.hasNext())
|
||||
{
|
||||
Map.Entry<String, JsonNode> index = indexes.next();
|
||||
JsonNode mappings = index.getValue().path("mappings");
|
||||
String prefix = index.getKey() + ": ";
|
||||
|
||||
int version = mappings.path("_meta").path(QuickSearchOpenSearchClient.META_MAPPING_VERSION).asInt(1);
|
||||
findings.add(version >= QuickSearchOpenSearchClient.MAPPING_VERSION
|
||||
? new Finding(SCOPE_MAPPING, "Mapping version", Level.OK, prefix + "version " + version + ".")
|
||||
: new Finding(SCOPE_MAPPING, "Mapping version", Level.FAIL, prefix + "version " + version + ", this release writes " + QuickSearchOpenSearchClient.MAPPING_VERSION + ". Run the full reindex."));
|
||||
|
||||
JsonNode properties = mappings.path("properties");
|
||||
List<String> problems = new ArrayList<>();
|
||||
expectType(properties, "sourceTable", "keyword", problems);
|
||||
expectType(properties, "recordId", "keyword", problems);
|
||||
expectType(properties, "indexedAt", "date", problems);
|
||||
expectText(properties.path("searchableText"), "searchableText", problems);
|
||||
expectText(properties.path("recordLabel"), "recordLabel", problems);
|
||||
|
||||
boolean template = false;
|
||||
for(JsonNode entry : mappings.path("dynamic_templates"))
|
||||
{
|
||||
template |= entry.has("fieldValuesAsText");
|
||||
}
|
||||
if(!template)
|
||||
{
|
||||
problems.add("the fieldValuesAsText dynamic template is missing, so new fields may be mapped as numbers or dates");
|
||||
}
|
||||
if(mappings.path("date_detection").asBoolean(true) || mappings.path("numeric_detection").asBoolean(false))
|
||||
{
|
||||
problems.add("date or numeric detection is on");
|
||||
}
|
||||
|
||||
Iterator<Map.Entry<String, JsonNode>> fields = properties.path("fieldValues").path("properties").fields();
|
||||
int count = 0;
|
||||
while(fields.hasNext())
|
||||
{
|
||||
Map.Entry<String, JsonNode> field = fields.next();
|
||||
expectText(field.getValue(), "fieldValues." + field.getKey(), problems);
|
||||
count++;
|
||||
}
|
||||
|
||||
findings.add(problems.isEmpty()
|
||||
? new Finding(SCOPE_MAPPING, "Field mappings", Level.OK, prefix + "fixed fields and " + count + " fieldValues field(s) mapped as expected.")
|
||||
: new Finding(SCOPE_MAPPING, "Field mappings", Level.FAIL, prefix + String.join("; ", problems) + ". Run the full reindex to rebuild the mapping."));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void expectType(JsonNode properties, String field, String type, List<String> problems)
|
||||
{
|
||||
String actual = properties.path(field).path("type").asText(null);
|
||||
if(!type.equals(actual))
|
||||
{
|
||||
problems.add(field + " is " + (actual == null ? "not mapped" : "mapped as " + actual) + ", expected " + type);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void expectText(JsonNode mapping, String field, List<String> problems)
|
||||
{
|
||||
String type = mapping.path("type").asText(null);
|
||||
String analyzer = mapping.path("analyzer").asText(null);
|
||||
String search = mapping.path("search_analyzer").asText(null);
|
||||
if(!"text".equals(type))
|
||||
{
|
||||
problems.add(field + " is " + (type == null ? "not mapped" : "mapped as " + type) + ", expected text");
|
||||
}
|
||||
else if(!QuickSearchOpenSearchClient.INDEX_ANALYZER.equals(analyzer) || !QuickSearchOpenSearchClient.SEARCH_ANALYZER.equals(search))
|
||||
{
|
||||
problems.add(field + " uses analyzer " + analyzer + " / search analyzer " + search + ", expected " + QuickSearchOpenSearchClient.INDEX_ANALYZER + " / " + QuickSearchOpenSearchClient.SEARCH_ANALYZER);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live analysis settings against the ones this version creates.
|
||||
*******************************************************************************/
|
||||
void checkSettings(QuickSearchDiagnosticsOperations ops, List<Finding> findings)
|
||||
{
|
||||
RawResponse response;
|
||||
try
|
||||
{
|
||||
response = ops.settings();
|
||||
QuickSearchDiagnosticsOperations.requireOk(response, "Settings read failed");
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, e.getMessage()));
|
||||
return;
|
||||
}
|
||||
|
||||
Iterator<Map.Entry<String, JsonNode>> indexes = response.body().fields();
|
||||
while(indexes.hasNext())
|
||||
{
|
||||
Map.Entry<String, JsonNode> index = indexes.next();
|
||||
JsonNode settings = index.getValue().path("settings").path("index");
|
||||
JsonNode analysis = settings.path("analysis");
|
||||
List<String> problems = new ArrayList<>();
|
||||
|
||||
JsonNode indexAnalyzer = analysis.path("analyzer").path(QuickSearchOpenSearchClient.INDEX_ANALYZER);
|
||||
JsonNode searchAnalyzer = analysis.path("analyzer").path(QuickSearchOpenSearchClient.SEARCH_ANALYZER);
|
||||
JsonNode ngram = analysis.path("filter").path("edge_ngram_filter");
|
||||
|
||||
if(!"standard".equals(indexAnalyzer.path("tokenizer").asText()) || !List.of("lowercase", "edge_ngram_filter").equals(textList(indexAnalyzer.path("filter"))))
|
||||
{
|
||||
problems.add(QuickSearchOpenSearchClient.INDEX_ANALYZER + " is " + (indexAnalyzer.isMissingNode() ? "missing" : indexAnalyzer.toString()));
|
||||
}
|
||||
if(!"standard".equals(searchAnalyzer.path("tokenizer").asText()) || !List.of("lowercase").equals(textList(searchAnalyzer.path("filter"))))
|
||||
{
|
||||
problems.add(QuickSearchOpenSearchClient.SEARCH_ANALYZER + " is " + (searchAnalyzer.isMissingNode() ? "missing" : searchAnalyzer.toString()));
|
||||
}
|
||||
if(!"edge_ngram".equals(ngram.path("type").asText()) || ngram.path("min_gram").asInt() != 2 || ngram.path("max_gram").asInt() != 20)
|
||||
{
|
||||
problems.add("edge_ngram_filter is " + (ngram.isMissingNode() ? "missing" : ngram.toString()));
|
||||
}
|
||||
|
||||
findings.add(problems.isEmpty()
|
||||
? new Finding(SCOPE_MAPPING, "Analysis settings", Level.OK, index.getKey() + ": analyzers and edge n-grams (2 to 20) as expected; "
|
||||
+ settings.path("number_of_shards").asText("?") + " shard(s), " + settings.path("number_of_replicas").asText("?") + " replica(s).")
|
||||
: new Finding(SCOPE_MAPPING, "Analysis settings", Level.FAIL, index.getKey() + ": " + String.join("; ", problems) + ". Run the full reindex."));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static List<String> textList(JsonNode array)
|
||||
{
|
||||
List<String> values = new ArrayList<>();
|
||||
array.forEach(v -> values.add(v.asText()));
|
||||
return (values);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Counts, sampled freshness and orphans of one table, plus its index row
|
||||
** and failed events.
|
||||
*******************************************************************************/
|
||||
TableStats checkTable(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, int sampleSize, List<Finding> findings)
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
Integer databaseCount = null;
|
||||
Long indexCount = null;
|
||||
try
|
||||
{
|
||||
databaseCount = DiagnosticRecordReads.count(tableName);
|
||||
indexCount = runtime.getClient().countDocumentsForTable(tableName);
|
||||
long difference = indexCount - databaseCount;
|
||||
long tolerance = Math.max(5, databaseCount / 100);
|
||||
Level level = difference == 0 ? Level.OK : (Math.abs(difference) <= tolerance ? Level.WARN : Level.FAIL);
|
||||
findings.add(new Finding(tableName, "Counts", level, "database " + databaseCount + ", index " + indexCount
|
||||
+ (difference == 0 ? "." : " (" + (difference > 0 ? difference + " more in the index" : (-difference) + " missing from the index") + "). Reconcile fixes counts.")));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(tableName, "Counts", Level.FAIL, e.getMessage()));
|
||||
}
|
||||
|
||||
List<String> missing = new ArrayList<>();
|
||||
Map<String, List<String>> staleFields = new LinkedHashMap<>();
|
||||
int sampled = 0;
|
||||
try
|
||||
{
|
||||
Map<String, QRecord> sample = new LinkedHashMap<>();
|
||||
for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, true, (sampleSize + 1) / 2))
|
||||
{
|
||||
sample.put(record.getValueString(tableConfig.getPrimaryKeyField()), record);
|
||||
}
|
||||
for(QRecord record : DiagnosticRecordReads.loadEdge(tableName, false, sampleSize / 2))
|
||||
{
|
||||
sample.putIfAbsent(record.getValueString(tableConfig.getPrimaryKeyField()), record);
|
||||
}
|
||||
sampled = sample.size();
|
||||
|
||||
Map<String, JsonNode> indexed = ops.getDocuments(tableName, sample.keySet());
|
||||
for(Map.Entry<String, QRecord> entry : sample.entrySet())
|
||||
{
|
||||
JsonNode source = indexed.get(entry.getKey());
|
||||
if(source == null)
|
||||
{
|
||||
missing.add(entry.getKey());
|
||||
continue;
|
||||
}
|
||||
List<String> differing = differingFields(tableConfig, IndexingUtils.buildDocument(entry.getValue(), tableConfig), source);
|
||||
if(!differing.isEmpty())
|
||||
{
|
||||
staleFields.put(entry.getKey(), differing);
|
||||
}
|
||||
}
|
||||
|
||||
findings.add(missing.isEmpty() && staleFields.isEmpty()
|
||||
? new Finding(tableName, "Sampled records", Level.OK, sampled + " record(s) checked: all indexed and current.")
|
||||
: new Finding(tableName, "Sampled records", Level.FAIL, sampled + " record(s) checked: " + missing.size() + " missing" + listIds(missing) + ", "
|
||||
+ staleFields.size() + " stale" + listIds(new ArrayList<>(staleFields.keySet())) + ". Reindex those records, or run reconcile for the table."));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(tableName, "Sampled records", Level.FAIL, e.getMessage()));
|
||||
}
|
||||
|
||||
List<String> extra = new ArrayList<>();
|
||||
int documentsSampled = 0;
|
||||
try
|
||||
{
|
||||
List<String> documentIds = ops.sampleRecordIds(tableName, sampleSize);
|
||||
documentsSampled = documentIds.size();
|
||||
Set<String> existing = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, documentIds));
|
||||
for(String recordId : documentIds)
|
||||
{
|
||||
if(!existing.contains(recordId))
|
||||
{
|
||||
extra.add(recordId);
|
||||
}
|
||||
}
|
||||
findings.add(extra.isEmpty()
|
||||
? new Finding(tableName, "Sampled documents", Level.OK, documentsSampled + " random document(s) checked: every one has its record.")
|
||||
: new Finding(tableName, "Sampled documents", Level.FAIL, documentsSampled + " random document(s) checked: " + extra.size() + " have no record (orphans)"
|
||||
+ listIds(extra) + ". Reconcile removes them."));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.add(new Finding(tableName, "Sampled documents", Level.FAIL, e.getMessage()));
|
||||
}
|
||||
|
||||
checkIndexRow(runtime, tableName, findings);
|
||||
checkFailedEvents(runtime, tableName, findings);
|
||||
|
||||
return (new TableStats(tableName, databaseCount, indexCount, sampled, missing, staleFields, documentsSampled, extra));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Names of the values that differ between an expected document and an
|
||||
** indexed source (recordLabel, searchableText, fieldValues.x).
|
||||
*******************************************************************************/
|
||||
static List<String> differingFields(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, JsonNode source)
|
||||
{
|
||||
return (RecordInspector.diff(tableConfig, expected, new QuickSearchDiagnosticsOperations.DocumentView(true, null, null, source)).stream()
|
||||
.filter(d -> !d.isSame())
|
||||
.map(RecordInspector.FieldDiff::name)
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void checkIndexRow(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
|
||||
{
|
||||
try
|
||||
{
|
||||
String indexTable = runtime.getConfig().getQuickSearchIndexTableName();
|
||||
if(QContext.getQInstance().getTable(indexTable) == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(indexTable);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
findings.add(new Finding(tableName, "Index row", Level.WARN, "No Quick Search Index row yet: no basepull, reconcile or reindex has run for this table."));
|
||||
return;
|
||||
}
|
||||
|
||||
QRecord row = rows.get(0);
|
||||
String status = row.getValueString("status");
|
||||
String lastRun = row.getValueString("lastRunStatus");
|
||||
if(Boolean.FALSE.equals(row.getValueBoolean("enabled")))
|
||||
{
|
||||
findings.add(new Finding(tableName, "Index row", Level.INFO, "Disabled: Quick Search neither indexes nor searches this table, so differences are expected."));
|
||||
}
|
||||
if(AbstractIndexingStep.STATUS_NEEDS_REINDEX.equals(status))
|
||||
{
|
||||
findings.add(new Finding(tableName, "Index row", Level.WARN, "Status NEEDS_REINDEX: the configured fields changed since the last reindex. Rebuild the table."));
|
||||
}
|
||||
else if(AbstractIndexingStep.STATUS_REBUILDING.equals(status))
|
||||
{
|
||||
findings.add(new Finding(tableName, "Index row", Level.WARN, "Status REBUILDING: a full reindex is running. If it was killed, basepull returns the table to ACTIVE once its run records show no sign of life for fullReindexStaleMinutes."));
|
||||
}
|
||||
if(AbstractIndexingStep.RUN_FAILED.equals(lastRun))
|
||||
{
|
||||
findings.add(new Finding(tableName, "Last run", Level.WARN, "The last indexing run failed: " + row.getValueString("lastErrorMessage")));
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Could not read the Quick Search Index row during validation", e, logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void checkFailedEvents(QuickSearchRuntime runtime, String tableName, List<Finding> findings)
|
||||
{
|
||||
try
|
||||
{
|
||||
String failedTable = runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
|
||||
if(QContext.getQInstance().getTable(failedTable) == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
CountInput countInput = new CountInput();
|
||||
countInput.setTableName(failedTable);
|
||||
countInput.setFilter(new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
|
||||
new QFilterCriteria("status", QCriteriaOperator.IN, List.of(QuickSearchFailedEvent.STATUS_PENDING, QuickSearchFailedEvent.STATUS_EXHAUSTED))));
|
||||
Integer count = new CountAction().execute(countInput).getCount();
|
||||
if(count != null && count > 0)
|
||||
{
|
||||
findings.add(new Finding(tableName, "Failed events", Level.WARN, count + " real-time index event(s) failed and wait for replay (or are exhausted). See Quick Search Failed Event."));
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Could not count failed events during validation", e, logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Save a report as a quickSearchValidationRun row (as the system user);
|
||||
** returns its id, or null when the table is not configured or the insert
|
||||
** fails (logged).
|
||||
*******************************************************************************/
|
||||
public Integer persist(QuickSearchRuntime runtime, Report report, String triggerType, String userName)
|
||||
{
|
||||
String tableName = runtime.getConfig().applyPrefix(QuickSearchValidationRun.TABLE_NAME);
|
||||
if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
QuickSearchValidationRun run = new QuickSearchValidationRun()
|
||||
.withStatus(report.status())
|
||||
.withTriggerType(triggerType)
|
||||
.withUserName(userName)
|
||||
.withTableNames(DiagnosticEventRecorder.truncate(String.join(",", report.tableNames()), 1000))
|
||||
.withSampleSize(report.sampleSize())
|
||||
.withTablesChecked(report.tableStats().size())
|
||||
.withFailCount(report.count(Level.FAIL))
|
||||
.withWarnCount(report.count(Level.WARN))
|
||||
.withSummary(DiagnosticEventRecorder.truncate(summary(report), 250))
|
||||
.withReportHtml(render(report))
|
||||
.withReportJson(toJson(report))
|
||||
.withStartTime(report.startTime())
|
||||
.withEndTime(report.endTime())
|
||||
.withCreateDate(Instant.now());
|
||||
|
||||
Integer[] id = new Integer[1];
|
||||
try
|
||||
{
|
||||
QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
|
||||
{
|
||||
InsertOutput output = new InsertAction().execute(new InsertInput(tableName).withRecordEntity(run));
|
||||
id[0] = output.getRecords().get(0).getValueInteger("id");
|
||||
});
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not save the Quick Search validation report", e);
|
||||
}
|
||||
return (id[0]);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One line: status and counts.
|
||||
*******************************************************************************/
|
||||
public static String summary(Report report)
|
||||
{
|
||||
return (report.status() + ": " + report.tableStats().size() + " table(s), " + report.count(Level.FAIL) + " failed check(s), " + report.count(Level.WARN) + " warning(s).");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The report as JSON.
|
||||
*******************************************************************************/
|
||||
public static String toJson(Report report)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (MAPPER.writeValueAsString(report));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
return ("{\"error\":\"report could not be serialized\"}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The report as HTML: summary, then the findings grouped by scope.
|
||||
*******************************************************************************/
|
||||
public static String render(Report report)
|
||||
{
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open();
|
||||
Level overall = switch(report.status())
|
||||
{
|
||||
case QuickSearchValidationRun.STATUS_PASS -> Level.OK;
|
||||
case QuickSearchValidationRun.STATUS_WARN -> Level.WARN;
|
||||
default -> Level.FAIL;
|
||||
};
|
||||
html.heading("Index validation");
|
||||
html.status(overall, summary(report));
|
||||
html.paragraph("Ran " + report.startTime() + " to " + report.endTime() + ", " + report.sampleSize() + " sampled record(s) and document(s) per table. Database reads ran as the system user.");
|
||||
|
||||
Map<String, List<DiagnosticsHtml.Check>> byScope = new LinkedHashMap<>();
|
||||
for(Finding finding : report.findings())
|
||||
{
|
||||
byScope.computeIfAbsent(finding.scope(), k -> new ArrayList<>()).add(new DiagnosticsHtml.Check(finding.check(), finding.level(), finding.message()));
|
||||
}
|
||||
for(Map.Entry<String, List<DiagnosticsHtml.Check>> scope : byScope.entrySet())
|
||||
{
|
||||
String heading = switch(scope.getKey())
|
||||
{
|
||||
case SCOPE_CLUSTER -> "Cluster and index";
|
||||
case SCOPE_MAPPING -> "Mapping and analyzers";
|
||||
default -> "Table " + scope.getKey();
|
||||
};
|
||||
html.subheading(heading);
|
||||
html.checks(scope.getValue());
|
||||
}
|
||||
|
||||
for(TableStats stats : report.tableStats())
|
||||
{
|
||||
if(!stats.staleFields().isEmpty())
|
||||
{
|
||||
List<String> items = new ArrayList<>();
|
||||
stats.staleFields().entrySet().stream().limit(MAX_LISTED_IDS).forEach(e ->
|
||||
items.add("#" + DiagnosticsHtml.escape(e.getKey()) + ": " + DiagnosticsHtml.escape(String.join(", ", e.getValue()))));
|
||||
html.subheading("Stale fields in " + stats.tableName());
|
||||
html.rawList(items);
|
||||
}
|
||||
}
|
||||
return (html.close().toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static String listIds(List<String> ids)
|
||||
{
|
||||
if(ids.isEmpty())
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
return (" (" + String.join(", ", ids.subList(0, Math.min(MAX_LISTED_IDS, ids.size()))) + (ids.size() > MAX_LISTED_IDS ? ", ..." : "") + ")");
|
||||
}
|
||||
|
||||
}
|
||||
-535
@@ -1,535 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanel;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelAction;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelBlock;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelContext;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelKeyValue;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelLink;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelTone;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The "Quick Search" panel of a record's developer view (QQQ's
|
||||
** DeveloperPanelProviderInterface, /app/{table}/{id}/dev).
|
||||
**
|
||||
** Shown for tables this QBit indexes, to users who may read the Quick Search
|
||||
** Index table. It shows the table's index row (enabled, status, counts, last
|
||||
** basepull and full reindex, searchable fields), the latest index run, the
|
||||
** record's indexed document compared with the database (Inspect Search
|
||||
** Document's comparison; needs the debug permission), the record's failed
|
||||
** index events, and the Inspect / Reindex record actions and admin links the
|
||||
** user may use. Every read checks the user's permissions; an unreachable
|
||||
** OpenSearch shows a warning instead of failing the panel.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDeveloperPanelProvider implements DeveloperPanelProviderInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchDeveloperPanelProvider.class);
|
||||
|
||||
public static final String ID = "quickSearch";
|
||||
public static final String TITLE = "Quick Search";
|
||||
|
||||
public static final String STATUS_INDEXED = "Indexed";
|
||||
public static final String STATUS_STALE = "Stale";
|
||||
public static final String STATUS_NOT_INDEXED = "Not indexed";
|
||||
public static final String STATUS_DISABLED = "Disabled";
|
||||
public static final String STATUS_ENABLED = "Enabled";
|
||||
public static final String STATUS_UNAVAILABLE = "OpenSearch unavailable";
|
||||
|
||||
static final int MAX_FAILED_EVENTS = 20;
|
||||
static final int MAX_VALUE_LENGTH = 300;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getId()
|
||||
{
|
||||
return (ID);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getTitle()
|
||||
{
|
||||
return (TITLE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Integer getOrder()
|
||||
{
|
||||
return (400);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getIcon()
|
||||
{
|
||||
return ("search");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Available for a table this QBit indexes, when the user may read the Quick
|
||||
** Search Index table (the panel shows admin-level indexing state).
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isAvailable(DeveloperPanelContext context)
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null || context == null || runtime.getTableConfig(context.getTableName()) == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
return (mayRead(runtime.getConfig().getQuickSearchIndexTableName()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the panel.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public DeveloperPanel build(DeveloperPanelContext context) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.getOrThrow();
|
||||
QuickSearchQBitConfig config = runtime.getConfig();
|
||||
String tableName = context.getTableName();
|
||||
QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
|
||||
Serializable primaryKey = context.getPrimaryKey();
|
||||
String recordId = primaryKey == null ? null : String.valueOf(primaryKey);
|
||||
String indexTable = config.getQuickSearchIndexTableName();
|
||||
|
||||
DeveloperPanel panel = new DeveloperPanel()
|
||||
.withDescription("How Quick Search (OpenSearch) indexes this table, and this record's search document.");
|
||||
|
||||
//////////////////////////////////////////////////////////////
|
||||
// the table's index row (readable: checked by isAvailable) //
|
||||
//////////////////////////////////////////////////////////////
|
||||
QRecord indexRow = null;
|
||||
if(mayRead(indexTable))
|
||||
{
|
||||
indexRow = queryFirst(indexTable, new QQueryFilter(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)));
|
||||
}
|
||||
boolean enabled = indexRow == null || !Boolean.FALSE.equals(indexRow.getValueBoolean("enabled"));
|
||||
panel.withBlock(indexBlock(config, tableConfig, indexTable, indexRow, enabled));
|
||||
|
||||
//////////////////////
|
||||
// latest index run //
|
||||
//////////////////////
|
||||
String runTable = config.getQuickSearchIndexRunTableName();
|
||||
if(indexRow != null && mayRead(runTable))
|
||||
{
|
||||
QRecord run = queryFirst(runTable, new QQueryFilter(new QFilterCriteria("quickSearchIndexId", QCriteriaOperator.EQUALS, indexRow.getValue("id")))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)));
|
||||
panel.withBlock(run == null
|
||||
? DeveloperPanelBlock.text("No index runs are recorded for this table yet.", DeveloperPanelTone.INFO).withLabel("Latest index run")
|
||||
: runBlock(runTable, run));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////
|
||||
// this record's document vs. database //
|
||||
/////////////////////////////////////////
|
||||
String verdict = null;
|
||||
boolean openSearchFailed = false;
|
||||
if(recordId != null)
|
||||
{
|
||||
if(!Boolean.TRUE.equals(config.getEnableDiagnosticTools()))
|
||||
{
|
||||
panel.withBlock(DeveloperPanelBlock.text("Quick Search diagnostic tools are disabled (enableDiagnosticTools), so this record's search document is not compared.", DeveloperPanelTone.INFO)
|
||||
.withLabel("Search document"));
|
||||
}
|
||||
else if(!mayInspect(config, tableName))
|
||||
{
|
||||
panel.withBlock(DeveloperPanelBlock.text("You do not have the Quick Search debug permission, so this record's search document is not shown.", DeveloperPanelTone.INFO)
|
||||
.withLabel("Search document"));
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
RecordInspector.Inspection inspection = inspect(runtime, tableName, recordId, new QueryInput(tableName));
|
||||
verdict = inspection.verdict();
|
||||
addInspectionBlocks(panel, inspection);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
openSearchFailed = true;
|
||||
LOG.info("Quick Search developer panel could not read the search document", logPair("tableName", tableName), logPair("recordId", recordId), logPair("error", e.getMessage()));
|
||||
panel.withBlock(DeveloperPanelBlock.text("OpenSearch could not be reached, so this record's search document cannot be shown: "
|
||||
+ DiagnosticsSupport.rootCause(e).getMessage(), DeveloperPanelTone.WARNING).withLabel("Search document"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////
|
||||
// failed index events, if any //
|
||||
/////////////////////////////////
|
||||
String failedTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
|
||||
if(recordId != null && mayRead(failedTable))
|
||||
{
|
||||
DeveloperPanelBlock failedBlock = failedEventsBlock(failedTable, tableName, recordId);
|
||||
if(failedBlock != null)
|
||||
{
|
||||
panel.withBlock(failedBlock);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////
|
||||
// actions the user may run //
|
||||
//////////////////////////////
|
||||
if(recordId != null)
|
||||
{
|
||||
DeveloperPanelBlock actions = DeveloperPanelBlock.actions("Actions");
|
||||
addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName), tableName, "Inspect search document",
|
||||
"Compare the indexed document with the database, and see why the record does or does not match a term.");
|
||||
addActionIfPermitted(actions, QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config, tableName), tableName, "Reindex this record",
|
||||
"Rebuild this record's search document from the database now.");
|
||||
if(!actions.getActions().isEmpty())
|
||||
{
|
||||
panel.withBlock(actions);
|
||||
}
|
||||
}
|
||||
|
||||
///////////
|
||||
// links //
|
||||
///////////
|
||||
DeveloperPanelBlock links = DeveloperPanelBlock.links("Links");
|
||||
String appName = config.applyPrefix(QuickSearchAdminAppMetaDataProducer.APP_NAME);
|
||||
if(QContext.getQInstance().getApp(appName) != null && PermissionsHelper.hasAppPermission(new QueryInput(indexTable), appName))
|
||||
{
|
||||
links.withLink(new DeveloperPanelLink().withLabel("Quick Search Admin").withHref("/app/" + appName).withDescription("Index status, runs, failed events and diagnostic tools."));
|
||||
}
|
||||
if(indexRow != null)
|
||||
{
|
||||
links.withLink(new DeveloperPanelLink().withLabel("Quick Search Index row for " + tableName).withHref(recordHref(indexTable, indexRow.getValue("id"))));
|
||||
}
|
||||
if(!links.getLinks().isEmpty())
|
||||
{
|
||||
panel.withBlock(links);
|
||||
}
|
||||
|
||||
setStatus(panel, enabled, verdict, openSearchFailed);
|
||||
return (panel);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect one record (overridable for tests).
|
||||
*******************************************************************************/
|
||||
protected RecordInspector.Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, AbstractActionInput permissionInput) throws QException
|
||||
{
|
||||
return (new RecordInspector().inspect(runtime, tableName, recordId, null, permissionInput));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The table's index row as key/values.
|
||||
*******************************************************************************/
|
||||
private DeveloperPanelBlock indexBlock(QuickSearchQBitConfig config, QuickSearchableTableConfig tableConfig, String indexTable, QRecord indexRow, boolean enabled)
|
||||
{
|
||||
DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Index");
|
||||
if(indexRow == null)
|
||||
{
|
||||
block.withItem("Index row", "none yet (the first indexing run creates it)");
|
||||
}
|
||||
else
|
||||
{
|
||||
block.withItem(new DeveloperPanelKeyValue().withLabel("Index row").withValue("#" + indexRow.getValueString("id")).withHref(recordHref(indexTable, indexRow.getValue("id"))));
|
||||
}
|
||||
block.withItem("Enabled", enabled ? "Yes" : "No");
|
||||
block.withItem("Status", indexRow == null ? null : indexRow.getValueString("status"));
|
||||
block.withItem("Indexed records", indexRow == null ? null : indexRow.getValueString("recordCount"));
|
||||
block.withItem("Last basepull", indexRow == null ? null : indexRow.getValueString("lastBasepullTime"));
|
||||
block.withItem("Last full reindex", indexRow == null ? null : indexRow.getValueString("lastFullReindexTime"));
|
||||
block.withItem("Searchable fields", String.join(", ", CollectionUtils.nonNullList(tableConfig.getSearchableFields())));
|
||||
block.withItem(new DeveloperPanelKeyValue().withLabel("OpenSearch index (alias)").withValue(config.getOpensearchIndexName()).withMonospace(true));
|
||||
block.withItem("Real-time indexing", Boolean.TRUE.equals(config.getEnableRealTimeIndexing()) ? "On" : "Off");
|
||||
return (block);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The latest index run as key/values.
|
||||
*******************************************************************************/
|
||||
private DeveloperPanelBlock runBlock(String runTable, QRecord run)
|
||||
{
|
||||
DeveloperPanelBlock block = DeveloperPanelBlock.keyValues("Latest index run")
|
||||
.withItem(new DeveloperPanelKeyValue().withLabel("Run").withValue("#" + run.getValueString("id")).withHref(recordHref(runTable, run.getValue("id"))))
|
||||
.withItem("Type", run.getValueString("runType"))
|
||||
.withItem("Status", run.getValueString("status"))
|
||||
.withItem("Started", run.getValueString("startTime"))
|
||||
.withItem("Ended", run.getValueString("endTime"))
|
||||
.withItem("Processed", run.getValueString("recordsProcessed"))
|
||||
.withItem("Indexed", run.getValueString("recordsIndexed"))
|
||||
.withItem("Errors", run.getValueString("errorCount"));
|
||||
if(run.getValueString("errorMessage") != null)
|
||||
{
|
||||
block.withItem("Error", truncate(run.getValueString("errorMessage")));
|
||||
}
|
||||
return (block);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The inspection: verdict text, document facts, field comparison table.
|
||||
*******************************************************************************/
|
||||
private void addInspectionBlocks(DeveloperPanel panel, RecordInspector.Inspection inspection)
|
||||
{
|
||||
if(inspection.error() != null)
|
||||
{
|
||||
panel.withBlock(DeveloperPanelBlock.text(inspection.error(), RecordInspector.VERDICT_NOT_INDEXED.equals(inspection.verdict()) ? DeveloperPanelTone.INFO : DeveloperPanelTone.WARNING)
|
||||
.withLabel("Search document"));
|
||||
return;
|
||||
}
|
||||
|
||||
switch(inspection.verdict())
|
||||
{
|
||||
case RecordInspector.VERDICT_CURRENT -> panel.withBlock(DeveloperPanelBlock.text("Indexed and current: the search document matches the database.", DeveloperPanelTone.SUCCESS)
|
||||
.withLabel("Search document"));
|
||||
case RecordInspector.VERDICT_STALE -> panel.withBlock(DeveloperPanelBlock.text("Indexed but stale: some values differ from the database. Reindex this record to refresh it.", DeveloperPanelTone.WARNING)
|
||||
.withLabel("Search document"));
|
||||
default -> panel.withBlock(DeveloperPanelBlock.text("Not in the index: the global search cannot find this record. Reindex this record to add it (or wait for the next basepull or reconcile).",
|
||||
DeveloperPanelTone.ERROR).withLabel("Search document"));
|
||||
}
|
||||
|
||||
DocumentView indexed = inspection.indexed();
|
||||
panel.withBlock(DeveloperPanelBlock.keyValues("Document")
|
||||
.withItem(new DeveloperPanelKeyValue().withLabel("Document id").withValue(OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId())).withMonospace(true))
|
||||
.withItem(new DeveloperPanelKeyValue().withLabel("Physical index").withValue(indexed == null ? null : indexed.physicalIndex()).withMonospace(true))
|
||||
.withItem("Indexed version", indexed == null || indexed.version() == null ? null : String.valueOf(indexed.version()))
|
||||
.withItem("Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)" : String.valueOf(inspection.expected().getVersion()))
|
||||
.withItem("Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null)));
|
||||
|
||||
DeveloperPanelBlock table = DeveloperPanelBlock.table("Database vs index", List.of("Field", "Database", "Index", "Match"));
|
||||
for(RecordInspector.FieldDiff diff : CollectionUtils.nonNullList(inspection.diffs()))
|
||||
{
|
||||
table.withRow(row(diff.name(), truncate(diff.databaseValue()), truncate(diff.indexValue()), diff.isSame() ? "yes" : "NO"));
|
||||
}
|
||||
panel.withBlock(table);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record's failed index events, or null when there are none.
|
||||
*******************************************************************************/
|
||||
private DeveloperPanelBlock failedEventsBlock(String failedTable, String tableName, String recordId) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(failedTable);
|
||||
queryInput.setFilter(new QQueryFilter(
|
||||
new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
|
||||
new QFilterCriteria("recordId", QCriteriaOperator.EQUALS, recordId))
|
||||
.withOrderBy(new QFilterOrderBy("id", false))
|
||||
.withLimit(MAX_FAILED_EVENTS));
|
||||
List<QRecord> events = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
|
||||
if(events.isEmpty())
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
DeveloperPanelBlock block = DeveloperPanelBlock.table("Failed index events" + (events.size() >= MAX_FAILED_EVENTS ? " (latest " + MAX_FAILED_EVENTS + ")" : ""),
|
||||
List.of("Id", "Action", "Status", "Attempts", "Created", "Error"));
|
||||
for(QRecord event : events)
|
||||
{
|
||||
block.withRow(row(event.getValueString("id"), event.getValueString("action"), event.getValueString("status"), event.getValueString("attempts"),
|
||||
event.getValueString("createDate"), truncate(event.getValueString("errorMessage"))));
|
||||
}
|
||||
return (block);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a record action when the process exists and the user may run it.
|
||||
*******************************************************************************/
|
||||
private void addActionIfPermitted(DeveloperPanelBlock actions, String processName, String tableName, String label, String description)
|
||||
{
|
||||
if(QContext.getQInstance().getProcess(processName) != null && PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName))
|
||||
{
|
||||
actions.withAction(new DeveloperPanelAction().withLabel(label).withProcessName(processName).withTableName(tableName).withDescription(description));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the user may inspect records: the Inspect Search Document
|
||||
** process's permission (debugPermissionRules), evaluated on the
|
||||
** registered process, or on an unregistered copy when the record actions
|
||||
** are turned off.
|
||||
*******************************************************************************/
|
||||
private boolean mayInspect(QuickSearchQBitConfig config, String tableName)
|
||||
{
|
||||
String processName = QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, tableName);
|
||||
if(QContext.getQInstance().getProcess(processName) != null)
|
||||
{
|
||||
return (PermissionsHelper.hasProcessPermission(new QueryInput(tableName), processName));
|
||||
}
|
||||
return (PermissionsHelper.getPermissionCheckResult(new QueryInput(tableName), QuickSearchDiagnosticsProcessMetaDataHelper.buildInspectProcess(config, tableName))
|
||||
== PermissionCheckResult.ALLOW);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Status label and tone from the enabled flag and the inspection verdict.
|
||||
*******************************************************************************/
|
||||
private void setStatus(DeveloperPanel panel, boolean enabled, String verdict, boolean openSearchFailed)
|
||||
{
|
||||
if(!enabled)
|
||||
{
|
||||
panel.withStatusLabel(STATUS_DISABLED).withStatusTone(DeveloperPanelTone.WARNING);
|
||||
}
|
||||
else if(openSearchFailed)
|
||||
{
|
||||
panel.withStatusLabel(STATUS_UNAVAILABLE).withStatusTone(DeveloperPanelTone.WARNING);
|
||||
}
|
||||
else if(RecordInspector.VERDICT_CURRENT.equals(verdict))
|
||||
{
|
||||
panel.withStatusLabel(STATUS_INDEXED).withStatusTone(DeveloperPanelTone.SUCCESS);
|
||||
}
|
||||
else if(RecordInspector.VERDICT_STALE.equals(verdict))
|
||||
{
|
||||
panel.withStatusLabel(STATUS_STALE).withStatusTone(DeveloperPanelTone.WARNING);
|
||||
}
|
||||
else if(RecordInspector.VERDICT_MISSING.equals(verdict))
|
||||
{
|
||||
panel.withStatusLabel(STATUS_NOT_INDEXED).withStatusTone(DeveloperPanelTone.ERROR);
|
||||
}
|
||||
else
|
||||
{
|
||||
panel.withStatusLabel(STATUS_ENABLED).withStatusTone(DeveloperPanelTone.INFO);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the table exists and the user may read it.
|
||||
*******************************************************************************/
|
||||
private static boolean mayRead(String tableName)
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
return (qInstance != null && qInstance.getTable(tableName) != null && PermissionsHelper.hasTablePermission(new QueryInput(tableName), tableName, TablePermissionSubType.READ));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** First row of a query (as the user), or null.
|
||||
*******************************************************************************/
|
||||
private static QRecord queryFirst(String tableName, QQueryFilter filter) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(tableName);
|
||||
queryInput.setFilter(filter.withLimit(1));
|
||||
List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
|
||||
return (CollectionUtils.nullSafeHasContents(records) ? records.get(0) : null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** App-relative link to a record.
|
||||
*******************************************************************************/
|
||||
private static String recordHref(String tableName, Serializable id)
|
||||
{
|
||||
return ("/app/" + tableName + "/" + id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A table row (null cells as empty strings).
|
||||
*******************************************************************************/
|
||||
private static List<String> row(String... cells)
|
||||
{
|
||||
List<String> row = new ArrayList<>();
|
||||
for(String cell : cells)
|
||||
{
|
||||
row.add(cell == null ? "" : cell);
|
||||
}
|
||||
return (row);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Long values cut for display.
|
||||
*******************************************************************************/
|
||||
static String truncate(String value)
|
||||
{
|
||||
if(value == null || value.length() <= MAX_VALUE_LENGTH)
|
||||
{
|
||||
return (value);
|
||||
}
|
||||
return (value.substring(0, MAX_VALUE_LENGTH) + "... (" + value.length() + " characters)");
|
||||
}
|
||||
|
||||
}
|
||||
-217
@@ -1,217 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Search Diagnostics widget: the record-search provider's state on the
|
||||
** server that renders it, the most recent search failures (from every
|
||||
** server, via Quick Search Diagnostic Event), the latest validation run,
|
||||
** and where the tools are. Every part says plainly when it has nothing to
|
||||
** show or cannot be read.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticsWidgetRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QuickSearchDiagnosticsWidgetRenderer.class);
|
||||
|
||||
public static final int RECENT_FAILURES = 10;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = QuickSearchRuntime.get();
|
||||
if(runtime == null)
|
||||
{
|
||||
return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", "<p>Quick Search is not configured in this application.</p>")));
|
||||
}
|
||||
return (new RenderWidgetOutput(new RawHTML("Search Diagnostics", renderHtml(runtime))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The widget's HTML.
|
||||
*******************************************************************************/
|
||||
static String renderHtml(QuickSearchRuntime runtime)
|
||||
{
|
||||
QuickSearchQBitConfig config = runtime.getConfig();
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open();
|
||||
|
||||
SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(QContext.getQInstance(), runtime);
|
||||
html.subheading("Global search on this server");
|
||||
if(!Boolean.TRUE.equals(state.quickSearchProvider()) || !Boolean.TRUE.equals(state.serveCoreRecordSearch()))
|
||||
{
|
||||
html.status(Level.INFO, "Quick Search does not serve the global search here (" + (state.registeredProvider() == null ? "no provider registered" : "provider " + state.registeredProvider())
|
||||
+ "); core searches every table itself.");
|
||||
}
|
||||
else if(state.backoffRemainingMillis() > 0)
|
||||
{
|
||||
html.status(Level.FAIL, "Backing off for " + (state.backoffRemainingMillis() / 1000) + " s after an OpenSearch failure: the global search uses core's database search meanwhile. Last error: "
|
||||
+ state.lastFailureMessage());
|
||||
}
|
||||
else
|
||||
{
|
||||
html.status(Level.OK, "Served by OpenSearch through Quick Search (timeout " + state.timeoutMillis() + " ms).");
|
||||
}
|
||||
html.facts(List.of(
|
||||
new String[] { "Failed OpenSearch searches since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) },
|
||||
new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) },
|
||||
new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() }));
|
||||
|
||||
renderRecentFailures(html, config);
|
||||
renderLastValidation(html, config);
|
||||
|
||||
html.subheading("Tools");
|
||||
html.paragraph("On Quick Search Index (Actions): Search Playground, Validate Search Index, Test Search Analyzer. "
|
||||
+ "On any indexed record and on Quick Search Failed Event rows: Inspect Search Document and Reindex in Search. "
|
||||
+ "Every run is recorded in Quick Search Diagnostic Event.");
|
||||
return (html.close().toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void renderRecentFailures(DiagnosticsHtml html, QuickSearchQBitConfig config)
|
||||
{
|
||||
String tableName = config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME);
|
||||
html.subheading("Recent search failures (all servers)");
|
||||
if(QContext.getQInstance().getTable(tableName) == null)
|
||||
{
|
||||
html.paragraph("Not recorded: the diagnostic event table is not configured.");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("eventType", QCriteriaOperator.EQUALS, QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED))
|
||||
.withOrderBy(new QFilterOrderBy("id", false))
|
||||
.withLimit(RECENT_FAILURES));
|
||||
List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
html.status(Level.OK, "None recorded (failures are kept for " + config.getRunHistoryRetentionDays() + " days).");
|
||||
return;
|
||||
}
|
||||
|
||||
List<String> items = new ArrayList<>();
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
items.add("<a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
|
||||
+ DiagnosticsHtml.escape(row.getValueString("createDate")) + "</a> on " + DiagnosticsHtml.escape(row.getValueString("hostName"))
|
||||
+ ": " + DiagnosticsHtml.escape(row.getValueString("source")) + ", " + DiagnosticsHtml.escape(row.getValueString("durationMillis")) + " ms, "
|
||||
+ DiagnosticsHtml.escape(shortClass(row.getValueString("errorClass"))) + ": " + DiagnosticsHtml.escape(abbreviate(row.getValueString("errorMessage"), 240)));
|
||||
}
|
||||
html.rawList(items);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Could not read recent search failures for the diagnostics widget", e);
|
||||
html.status(Level.WARN, "Could not read the diagnostic events: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void renderLastValidation(DiagnosticsHtml html, QuickSearchQBitConfig config)
|
||||
{
|
||||
String tableName = config.applyPrefix(QuickSearchValidationRun.TABLE_NAME);
|
||||
html.subheading("Latest index validation");
|
||||
if(QContext.getQInstance().getTable(tableName) == null)
|
||||
{
|
||||
html.paragraph("Not kept: the validation run table is not configured.");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
List<QRecord> rows = CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
html.paragraph("None yet. Run Validate Search Index from Quick Search Index (Actions)"
|
||||
+ (config.getValidationCronExpression() == null ? "." : "; it is also scheduled (" + config.getValidationCronExpression() + ")."));
|
||||
return;
|
||||
}
|
||||
|
||||
QRecord row = rows.get(0);
|
||||
String status = row.getValueString("status");
|
||||
Level level = QuickSearchValidationRun.STATUS_PASS.equals(status) ? Level.OK : (QuickSearchValidationRun.STATUS_WARN.equals(status) ? Level.WARN : Level.FAIL);
|
||||
html.raw("<p>" + DiagnosticsHtml.badge(level) + " <a href=\"/app/" + DiagnosticsHtml.escape(tableName) + "/" + DiagnosticsHtml.escape(row.getValueString("id")) + "\">"
|
||||
+ DiagnosticsHtml.escape(row.getValueString("startTime")) + "</a> (" + DiagnosticsHtml.escape(row.getValueString("triggerType")) + "): "
|
||||
+ DiagnosticsHtml.escape(row.getValueString("summary")) + "</p>");
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Could not read the latest validation run for the diagnostics widget", e);
|
||||
html.status(Level.WARN, "Could not read the validation runs: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static String shortClass(String className)
|
||||
{
|
||||
return (className == null ? "error" : className.substring(className.lastIndexOf('.') + 1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static String abbreviate(String text, int max)
|
||||
{
|
||||
return (text == null ? "" : (text.length() > max ? text.substring(0, max) + "..." : text));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,410 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Check;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.DocumentView;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect one record's search document: the document as indexed next to the
|
||||
** document indexing would write from the database now (field by field),
|
||||
** and, given a term, why the record does or does not match it.
|
||||
** The match analysis walks the same gates as the global search: term
|
||||
** length, table indexed and enabled, read permission, record readable,
|
||||
** document present and current, every query token present among the
|
||||
** document's indexed tokens (edge n-grams of 2 to 20 characters, AND
|
||||
** semantics), explain, and the record's rank against the provider's
|
||||
** per-table id window. Read-only.
|
||||
*******************************************************************************/
|
||||
public class RecordInspector
|
||||
{
|
||||
public static final String VERDICT_CURRENT = "CURRENT";
|
||||
public static final String VERDICT_STALE = "STALE";
|
||||
public static final String VERDICT_MISSING = "MISSING";
|
||||
public static final String VERDICT_NOT_INDEXED = "NOT_INDEXED";
|
||||
public static final String VERDICT_NO_RECORD = "NO_RECORD";
|
||||
|
||||
private static final int RANK_WINDOW = RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One compared value: name, value from the database, value in the index.
|
||||
*******************************************************************************/
|
||||
public record FieldDiff(String name, String databaseValue, String indexValue)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether the two values are equal.
|
||||
*******************************************************************************/
|
||||
public boolean isSame()
|
||||
{
|
||||
return (Objects.equals(databaseValue, indexValue));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The inspection of one record.
|
||||
*******************************************************************************/
|
||||
public record Inspection(String tableName, String recordId, String verdict, List<FieldDiff> diffs, DocumentView indexed, OpenSearchDocument expected,
|
||||
List<Check> matchChecks, ExplainResult explain, String error)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect one record, with an optional term to analyze.
|
||||
*******************************************************************************/
|
||||
public Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, String term, AbstractActionInput permissionInput) throws QException
|
||||
{
|
||||
QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
return (new Inspection(tableName, recordId, VERDICT_NOT_INDEXED, List.of(), null, null, List.of(), null, "Quick Search does not index table [" + tableName + "]."));
|
||||
}
|
||||
|
||||
Map<String, QRecord> records = DiagnosticRecordReads.loadForIndexing(tableName, List.of(recordId));
|
||||
QRecord record = records.get(recordId);
|
||||
if(record == null && !records.isEmpty())
|
||||
{
|
||||
record = records.values().iterator().next();
|
||||
}
|
||||
if(record == null)
|
||||
{
|
||||
return (new Inspection(tableName, recordId, VERDICT_NO_RECORD, List.of(), null, null, List.of(), null,
|
||||
"Record " + recordId + " does not exist, or you cannot read it."));
|
||||
}
|
||||
|
||||
QuickSearchDiagnosticsOperations ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
|
||||
OpenSearchDocument expected = IndexingUtils.buildDocument(record, tableConfig);
|
||||
DocumentView indexed = ops.getDocument(tableName, recordId);
|
||||
List<FieldDiff> diffs = diff(tableConfig, expected, indexed);
|
||||
|
||||
String verdict = !Boolean.TRUE.equals(indexed.found()) ? VERDICT_MISSING : (diffs.stream().allMatch(FieldDiff::isSame) ? VERDICT_CURRENT : VERDICT_STALE);
|
||||
|
||||
List<Check> checks = new ArrayList<>();
|
||||
ExplainResult explain = null;
|
||||
String cleaned = IndexingUtils.normalizeSearchText(term);
|
||||
if(StringUtils.hasContent(cleaned))
|
||||
{
|
||||
explain = analyzeMatch(runtime, ops, tableConfig, recordId, cleaned, verdict, expected, indexed, checks, permissionInput);
|
||||
}
|
||||
|
||||
return (new Inspection(tableName, recordId, verdict, diffs, indexed, expected, checks, explain, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compare the expected document with the indexed one: label, searchable
|
||||
** text, then each configured field (in configuration order) and any extra
|
||||
** field only the index holds.
|
||||
*******************************************************************************/
|
||||
static List<FieldDiff> diff(QuickSearchableTableConfig tableConfig, OpenSearchDocument expected, DocumentView indexed)
|
||||
{
|
||||
JsonNode source = indexed == null || indexed.source() == null ? null : indexed.source();
|
||||
|
||||
List<FieldDiff> diffs = new ArrayList<>();
|
||||
diffs.add(new FieldDiff("recordLabel", expected == null ? null : blankToNull(expected.getRecordLabel()), text(source, "recordLabel")));
|
||||
diffs.add(new FieldDiff("searchableText", expected == null ? null : blankToNull(expected.getSearchableText()), text(source, "searchableText")));
|
||||
|
||||
Set<String> fieldNames = new LinkedHashSet<>(CollectionUtils.nonNullList(tableConfig.getSearchableFields()));
|
||||
if(source != null && source.has("fieldValues"))
|
||||
{
|
||||
source.get("fieldValues").fieldNames().forEachRemaining(fieldNames::add);
|
||||
}
|
||||
for(String field : fieldNames)
|
||||
{
|
||||
Object expectedValue = expected == null || expected.getFieldValues() == null ? null : expected.getFieldValues().get(field);
|
||||
diffs.add(new FieldDiff("fieldValues." + field, expectedValue == null ? null : String.valueOf(expectedValue),
|
||||
source == null ? null : text(source.path("fieldValues"), field)));
|
||||
}
|
||||
return (diffs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Walk the global search's gates for this record and term, adding a check
|
||||
** per gate; returns the explain result when the document exists.
|
||||
*******************************************************************************/
|
||||
private ExplainResult analyzeMatch(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, QuickSearchableTableConfig tableConfig, String recordId, String term, String verdict,
|
||||
OpenSearchDocument expected, DocumentView indexed, List<Check> checks, AbstractActionInput permissionInput) throws QException
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
|
||||
checks.add(term.length() < QuickSearchAction.MIN_TERM_LENGTH
|
||||
? new Check("Term length", Level.FAIL, "Terms shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters never reach OpenSearch.")
|
||||
: new Check("Term length", Level.OK, term.length() + " characters."));
|
||||
|
||||
boolean enabled = runtime.isTableEnabled(tableName);
|
||||
checks.add(enabled
|
||||
? new Check("Table enabled", Level.OK, "The table is enabled in Quick Search Index.")
|
||||
: new Check("Table enabled", Level.FAIL, "The table is disabled in Quick Search Index, so Quick Search does not search it (core searches it with its own database search)."));
|
||||
|
||||
boolean readable = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
|
||||
checks.add(readable
|
||||
? new Check("Table permission", Level.OK, "You can read this table.")
|
||||
: new Check("Table permission", Level.FAIL, "You cannot read this table, so the global search skips it for you."));
|
||||
|
||||
checks.add(new Check("Record access", Level.OK, "You can read this record (record security locks allow it)."));
|
||||
|
||||
switch(verdict)
|
||||
{
|
||||
case VERDICT_MISSING -> checks.add(new Check("Indexed document", Level.FAIL,
|
||||
"The record has no document in the index, so OpenSearch cannot return it. Use Reindex in Search on the record (or wait for the next basepull / reconcile)."));
|
||||
case VERDICT_STALE -> checks.add(new Check("Indexed document", Level.WARN,
|
||||
"The indexed document differs from the database (see the comparison); the search matches the indexed values. Reindex the record to refresh it."));
|
||||
default -> checks.add(new Check("Indexed document", Level.OK, "The indexed document matches the database."));
|
||||
}
|
||||
|
||||
List<AnalyzedToken> queryTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
|
||||
Set<String> indexedGrams = Boolean.TRUE.equals(indexed.found()) ? tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, text(indexed.source(), "searchableText")))
|
||||
: Set.of();
|
||||
Set<String> expectedGrams = expected == null ? Set.of() : tokenSet(ops.analyze(QuickSearchOpenSearchClient.INDEX_ANALYZER, expected.getSearchableText()));
|
||||
|
||||
if(queryTokens.isEmpty())
|
||||
{
|
||||
checks.add(new Check("Term tokens", Level.FAIL, "The term has no searchable characters after analysis."));
|
||||
}
|
||||
for(AnalyzedToken token : queryTokens)
|
||||
{
|
||||
String value = token.token();
|
||||
if(value.length() < 2)
|
||||
{
|
||||
checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "One character: the index holds edge n-grams of 2 to 20 characters, so a single character never matches, and every token must match."));
|
||||
}
|
||||
else if(value.length() > 20)
|
||||
{
|
||||
checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Longer than 20 characters: the index holds edge n-grams of at most 20 characters, so this token never matches. Search a shorter prefix."));
|
||||
}
|
||||
else if(indexedGrams.contains(value))
|
||||
{
|
||||
checks.add(new Check("Token \"" + value + "\"", Level.OK, "Found in the indexed text (as a word or a word prefix)."));
|
||||
}
|
||||
else if(expectedGrams.contains(value))
|
||||
{
|
||||
checks.add(new Check("Token \"" + value + "\"", Level.WARN, "Not in the indexed text, but in the database's current values: it will match once the record is reindexed."));
|
||||
}
|
||||
else
|
||||
{
|
||||
checks.add(new Check("Token \"" + value + "\"", Level.FAIL, "Not in this record's searchable text: no searchable field holds a word starting with it. Every token must match (AND)."));
|
||||
}
|
||||
}
|
||||
|
||||
ExplainResult explain = null;
|
||||
if(Boolean.TRUE.equals(indexed.found()))
|
||||
{
|
||||
explain = ops.explain(term, tableName, recordId, runtime.getDiscoveredTables());
|
||||
checks.add(Boolean.TRUE.equals(explain.matched())
|
||||
? new Check("OpenSearch explain", Level.OK, "The search query matches this document with score " + String.format("%.4f", explain.score()) + ".")
|
||||
: new Check("OpenSearch explain", Level.FAIL, "The search query does not match this document."));
|
||||
|
||||
if(Boolean.TRUE.equals(explain.matched()))
|
||||
{
|
||||
checks.add(rankCheck(runtime, ops, tableName, recordId, term));
|
||||
}
|
||||
}
|
||||
return (explain);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Where the record ranks among the table's matches, against the number of
|
||||
** ids the provider asks for with the default per-table limit.
|
||||
*******************************************************************************/
|
||||
private Check rankCheck(QuickSearchRuntime runtime, QuickSearchDiagnosticsOperations ops, String tableName, String recordId, String term) throws QException
|
||||
{
|
||||
int window = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
|
||||
SearchProbe probe = ops.probeSearch(term, tableName, RANK_WINDOW, runtime.getDiscoveredTables());
|
||||
int rank = 0;
|
||||
for(ProbeHit hit : probe.hits())
|
||||
{
|
||||
rank++;
|
||||
if(recordId.equals(hit.recordId()))
|
||||
{
|
||||
if(rank <= RecordSearchAction.DEFAULT_LIMIT_PER_TABLE)
|
||||
{
|
||||
return (new Check("Rank", Level.OK, "#" + rank + " of " + probe.totalHits() + " matches in this table: within the global search's default " + RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " results."));
|
||||
}
|
||||
if(rank <= window)
|
||||
{
|
||||
return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: Quick Search returns it, but the global search shows only the first "
|
||||
+ RecordSearchAction.DEFAULT_LIMIT_PER_TABLE + " readable results per table by default. A more specific term ranks it higher."));
|
||||
}
|
||||
return (new Check("Rank", Level.WARN, "#" + rank + " of " + probe.totalHits() + " matches: beyond the top " + window + " ids the provider requests per table, so the global search does not show it."));
|
||||
}
|
||||
}
|
||||
return (new Check("Rank", Level.WARN, "Not in the top " + RANK_WINDOW + " of " + probe.totalHits() + " matches in this table, so the global search does not show it."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render inspections as HTML.
|
||||
*******************************************************************************/
|
||||
public static String render(List<Inspection> inspections, String term)
|
||||
{
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open();
|
||||
for(Inspection inspection : inspections)
|
||||
{
|
||||
html.heading("Search document for " + inspection.tableName() + " #" + inspection.recordId());
|
||||
if(inspection.error() != null)
|
||||
{
|
||||
html.status(Level.FAIL, inspection.error());
|
||||
continue;
|
||||
}
|
||||
|
||||
switch(inspection.verdict())
|
||||
{
|
||||
case VERDICT_CURRENT -> html.status(Level.OK, "Indexed and current: the document matches the database.");
|
||||
case VERDICT_STALE -> html.status(Level.WARN, "Indexed but stale: some values differ from the database. Reindex in Search refreshes it.");
|
||||
default -> html.status(Level.FAIL, "Not in the index. Reindex in Search adds it.");
|
||||
}
|
||||
|
||||
DocumentView indexed = inspection.indexed();
|
||||
List<String[]> facts = new ArrayList<>();
|
||||
facts.add(new String[] { "Document id", OpenSearchDocument.buildDocumentId(inspection.tableName(), inspection.recordId()) });
|
||||
facts.add(new String[] { "Physical index", indexed == null ? null : indexed.physicalIndex() });
|
||||
facts.add(new String[] { "Indexed version", indexed == null || indexed.version() == null ? null : indexed.version() + versionNote(indexed.version()) });
|
||||
facts.add(new String[] { "Database version", inspection.expected() == null || inspection.expected().getVersion() == null ? "not versioned (no timestamp field)"
|
||||
: inspection.expected().getVersion() + versionNote(inspection.expected().getVersion()) });
|
||||
facts.add(new String[] { "Indexed at", indexed == null || indexed.source() == null ? null : indexed.source().path("indexedAt").asText(null) });
|
||||
html.facts(facts);
|
||||
|
||||
html.subheading("Database vs index");
|
||||
List<String> items = new ArrayList<>();
|
||||
for(FieldDiff diff : inspection.diffs())
|
||||
{
|
||||
items.add(DiagnosticsHtml.badge(diff.isSame() ? Level.OK : Level.WARN) + " <strong>" + DiagnosticsHtml.escape(diff.name()) + "</strong>"
|
||||
+ (diff.isSame() ? ": " + valueHtml(diff.databaseValue())
|
||||
: "<br>database: " + valueHtml(diff.databaseValue()) + "<br>index: " + valueHtml(diff.indexValue())));
|
||||
}
|
||||
html.rawList(items);
|
||||
|
||||
if(!inspection.matchChecks().isEmpty())
|
||||
{
|
||||
html.subheading("Why does (or doesn't) this record match \"" + term + "\"?");
|
||||
html.checks(inspection.matchChecks());
|
||||
if(inspection.explain() != null && inspection.explain().explanationText() != null)
|
||||
{
|
||||
html.textDetails("Score explanation", inspection.explain().explanationText());
|
||||
}
|
||||
}
|
||||
|
||||
if(indexed != null && indexed.source() != null)
|
||||
{
|
||||
html.jsonDetails("Indexed document (JSON)", indexed.source().toString());
|
||||
}
|
||||
}
|
||||
return (html.close().toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static String valueHtml(String value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return ("<em>none</em>");
|
||||
}
|
||||
String shown = value.length() > 500 ? value.substring(0, 500) + "... (" + value.length() + " characters)" : value;
|
||||
return ("<code>" + DiagnosticsHtml.escape(shown) + "</code>");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** " (time)" for a version that looks like epoch millis.
|
||||
*******************************************************************************/
|
||||
private static String versionNote(Long version)
|
||||
{
|
||||
return (version != null && version > 946_684_800_000L ? " (" + java.time.Instant.ofEpochMilli(version) + ")" : "");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static Set<String> tokenSet(List<AnalyzedToken> tokens)
|
||||
{
|
||||
Set<String> set = new LinkedHashSet<>();
|
||||
for(AnalyzedToken token : tokens)
|
||||
{
|
||||
set.add(token.token());
|
||||
}
|
||||
return (set);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static String text(JsonNode node, String field)
|
||||
{
|
||||
if(node == null || !node.has(field) || node.get(field).isNull())
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
return (blankToNull(node.get(field).isTextual() ? node.get(field).asText() : node.get(field).toString()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static String blankToNull(String value)
|
||||
{
|
||||
return (value == null || value.isEmpty() ? null : value);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.context.CapturedContext;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reindex chosen records of one table now: each record the user can read is
|
||||
** rebuilt from the database and written to the index (versioned, so a newer
|
||||
** document is never overwritten); a record that no longer exists has its
|
||||
** document removed; a record that exists but that the user cannot read is
|
||||
** skipped. The one diagnostic tool that writes to the index; it never
|
||||
** changes a record. A disabled table is refused.
|
||||
*******************************************************************************/
|
||||
public class RecordReindexer
|
||||
{
|
||||
public static final String OUTCOME_INDEXED = "INDEXED";
|
||||
public static final String OUTCOME_DELETED = "DELETED";
|
||||
public static final String OUTCOME_SKIPPED = "SKIPPED";
|
||||
public static final String OUTCOME_FAILED = "FAILED";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What happened to one record.
|
||||
*******************************************************************************/
|
||||
public record Outcome(String recordId, String outcome, String message)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reindex the records of a table by id.
|
||||
*******************************************************************************/
|
||||
public List<Outcome> reindex(QuickSearchRuntime runtime, String tableName, List<String> recordIds) throws QException
|
||||
{
|
||||
QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
throw (new QException("Quick Search does not index table [" + tableName + "]"));
|
||||
}
|
||||
if(!runtime.isTableEnabled(tableName))
|
||||
{
|
||||
throw (new QException("Table [" + tableName + "] is disabled in Quick Search Index; enable it before reindexing its records"));
|
||||
}
|
||||
|
||||
Set<String> ids = new LinkedHashSet<>(recordIds);
|
||||
Map<String, QRecord> readable = DiagnosticRecordReads.loadForIndexing(tableName, ids);
|
||||
Set<String> validIds = new HashSet<>();
|
||||
DiagnosticRecordReads.typedIds(QContext.getQInstance().getTable(tableName), ids).forEach(id -> validIds.add(String.valueOf(id)));
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// ids the user cannot read: exists at all (system view)? if not, //
|
||||
// the record is gone and its document should go too //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
List<String> unreadable = ids.stream().filter(id -> !readable.containsKey(id)).toList();
|
||||
Set<String> existing = new HashSet<>();
|
||||
if(!unreadable.isEmpty())
|
||||
{
|
||||
List<String> found = new ArrayList<>();
|
||||
QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), new QSystemUserSession()), () ->
|
||||
found.addAll(DiagnosticRecordReads.existingIds(tableName, unreadable)));
|
||||
existing.addAll(found);
|
||||
}
|
||||
|
||||
QuickSearchOpenSearchClient client = runtime.getClient();
|
||||
List<Outcome> outcomes = new ArrayList<>();
|
||||
|
||||
List<OpenSearchDocument> docs = new ArrayList<>();
|
||||
for(String id : ids)
|
||||
{
|
||||
QRecord record = readable.get(id);
|
||||
if(record != null)
|
||||
{
|
||||
OpenSearchDocument doc = IndexingUtils.buildDocument(record, tableConfig);
|
||||
if(doc != null)
|
||||
{
|
||||
docs.add(doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BulkIndexResult indexResult = client.indexDocuments(docs, Math.max(1, runtime.getConfig().getBulkBatchSize() == null ? 500 : runtime.getConfig().getBulkBatchSize()));
|
||||
Set<String> failedIds = new HashSet<>();
|
||||
for(String failure : indexResult.getErrors())
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// failures read "table:recordId: reason" //
|
||||
////////////////////////////////////////////////////
|
||||
int reasonStart = failure.indexOf(": ");
|
||||
String documentId = reasonStart < 0 ? failure : failure.substring(0, reasonStart);
|
||||
failedIds.add(documentId.substring(documentId.indexOf(':') + 1));
|
||||
}
|
||||
|
||||
List<String> toDelete = new ArrayList<>();
|
||||
for(String id : ids)
|
||||
{
|
||||
if(readable.containsKey(id))
|
||||
{
|
||||
outcomes.add(failedIds.contains(id) ? new Outcome(id, OUTCOME_FAILED, "OpenSearch rejected the document; see the run's errors")
|
||||
: new Outcome(id, OUTCOME_INDEXED, "Rebuilt from the database and written to the index."));
|
||||
}
|
||||
else if(!validIds.contains(id))
|
||||
{
|
||||
outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "Not a valid id for this table."));
|
||||
}
|
||||
else if(existing.contains(id))
|
||||
{
|
||||
outcomes.add(new Outcome(id, OUTCOME_SKIPPED, "The record exists but you cannot read it, so it was not reindexed."));
|
||||
}
|
||||
else
|
||||
{
|
||||
toDelete.add(OpenSearchDocument.buildDocumentId(tableName, id));
|
||||
outcomes.add(new Outcome(id, OUTCOME_DELETED, "The record no longer exists; its document (if any) was removed."));
|
||||
}
|
||||
}
|
||||
|
||||
if(!toDelete.isEmpty())
|
||||
{
|
||||
BulkIndexResult deleteResult = client.deleteDocuments(toDelete, 500);
|
||||
if(!deleteResult.getErrors().isEmpty())
|
||||
{
|
||||
throw (new QException("Removing documents of deleted records failed: " + String.join("; ", deleteResult.getErrors())));
|
||||
}
|
||||
}
|
||||
|
||||
if(!indexResult.getErrors().isEmpty())
|
||||
{
|
||||
outcomes.add(new Outcome("-", OUTCOME_FAILED, "Index errors: " + String.join("; ", indexResult.getErrors())));
|
||||
}
|
||||
|
||||
client.refreshIndex();
|
||||
return (outcomes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render outcomes as HTML.
|
||||
*******************************************************************************/
|
||||
public static String render(String tableName, List<Outcome> outcomes)
|
||||
{
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open();
|
||||
html.heading("Reindex in search: " + tableName);
|
||||
long indexed = outcomes.stream().filter(o -> OUTCOME_INDEXED.equals(o.outcome())).count();
|
||||
long deleted = outcomes.stream().filter(o -> OUTCOME_DELETED.equals(o.outcome())).count();
|
||||
long skipped = outcomes.stream().filter(o -> OUTCOME_SKIPPED.equals(o.outcome())).count();
|
||||
long failed = outcomes.stream().filter(o -> OUTCOME_FAILED.equals(o.outcome())).count();
|
||||
html.status(failed > 0 ? Level.FAIL : (skipped > 0 ? Level.WARN : Level.OK),
|
||||
indexed + " reindexed, " + deleted + " removed (record gone), " + skipped + " skipped, " + failed + " failed. The documents are searchable now.");
|
||||
|
||||
List<String> items = new ArrayList<>();
|
||||
for(Outcome outcome : outcomes)
|
||||
{
|
||||
Level level = switch(outcome.outcome())
|
||||
{
|
||||
case OUTCOME_INDEXED, OUTCOME_DELETED -> Level.OK;
|
||||
case OUTCOME_SKIPPED -> Level.WARN;
|
||||
default -> Level.FAIL;
|
||||
};
|
||||
items.add(DiagnosticsHtml.badge(level) + " #" + DiagnosticsHtml.escape(outcome.recordId()) + ": " + DiagnosticsHtml.escape(outcome.message()));
|
||||
}
|
||||
html.rawList(items);
|
||||
return (html.close().toString());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Which path the global search (core RecordSearchAction, POST
|
||||
** /qqq/v1/search) takes for each table, and why: OpenSearch through the
|
||||
** Quick Search record-search provider, core's own database search, or not
|
||||
** searched at all. Mirrors RecordSearchAction's table selection and
|
||||
** QuickSearchRecordSearchProvider.claimsTable without calling them, so a
|
||||
** diagnosis changes no state (no backoff, no counters).
|
||||
*******************************************************************************/
|
||||
public final class SearchPathDiagnosis
|
||||
{
|
||||
public static final String PATH_OPENSEARCH = "OpenSearch (Quick Search provider)";
|
||||
public static final String PATH_CORE = "Core database search";
|
||||
public static final String PATH_NONE = "Not searched";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record-search provider's state in this JVM.
|
||||
*******************************************************************************/
|
||||
public record ProviderState(String registeredProvider, Boolean quickSearchProvider, Boolean serveCoreRecordSearch, Long backoffRemainingMillis,
|
||||
Long failureCount, Instant lastFailureTime, String lastFailureMessage, Long backoffDeclinedClaims, Integer timeoutMillis, Instant runtimeStartedAt)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One table's path and the reason for it.
|
||||
*******************************************************************************/
|
||||
public record TablePath(String tableName, String tableLabel, Boolean indexed, Boolean enabled, Boolean readable, Boolean coreSearchable, String path, String reason)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class - no instances.
|
||||
*******************************************************************************/
|
||||
private SearchPathDiagnosis()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The provider's state from the instance and the runtime.
|
||||
*******************************************************************************/
|
||||
public static ProviderState providerState(QInstance qInstance, QuickSearchRuntime runtime)
|
||||
{
|
||||
QCodeReference provider = qInstance.getRecordSearchProvider();
|
||||
QuickSearchQBitConfig config = runtime.getConfig();
|
||||
return (new ProviderState(
|
||||
provider == null ? null : provider.getName(),
|
||||
provider != null && QuickSearchRecordSearchProvider.class.getName().equals(provider.getName()),
|
||||
Boolean.TRUE.equals(config.getServeCoreRecordSearch()),
|
||||
runtime.getRecordSearchBackoffRemainingMillis(),
|
||||
runtime.getRecordSearchFailureCount(),
|
||||
runtime.getLastRecordSearchFailureTime(),
|
||||
runtime.getLastRecordSearchFailureMessage(),
|
||||
runtime.getBackoffDeclinedClaimCount(),
|
||||
config.getRecordSearchTimeoutMillis() == null ? QuickSearchQBitConfig.DEFAULT_RECORD_SEARCH_TIMEOUT_MILLIS : config.getRecordSearchTimeoutMillis(),
|
||||
runtime.getStartedAt()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The path of each named table (unknown names are reported as not
|
||||
** searched), as seen by the session of permissionInput.
|
||||
*******************************************************************************/
|
||||
public static List<TablePath> diagnose(QInstance qInstance, QuickSearchRuntime runtime, List<String> tableNames, AbstractActionInput permissionInput)
|
||||
{
|
||||
ProviderState state = providerState(qInstance, runtime);
|
||||
List<TablePath> paths = new ArrayList<>();
|
||||
for(String tableName : CollectionUtils.nonNullList(tableNames))
|
||||
{
|
||||
paths.add(diagnoseTable(qInstance, runtime, state, tableName, permissionInput));
|
||||
}
|
||||
return (paths);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static TablePath diagnoseTable(QInstance qInstance, QuickSearchRuntime runtime, ProviderState state, String tableName, AbstractActionInput permissionInput)
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(tableName);
|
||||
if(table == null)
|
||||
{
|
||||
return (new TablePath(tableName, tableName, false, false, false, false, PATH_NONE, "There is no table with this name."));
|
||||
}
|
||||
|
||||
boolean indexed = runtime.getTableConfig(tableName) != null;
|
||||
boolean enabled = indexed && runtime.isTableEnabled(tableName);
|
||||
boolean readable = PermissionsHelper.hasTablePermission(permissionInput, tableName, TablePermissionSubType.READ);
|
||||
boolean coreSearchable = !CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) && table.isCapabilityEnabled(qInstance.getBackendForTable(tableName), Capability.TABLE_QUERY);
|
||||
String label = table.getLabel() == null ? tableName : table.getLabel();
|
||||
|
||||
if(!coreSearchable)
|
||||
{
|
||||
return (new TablePath(tableName, label, indexed, enabled, readable, false, PATH_NONE,
|
||||
"Global search skips this table: it has no core searchFields (or does not support query)."));
|
||||
}
|
||||
if(!readable)
|
||||
{
|
||||
return (new TablePath(tableName, label, indexed, enabled, false, true, PATH_NONE,
|
||||
"Global search skips this table for you: you do not have read permission on it."));
|
||||
}
|
||||
if(!indexed)
|
||||
{
|
||||
return (new TablePath(tableName, label, false, false, true, true, PATH_CORE, "Quick Search does not index this table."));
|
||||
}
|
||||
if(!Boolean.TRUE.equals(state.serveCoreRecordSearch()))
|
||||
{
|
||||
return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, "serveCoreRecordSearch is off in the Quick Search configuration."));
|
||||
}
|
||||
if(!Boolean.TRUE.equals(state.quickSearchProvider()))
|
||||
{
|
||||
String reason = state.registeredProvider() == null
|
||||
? "No record search provider is registered on the instance."
|
||||
: "Another record search provider is registered (" + state.registeredProvider() + "), so Quick Search does not serve global search.";
|
||||
return (new TablePath(tableName, label, true, enabled, true, true, PATH_CORE, reason));
|
||||
}
|
||||
if(!enabled)
|
||||
{
|
||||
return (new TablePath(tableName, label, true, false, true, true, PATH_CORE, "The table is disabled in Quick Search Index (enabled = false)."));
|
||||
}
|
||||
if(state.backoffRemainingMillis() != null && state.backoffRemainingMillis() > 0)
|
||||
{
|
||||
return (new TablePath(tableName, label, true, true, true, true, PATH_CORE,
|
||||
"Quick Search is backing off for " + Math.max(1, state.backoffRemainingMillis() / 1000) + " more second(s) after an OpenSearch failure"
|
||||
+ (state.lastFailureTime() == null ? "" : " at " + state.lastFailureTime())
|
||||
+ (state.lastFailureMessage() == null ? "" : ": " + state.lastFailureMessage())));
|
||||
}
|
||||
|
||||
return (new TablePath(tableName, label, true, true, true, true, PATH_OPENSEARCH,
|
||||
"Claimed by the Quick Search provider. If OpenSearch fails or takes longer than " + state.timeoutMillis()
|
||||
+ " ms, core searches this table itself for that request and Quick Search backs off for "
|
||||
+ (QuickSearchRuntime.RECORD_SEARCH_BACKOFF_MILLIS / 1000) + " seconds."));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,654 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.AnalyzedToken;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ExplainResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.ProbeHit;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations.SearchProbe;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The search playground: runs a term the way the global search does, as the
|
||||
** current user, and shows everything behind it.
|
||||
** 1. Which path serves each table and why (SearchPathDiagnosis).
|
||||
** 2. The provider's own OpenSearch request (_msearch of record ids), timed,
|
||||
** without starting a backoff when it fails.
|
||||
** 3. The real global search (core RecordSearchAction) as the current user:
|
||||
** what users see, per table, timed. Like any user search, a provider
|
||||
** failure here starts the provider's normal backoff.
|
||||
** 4. Per indexed table, the production query with scores, highlights and
|
||||
** matched clauses, which hits users can read (record locks), which ones
|
||||
** the global search shows, the query DSL and the raw response (hits the
|
||||
** user cannot read are redacted to their id).
|
||||
** 5. Optionally, explain for one hit, and the term's tokens.
|
||||
** Read-only: no index or record is changed.
|
||||
*******************************************************************************/
|
||||
public class SearchPlayground
|
||||
{
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What to run: term, tables (empty for every searchable table), results per
|
||||
** table as in the global search, and an optional hit to explain
|
||||
** ("table:recordId").
|
||||
*******************************************************************************/
|
||||
public record Input(String term, List<String> tableNames, Integer limitPerTable, String explainDocumentId)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One table's probe: the search, which hit ids the user can read, which
|
||||
** the global search showed, and an error when the probe failed.
|
||||
*******************************************************************************/
|
||||
public record TableProbe(String tableName, SearchProbe probe, Set<String> readableIds, Set<String> shownIds, Integer hiddenCount, String error)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Everything the playground found.
|
||||
*******************************************************************************/
|
||||
public record Report(String term, List<String> notices, List<AnalyzedToken> termTokens, String termTokensError,
|
||||
SearchPathDiagnosis.ProviderState providerState, List<SearchPathDiagnosis.TablePath> paths,
|
||||
Map<String, List<String>> providerIds, Map<String, Integer> providerHiddenCounts, Long providerMillis, String providerError,
|
||||
Map<String, List<RecordSearchResult>> coreResults, Long coreMillis, String coreError,
|
||||
List<TableProbe> probes, String explainDocumentId, ExplainResult explain, String explainError,
|
||||
Integer limitPerTable, Integer idsPerTable)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run the playground for the input as the current session.
|
||||
*******************************************************************************/
|
||||
public Report run(QuickSearchRuntime runtime, Input input, AbstractActionInput permissionInput)
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
QuickSearchQBitConfig config = runtime.getConfig();
|
||||
List<String> notices = new ArrayList<>();
|
||||
|
||||
String term = IndexingUtils.normalizeSearchText(input.term());
|
||||
int limitPerTable = input.limitPerTable() == null || input.limitPerTable() <= 0 ? RecordSearchAction.DEFAULT_LIMIT_PER_TABLE : Math.min(input.limitPerTable(), RecordSearchAction.MAX_LIMIT_PER_TABLE);
|
||||
int idsPerTable = Math.min(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE, limitPerTable * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER);
|
||||
|
||||
List<String> tables = resolveTables(qInstance, runtime, input.tableNames(), notices);
|
||||
List<SearchPathDiagnosis.TablePath> paths = SearchPathDiagnosis.diagnose(qInstance, runtime, tables, permissionInput);
|
||||
SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(qInstance, runtime);
|
||||
|
||||
if(term.length() < QuickSearchAction.MIN_TERM_LENGTH)
|
||||
{
|
||||
notices.add("The term is shorter than " + QuickSearchAction.MIN_TERM_LENGTH + " characters: the Quick Search provider returns nothing for it, so only core's own search can match.");
|
||||
}
|
||||
if(term.length() > RecordSearchAction.MAX_SEARCH_TERM_LENGTH)
|
||||
{
|
||||
notices.add("The term is longer than " + RecordSearchAction.MAX_SEARCH_TERM_LENGTH + " characters: the global search rejects it.");
|
||||
}
|
||||
|
||||
QuickSearchDiagnosticsOperations ops = null;
|
||||
String opsError = null;
|
||||
try
|
||||
{
|
||||
ops = new QuickSearchDiagnosticsOperations(runtime.getClient());
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
opsError = e.getMessage();
|
||||
notices.add("The OpenSearch client could not be built: " + e.getMessage());
|
||||
}
|
||||
|
||||
////////////////////////
|
||||
// term tokens //
|
||||
////////////////////////
|
||||
List<AnalyzedToken> termTokens = null;
|
||||
String termTokensError = opsError;
|
||||
if(ops != null && StringUtils.hasContent(term))
|
||||
{
|
||||
try
|
||||
{
|
||||
termTokens = ops.analyze(QuickSearchOpenSearchClient.SEARCH_ANALYZER, term);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
termTokensError = e.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// the provider's own request, for the tables it claims //
|
||||
///////////////////////////////////////////////////////////
|
||||
List<String> claimed = paths.stream().filter(p -> SearchPathDiagnosis.PATH_OPENSEARCH.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
|
||||
Map<String, List<String>> providerIds = null;
|
||||
Map<String, Integer> providerHidden = new LinkedHashMap<>();
|
||||
Long providerMillis = null;
|
||||
String providerError = null;
|
||||
if(!claimed.isEmpty() && ops != null && term.length() >= QuickSearchAction.MIN_TERM_LENGTH)
|
||||
{
|
||||
int timeout = state.timeoutMillis();
|
||||
long start = System.currentTimeMillis();
|
||||
try
|
||||
{
|
||||
providerIds = runtime.getClient().searchRecordIdsPerTable(term, claimed, idsPerTable, runtime.getDiscoveredTables(), timeout);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
providerError = e.getClass().getSimpleName() + ": " + e.getMessage();
|
||||
}
|
||||
providerMillis = System.currentTimeMillis() - start;
|
||||
providerIds = providerIds == null ? null : readableOnly(providerIds, providerHidden);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// the real global search, as the current user //
|
||||
///////////////////////////////////////////////////////
|
||||
Map<String, List<RecordSearchResult>> coreResults = new LinkedHashMap<>();
|
||||
Long coreMillis = null;
|
||||
String coreError = null;
|
||||
List<String> searched = paths.stream().filter(p -> !SearchPathDiagnosis.PATH_NONE.equals(p.path())).map(SearchPathDiagnosis.TablePath::tableName).toList();
|
||||
if(!searched.isEmpty() && StringUtils.hasContent(term))
|
||||
{
|
||||
long start = System.currentTimeMillis();
|
||||
try
|
||||
{
|
||||
RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm(term).withTableNames(searched).withLimitPerTable(limitPerTable));
|
||||
for(RecordSearchResult result : CollectionUtils.nonNullList(output.getResults()))
|
||||
{
|
||||
coreResults.computeIfAbsent(result.getTableName(), k -> new ArrayList<>()).add(result);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
coreError = e.getClass().getSimpleName() + ": " + e.getMessage();
|
||||
}
|
||||
coreMillis = System.currentTimeMillis() - start;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
// per indexed table, the probe search //
|
||||
//////////////////////////////////////////
|
||||
List<TableProbe> probes = new ArrayList<>();
|
||||
for(SearchPathDiagnosis.TablePath path : paths)
|
||||
{
|
||||
if(!Boolean.TRUE.equals(path.indexed()) || !Boolean.TRUE.equals(path.readable()) || term.length() < QuickSearchAction.MIN_TERM_LENGTH)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(ops == null)
|
||||
{
|
||||
probes.add(new TableProbe(path.tableName(), null, Set.of(), Set.of(), 0, opsError));
|
||||
continue;
|
||||
}
|
||||
probes.add(probeTable(ops, runtime, term, path.tableName(), idsPerTable, coreResults.get(path.tableName())));
|
||||
}
|
||||
|
||||
//////////////////////
|
||||
// explain one hit //
|
||||
//////////////////////
|
||||
ExplainResult explain = null;
|
||||
String explainError = null;
|
||||
String explainId = StringUtils.hasContent(input.explainDocumentId()) ? input.explainDocumentId().trim() : null;
|
||||
if(explainId != null && ops != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
explain = explainHit(ops, runtime, term, explainId);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
explainError = e.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
return (new Report(term, notices, termTokens, termTokensError, state, paths, providerIds, providerHidden, providerMillis, providerError,
|
||||
coreResults, coreMillis, coreError, probes, explainId, explain, explainError, limitPerTable, idsPerTable));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The requested tables that exist (else every table with core searchFields
|
||||
** or a Quick Search index), in instance order.
|
||||
*******************************************************************************/
|
||||
static List<String> resolveTables(QInstance qInstance, QuickSearchRuntime runtime, List<String> requested, List<String> notices)
|
||||
{
|
||||
List<String> tables = new ArrayList<>();
|
||||
if(!CollectionUtils.nullSafeIsEmpty(requested))
|
||||
{
|
||||
for(String tableName : new LinkedHashSet<>(requested))
|
||||
{
|
||||
if(qInstance.getTable(tableName) == null)
|
||||
{
|
||||
notices.add("Ignored unknown table [" + tableName + "].");
|
||||
}
|
||||
else
|
||||
{
|
||||
tables.add(tableName);
|
||||
}
|
||||
}
|
||||
return (tables);
|
||||
}
|
||||
|
||||
for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values())
|
||||
{
|
||||
if(!CollectionUtils.nullSafeIsEmpty(table.getSearchFields()) || runtime.getTableConfig(table.getName()) != null)
|
||||
{
|
||||
tables.add(table.getName());
|
||||
}
|
||||
}
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Probe one table and mark which hits the user can read and which ones the
|
||||
** global search showed.
|
||||
*******************************************************************************/
|
||||
private TableProbe probeTable(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String tableName, int size, List<RecordSearchResult> shown)
|
||||
{
|
||||
try
|
||||
{
|
||||
SearchProbe probe = ops.probeSearch(term, tableName, size, runtime.getDiscoveredTables());
|
||||
List<String> ids = probe.hits().stream().map(ProbeHit::recordId).toList();
|
||||
|
||||
Set<String> readable = new HashSet<>(DiagnosticRecordReads.existingIds(tableName, ids));
|
||||
Set<String> shownIds = new HashSet<>();
|
||||
for(RecordSearchResult result : CollectionUtils.nonNullList(shown))
|
||||
{
|
||||
shownIds.add(String.valueOf(result.getRecordId()));
|
||||
}
|
||||
|
||||
List<ProbeHit> visible = probe.hits().stream().filter(h -> readable.contains(h.recordId())).toList();
|
||||
int hidden = probe.hits().size() - visible.size();
|
||||
SearchProbe redacted = new SearchProbe(probe.tableName(), probe.requestJson(), redactResponse(probe.responseJson(), readable),
|
||||
probe.tookMillis(), probe.wallMillis(), hidden == 0 ? probe.totalHits() : null, probe.totalIsLowerBound(), visible);
|
||||
return (new TableProbe(tableName, redacted, readable, shownIds, hidden, null));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
return (new TableProbe(tableName, null, Set.of(), Set.of(), 0, e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The provider's ids per table, keeping only records the user can read;
|
||||
** how many were dropped goes in hiddenCounts. Ids of records the user
|
||||
** cannot read are never shown (they would reveal what a locked record
|
||||
** contains).
|
||||
*******************************************************************************/
|
||||
static Map<String, List<String>> readableOnly(Map<String, List<String>> idsByTable, Map<String, Integer> hiddenCounts)
|
||||
{
|
||||
Map<String, List<String>> readable = new LinkedHashMap<>();
|
||||
for(Map.Entry<String, List<String>> entry : idsByTable.entrySet())
|
||||
{
|
||||
List<String> ids = entry.getValue() == null ? List.of() : entry.getValue();
|
||||
Set<String> ok;
|
||||
try
|
||||
{
|
||||
ok = new HashSet<>(DiagnosticRecordReads.existingIds(entry.getKey(), ids));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
ok = Set.of();
|
||||
}
|
||||
Set<String> finalOk = ok;
|
||||
readable.put(entry.getKey(), ids.stream().filter(finalOk::contains).toList());
|
||||
hiddenCounts.put(entry.getKey(), (int) ids.stream().filter(id -> !finalOk.contains(id)).count());
|
||||
}
|
||||
return (readable);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The raw response without the hits the user cannot read: they are
|
||||
** removed whole (id, score, source, highlights, matched queries), and
|
||||
** when any was removed the total and max score are redacted too, so the
|
||||
** response reveals nothing about records the user cannot read.
|
||||
*******************************************************************************/
|
||||
static String redactResponse(String responseJson, Set<String> readableIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
JsonNode root = MAPPER.readTree(responseJson);
|
||||
JsonNode hitList = root.path("hits").path("hits");
|
||||
int removed = 0;
|
||||
if(hitList.isArray())
|
||||
{
|
||||
ArrayNode array = (ArrayNode) hitList;
|
||||
for(int i = array.size() - 1; i >= 0; i--)
|
||||
{
|
||||
String recordId = array.get(i).path("_source").path("recordId").asText(null);
|
||||
if(recordId == null || !readableIds.contains(recordId))
|
||||
{
|
||||
array.remove(i);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(removed > 0 && root.path("hits") instanceof ObjectNode hits)
|
||||
{
|
||||
hits.put("total", "redacted: " + removed + " hit(s) you cannot read were removed");
|
||||
hits.remove("max_score");
|
||||
}
|
||||
return (root.toString());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
return ("{\"redacted\":\"response could not be parsed for redaction\"}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Explain one hit ("table:recordId") when the table is indexed and the user
|
||||
** can read the record.
|
||||
*******************************************************************************/
|
||||
private ExplainResult explainHit(QuickSearchDiagnosticsOperations ops, QuickSearchRuntime runtime, String term, String documentId) throws QException
|
||||
{
|
||||
int colon = documentId.indexOf(':');
|
||||
if(colon <= 0 || colon == documentId.length() - 1)
|
||||
{
|
||||
throw (new QException("Give the hit as table:recordId (its document id), for example blogPost:42"));
|
||||
}
|
||||
String tableName = documentId.substring(0, colon);
|
||||
String recordId = documentId.substring(colon + 1);
|
||||
|
||||
QuickSearchableTableConfig tableConfig = runtime.getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
throw (new QException("Table [" + tableName + "] is not indexed by Quick Search"));
|
||||
}
|
||||
if(DiagnosticRecordReads.existingIds(tableName, List.of(recordId)).isEmpty())
|
||||
{
|
||||
throw (new QException("Record " + recordId + " of [" + tableName + "] does not exist or you cannot read it"));
|
||||
}
|
||||
return (ops.explain(term, tableName, recordId, runtime.getDiscoveredTables()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render a report as HTML.
|
||||
*******************************************************************************/
|
||||
public static String render(Report report)
|
||||
{
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open();
|
||||
html.heading("Search playground: \"" + report.term() + "\"");
|
||||
html.paragraph("Run as you, at " + Instant.now() + ". The global search shows up to " + report.limitPerTable() + " results per table; the Quick Search provider asks OpenSearch for the top "
|
||||
+ report.idsPerTable() + " ids per table and core keeps the first " + report.limitPerTable() + " you can read.");
|
||||
|
||||
for(String notice : report.notices())
|
||||
{
|
||||
html.status(Level.WARN, notice);
|
||||
}
|
||||
|
||||
renderProviderState(html, report.providerState());
|
||||
|
||||
html.subheading("Path per table");
|
||||
List<String> pathItems = new ArrayList<>();
|
||||
for(SearchPathDiagnosis.TablePath path : report.paths())
|
||||
{
|
||||
Level level = SearchPathDiagnosis.PATH_OPENSEARCH.equals(path.path()) ? Level.OK : (SearchPathDiagnosis.PATH_CORE.equals(path.path()) ? Level.INFO : Level.WARN);
|
||||
pathItems.add(DiagnosticsHtml.badge(level) + " <strong>" + DiagnosticsHtml.escape(path.tableLabel()) + "</strong> (" + DiagnosticsHtml.escape(path.tableName()) + "): "
|
||||
+ DiagnosticsHtml.escape(path.path()) + ". " + DiagnosticsHtml.escape(path.reason()));
|
||||
}
|
||||
if(pathItems.isEmpty())
|
||||
{
|
||||
html.paragraph("No table is searchable by the global search.");
|
||||
}
|
||||
html.rawList(pathItems);
|
||||
|
||||
html.subheading("Term tokens (search analyzer)");
|
||||
if(report.termTokensError() != null)
|
||||
{
|
||||
html.status(Level.FAIL, "Could not analyze the term: " + report.termTokensError());
|
||||
}
|
||||
else if(report.termTokens() != null)
|
||||
{
|
||||
html.paragraph(tokenSummary(report.termTokens()));
|
||||
for(AnalyzedToken token : report.termTokens())
|
||||
{
|
||||
if(token.token().length() < 2)
|
||||
{
|
||||
html.status(Level.WARN, "\"" + token.token() + "\" is one character: the index holds edge n-grams of 2 to 20 characters, so it can never match, and every term must match.");
|
||||
}
|
||||
else if(token.token().length() > 20)
|
||||
{
|
||||
html.status(Level.WARN, "\"" + token.token() + "\" is longer than 20 characters: the index holds edge n-grams of at most 20 characters, so it can never match.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
html.subheading("Quick Search provider request");
|
||||
if(report.providerMillis() == null)
|
||||
{
|
||||
html.paragraph("Not run: the provider claims none of these tables (see the paths above), or the term is too short.");
|
||||
}
|
||||
else if(report.providerError() != null)
|
||||
{
|
||||
html.status(Level.FAIL, "Failed after " + report.providerMillis() + " ms: " + report.providerError() + ". A user search hitting this falls back to core's database search for these tables.");
|
||||
}
|
||||
else
|
||||
{
|
||||
html.status(Level.OK, "Answered in " + report.providerMillis() + " ms (wall clock, limit " + report.providerState().timeoutMillis() + " ms).");
|
||||
List<String> items = new ArrayList<>();
|
||||
for(Map.Entry<String, List<String>> entry : report.providerIds().entrySet())
|
||||
{
|
||||
Integer hidden = report.providerHiddenCounts().getOrDefault(entry.getKey(), 0);
|
||||
items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + entry.getValue().size() + " id(s) you can read"
|
||||
+ (hidden > 0 ? " (" + hidden + " more you cannot read, not shown)" : "")
|
||||
+ (entry.getValue().isEmpty() ? "" : ": " + DiagnosticsHtml.escape(String.join(", ", entry.getValue().subList(0, Math.min(20, entry.getValue().size()))))
|
||||
+ (entry.getValue().size() > 20 ? ", ..." : "")));
|
||||
}
|
||||
html.rawList(items);
|
||||
}
|
||||
|
||||
html.subheading("Global search results (what you see)");
|
||||
if(report.coreMillis() == null)
|
||||
{
|
||||
html.paragraph("Not run: no table is searchable, or the term is empty.");
|
||||
}
|
||||
else if(report.coreError() != null)
|
||||
{
|
||||
html.status(Level.FAIL, "The global search failed after " + report.coreMillis() + " ms: " + report.coreError());
|
||||
}
|
||||
else
|
||||
{
|
||||
int total = report.coreResults().values().stream().mapToInt(List::size).sum();
|
||||
html.status(total == 0 ? Level.INFO : Level.OK, total + " result(s) in " + report.coreMillis() + " ms.");
|
||||
List<String> items = new ArrayList<>();
|
||||
for(Map.Entry<String, List<RecordSearchResult>> entry : report.coreResults().entrySet())
|
||||
{
|
||||
List<String> labels = new ArrayList<>();
|
||||
for(RecordSearchResult result : entry.getValue())
|
||||
{
|
||||
labels.add(DiagnosticsHtml.escape(result.getRecordLabel() == null ? String.valueOf(result.getRecordId()) : result.getRecordLabel()) + " (#" + DiagnosticsHtml.escape(result.getRecordId()) + ")");
|
||||
}
|
||||
items.add("<strong>" + DiagnosticsHtml.escape(entry.getKey()) + "</strong>: " + String.join("; ", labels));
|
||||
}
|
||||
html.rawList(items);
|
||||
}
|
||||
|
||||
html.subheading("OpenSearch detail per indexed table");
|
||||
if(report.probes().isEmpty())
|
||||
{
|
||||
html.paragraph("No indexed table you can read was searched, or the term is too short.");
|
||||
}
|
||||
for(TableProbe tableProbe : report.probes())
|
||||
{
|
||||
renderProbe(html, tableProbe);
|
||||
}
|
||||
|
||||
if(report.explainDocumentId() != null)
|
||||
{
|
||||
html.subheading("Explain " + report.explainDocumentId());
|
||||
if(report.explainError() != null)
|
||||
{
|
||||
html.status(Level.FAIL, report.explainError());
|
||||
}
|
||||
else if(report.explain() != null && !Boolean.TRUE.equals(report.explain().documentFound()))
|
||||
{
|
||||
html.status(Level.FAIL, "There is no document " + report.explainDocumentId() + " in the index.");
|
||||
}
|
||||
else if(report.explain() != null)
|
||||
{
|
||||
html.status(Boolean.TRUE.equals(report.explain().matched()) ? Level.OK : Level.WARN,
|
||||
Boolean.TRUE.equals(report.explain().matched()) ? "Matches with score " + report.explain().score() + "." : "Does not match this term.");
|
||||
html.textDetails("Score explanation", report.explain().explanationText());
|
||||
html.jsonDetails("Explain response (JSON)", report.explain().json());
|
||||
}
|
||||
}
|
||||
|
||||
return (html.close().toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void renderProviderState(DiagnosticsHtml html, SearchPathDiagnosis.ProviderState state)
|
||||
{
|
||||
html.subheading("Record search provider (this server)");
|
||||
List<String[]> facts = new ArrayList<>();
|
||||
facts.add(new String[] { "Registered provider", state.registeredProvider() == null ? null : state.registeredProvider() + (Boolean.TRUE.equals(state.quickSearchProvider()) ? " (Quick Search)" : " (not Quick Search)") });
|
||||
facts.add(new String[] { "serveCoreRecordSearch", String.valueOf(state.serveCoreRecordSearch()) });
|
||||
facts.add(new String[] { "Timeout", state.timeoutMillis() + " ms" });
|
||||
facts.add(new String[] { "Backing off", state.backoffRemainingMillis() > 0 ? "yes, " + (state.backoffRemainingMillis() / 1000) + " s left" : "no" });
|
||||
facts.add(new String[] { "Failures since " + state.runtimeStartedAt(), String.valueOf(state.failureCount()) });
|
||||
facts.add(new String[] { "Table searches core ran itself during backoffs", String.valueOf(state.backoffDeclinedClaims()) });
|
||||
facts.add(new String[] { "Last failure", state.lastFailureTime() == null ? null : state.lastFailureTime() + ": " + state.lastFailureMessage() });
|
||||
html.facts(facts);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void renderProbe(DiagnosticsHtml html, TableProbe tableProbe)
|
||||
{
|
||||
html.raw("<h4>" + DiagnosticsHtml.escape(tableProbe.tableName()) + "</h4>");
|
||||
if(tableProbe.error() != null)
|
||||
{
|
||||
html.status(Level.FAIL, "OpenSearch search failed: " + tableProbe.error());
|
||||
return;
|
||||
}
|
||||
|
||||
SearchProbe probe = tableProbe.probe();
|
||||
html.paragraph((probe.totalHits() == null ? "Matching documents: not shown (some are records you cannot read)"
|
||||
: probe.totalHits() + (Boolean.TRUE.equals(probe.totalIsLowerBound()) ? "+" : "") + " matching document(s)")
|
||||
+ "; OpenSearch took " + probe.tookMillis() + " ms (" + probe.wallMillis() + " ms wall clock). Showing the " + probe.hits().size() + " top hit(s) you can read.");
|
||||
if(tableProbe.hiddenCount() != null && tableProbe.hiddenCount() > 0)
|
||||
{
|
||||
html.status(Level.INFO, tableProbe.hiddenCount() + " of the top hits are records you cannot read (deleted, or hidden by a record security lock); they are not shown.");
|
||||
}
|
||||
|
||||
List<String> items = new ArrayList<>();
|
||||
int position = 0;
|
||||
for(ProbeHit hit : probe.hits())
|
||||
{
|
||||
position++;
|
||||
StringBuilder item = new StringBuilder();
|
||||
item.append("#").append(position).append(" <strong>")
|
||||
.append(DiagnosticsHtml.escape(hit.recordLabel() == null ? "(no label)" : hit.recordLabel()))
|
||||
.append("</strong> · document <code>").append(DiagnosticsHtml.escape(OpenSearchDocument.buildDocumentId(tableProbe.tableName(), hit.recordId()))).append("</code>")
|
||||
.append(" · score ").append(hit.score() == null ? "n/a" : String.format("%.3f", hit.score()))
|
||||
.append(" · ").append(tableProbe.shownIds().contains(hit.recordId()) ? "shown in global search" : "not shown (beyond the per-table limit)");
|
||||
if(!hit.matchedClauses().isEmpty())
|
||||
{
|
||||
List<String> fields = hit.matchedClauses().stream().map(c -> c.startsWith(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX) ? c.substring(QuickSearchOpenSearchClient.FIELD_CLAUSE_PREFIX.length()) : c).toList();
|
||||
item.append("<br>Matched: ").append(DiagnosticsHtml.escape(String.join(", ", fields)));
|
||||
}
|
||||
for(Map.Entry<String, List<String>> highlight : hit.highlights().entrySet())
|
||||
{
|
||||
item.append("<br><em>").append(DiagnosticsHtml.escape(highlight.getKey())).append(":</em> ")
|
||||
.append(DiagnosticsHtml.highlight(String.join(" ... ", highlight.getValue())));
|
||||
}
|
||||
items.add(item.toString());
|
||||
}
|
||||
if(items.isEmpty())
|
||||
{
|
||||
html.paragraph("No document of this table that you can read matches every term.");
|
||||
}
|
||||
html.rawList(items);
|
||||
html.jsonDetails("Query DSL", probe.requestJson());
|
||||
html.jsonDetails("Raw response (hits you cannot read removed)", probe.responseJson());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** "a, b, c" of token strings.
|
||||
*******************************************************************************/
|
||||
static String tokenSummary(List<AnalyzedToken> tokens)
|
||||
{
|
||||
if(tokens.isEmpty())
|
||||
{
|
||||
return ("No tokens: the term has no searchable characters.");
|
||||
}
|
||||
List<String> parts = new ArrayList<>();
|
||||
for(AnalyzedToken token : tokens)
|
||||
{
|
||||
parts.add("\"" + token.token() + "\"");
|
||||
}
|
||||
return (tokens.size() + " token(s), all of which must match: " + String.join(", ", parts));
|
||||
}
|
||||
|
||||
}
|
||||
+14
-20
@@ -51,24 +51,19 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** Real-time indexing hook. Registered once per QInstance; applies to the
|
||||
** tables this QBit indexes.
|
||||
**
|
||||
** Events are built while the write's transaction is open, but published only
|
||||
** after that transaction commits, via
|
||||
** Events are built while the write's transaction is open (so inserted and
|
||||
** updated records can be re-read in full, with display values), but
|
||||
** published only after that transaction commits, via
|
||||
** QBackendTransaction.addAfterCommitCallback. A write without a transaction
|
||||
** is already committed, so it publishes immediately. A rolled-back write never
|
||||
** reaches the index.
|
||||
**
|
||||
** Inserted and updated records are re-read inside that transaction, with
|
||||
** display values and possible-value labels, so a real-time document matches
|
||||
** the one basepull, reconcile and full reindex build. The event's own records
|
||||
** are not enough: for an update they are the sparse input, and for an insert
|
||||
** they carry only raw values.
|
||||
**
|
||||
** This listener never fails the host's write. When the re-read of inserted or
|
||||
** updated records or publishing fails (including a LinkageError from a missing
|
||||
** optional library while the client is built), each affected (table, record,
|
||||
** action) is written to quickSearchFailedEvent for the scheduled basepull to
|
||||
** replay. A failed re-read is recorded after the commit too, so a rolled-back
|
||||
** write leaves no row.
|
||||
** This listener never fails the host's write. When the re-read of inserted
|
||||
** or updated records or publishing fails (including a LinkageError from a
|
||||
** missing optional library while the client is built), each affected (table,
|
||||
** record, action) is written to quickSearchFailedEvent for the scheduled
|
||||
** basepull to replay. A failed re-read is recorded after the commit too, so a
|
||||
** rolled-back write leaves no row.
|
||||
**
|
||||
** While a full reindex rebuilds into a new physical index, the alias still
|
||||
** points at the old one, so a published delete never reaches the new index.
|
||||
@@ -135,9 +130,8 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
|
||||
// would leave no trace. Then record the written keys after commit, //
|
||||
// so basepull re-reads and indexes them. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
LOG.warn("Could not re-read " + (event.getType() == RecordChangeType.INSERT ? "inserted" : "updated")
|
||||
+ " records for indexing; recording them for replay once the write commits", e,
|
||||
logPair("tableName", tableName), logPair("recordCount", event.getRecords().size()));
|
||||
LOG.warn("Could not re-read inserted or updated records for indexing; recording them for replay once the write commits", e,
|
||||
logPair("tableName", tableName), logPair("changeType", event.getType()), logPair("recordCount", event.getRecords().size()));
|
||||
List<IndexEvent> failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords());
|
||||
runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e));
|
||||
return;
|
||||
@@ -450,9 +444,9 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The event's records are "as the backend returned them": for an update on
|
||||
** RDBMS that is the sparse input, and for an insert the raw values with no
|
||||
** display values. Re-read the full rows inside the same transaction, with
|
||||
** Changed records are "as the backend returned them": for an update on
|
||||
** RDBMS that is the sparse input, and an insert has raw values only. Re-read
|
||||
** the full rows inside the same transaction, in one query per event, with
|
||||
** display values so possible-value labels are indexed (which can run host
|
||||
** possible-value providers).
|
||||
*******************************************************************************/
|
||||
|
||||
+1
-13
@@ -17,12 +17,9 @@
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
@@ -30,11 +27,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaData
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -67,14 +62,7 @@ public class QuickSearchAdminAppMetaDataProducer extends QBitComponentMetaDataPr
|
||||
return (app);
|
||||
}
|
||||
|
||||
QWidgetMetaData diagnosticsWidget = produced.get(QWidgetMetaData.class, config.applyPrefix(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME));
|
||||
if(diagnosticsWidget != null)
|
||||
{
|
||||
app.withWidgets(new ArrayList<>(List.of(diagnosticsWidget.getName())));
|
||||
}
|
||||
|
||||
for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME, QuickSearchFailedEvent.TABLE_NAME,
|
||||
QuickSearchValidationRun.TABLE_NAME, QuickSearchDiagnosticEvent.TABLE_NAME })
|
||||
for(String tableName : new String[] { QuickSearchIndex.TABLE_NAME, QuickSearchIndexRun.TABLE_NAME, QuickSearchFailedEvent.TABLE_NAME })
|
||||
{
|
||||
QTableMetaData table = produced.get(QTableMetaData.class, config.applyPrefix(tableName));
|
||||
if(table != null)
|
||||
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelsMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDeveloperPanelProvider;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Registers the Quick Search record developer panel
|
||||
** ({@link QuickSearchDeveloperPanelProvider}) in the instance's developer
|
||||
** panels. The provider decides per table and user whether the panel shows
|
||||
** (indexed tables, users who may read Quick Search Index). Registration is
|
||||
** idempotent, so producing the QBit again adds nothing twice.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDeveloperPanelsMetaDataProducer extends QBitComponentMetaDataProducer<DeveloperPanelsMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public DeveloperPanelsMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (DeveloperPanelsMetaData.ofOrWithNew(qInstance).withProvider(new QCodeReference(QuickSearchDeveloperPanelProvider.class)));
|
||||
}
|
||||
|
||||
}
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the quickSearchDiagnosticEvent table when enableDiagnosticTools is true.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticEventTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchTableMetaDataHelper.buildDiagnosticEventTable(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tables before processes and the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
-279
@@ -1,279 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.AbstractDiagnosticToolStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.AnalyzeTextStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.InspectRecordStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.ReindexRecordsStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.SearchPlaygroundStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.ValidateIndexStep;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Builds the diagnostic tool processes. Every one is a table or record
|
||||
** action (it has a tableName and is never added to an app's nav):
|
||||
** - on Quick Search Index: Search Playground, Validate Search Index (also
|
||||
** schedulable), Test Search Analyzer;
|
||||
** - on every indexed table and on Quick Search Failed Event: Inspect Search
|
||||
** Document and Reindex in Search.
|
||||
** Read-only tools use debugPermissionRules; Reindex in Search uses
|
||||
** adminPermissionRules.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticsProcessMetaDataHelper
|
||||
{
|
||||
public static final String PLAYGROUND_PROCESS_NAME = "quickSearchPlayground";
|
||||
public static final String VALIDATE_PROCESS_NAME = "quickSearchValidateIndex";
|
||||
public static final String ANALYZE_PROCESS_NAME = "quickSearchAnalyzeText";
|
||||
public static final String INSPECT_PROCESS_PREFIX = "quickSearchInspect";
|
||||
public static final String REINDEX_RECORDS_PROCESS_PREFIX = "quickSearchReindexRecords";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Name of the Inspect Search Document process of a table.
|
||||
*******************************************************************************/
|
||||
public static String inspectProcessName(QuickSearchQBitConfig config, String tableName)
|
||||
{
|
||||
return (config.applyPrefix(INSPECT_PROCESS_PREFIX) + "_" + tableName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Name of the Reindex in Search process of a table.
|
||||
*******************************************************************************/
|
||||
public static String reindexRecordsProcessName(QuickSearchQBitConfig config, String tableName)
|
||||
{
|
||||
return (config.applyPrefix(REINDEX_RECORDS_PROCESS_PREFIX) + "_" + tableName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Search Playground, a table action on Quick Search Index.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildPlaygroundProcess(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QProcessMetaData()
|
||||
.withName(config.applyPrefix(PLAYGROUND_PROCESS_NAME))
|
||||
.withLabel("Search Playground")
|
||||
.withIcon(new QIcon("travel_explore"))
|
||||
.withTableName(config.getQuickSearchIndexTableName())
|
||||
.withMinInputRecords(0)
|
||||
.withPermissionRules(clone(config.getDebugPermissionRules()))
|
||||
.withStep(backend(SearchPlaygroundStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
|
||||
.withStep(form("Search", List.of(
|
||||
new QFieldMetaData(SearchPlaygroundStep.FIELD_TERM, QFieldType.STRING).withLabel("Search term").withIsRequired(true).withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH),
|
||||
new QFieldMetaData(SearchPlaygroundStep.FIELD_TABLES, QFieldType.STRING).withLabel("Tables (optional; names separated by commas)"),
|
||||
new QFieldMetaData(SearchPlaygroundStep.FIELD_LIMIT_PER_TABLE, QFieldType.INTEGER).withLabel("Results per table").withDefaultValue(RecordSearchAction.DEFAULT_LIMIT_PER_TABLE),
|
||||
new QFieldMetaData(SearchPlaygroundStep.FIELD_EXPLAIN_HIT, QFieldType.STRING).withLabel("Explain hit (optional document id, e.g. blogPost:42)"))))
|
||||
.withStep(backend(SearchPlaygroundStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test Search Analyzer, a table action on Quick Search Index.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildAnalyzeProcess(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QProcessMetaData()
|
||||
.withName(config.applyPrefix(ANALYZE_PROCESS_NAME))
|
||||
.withLabel("Test Search Analyzer")
|
||||
.withIcon(new QIcon("text_fields"))
|
||||
.withTableName(config.getQuickSearchIndexTableName())
|
||||
.withMinInputRecords(0)
|
||||
.withPermissionRules(clone(config.getDebugPermissionRules()))
|
||||
.withStep(backend(AnalyzeTextStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
|
||||
.withStep(form("Analyze", List.of(
|
||||
new QFieldMetaData(AnalyzeTextStep.FIELD_TEXT, QFieldType.TEXT).withLabel("Text (as a field value)").withIsRequired(true),
|
||||
new QFieldMetaData(AnalyzeTextStep.FIELD_QUERY, QFieldType.STRING).withLabel("Search term (optional)").withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH))))
|
||||
.withStep(backend(AnalyzeTextStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate Search Index, a table action on Quick Search Index; scheduled
|
||||
** by cron when validationCronExpression and a schedulerName are set.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildValidateProcess(QuickSearchQBitConfig config)
|
||||
{
|
||||
QProcessMetaData process = new QProcessMetaData()
|
||||
.withName(config.applyPrefix(VALIDATE_PROCESS_NAME))
|
||||
.withLabel("Validate Search Index")
|
||||
.withIcon(new QIcon("fact_check"))
|
||||
.withTableName(config.getQuickSearchIndexTableName())
|
||||
.withMinInputRecords(0)
|
||||
.withPermissionRules(clone(config.getDebugPermissionRules()))
|
||||
.withStep(backend(ValidateIndexStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, null))
|
||||
.withStep(form("Validate", List.of(
|
||||
new QFieldMetaData(ValidateIndexStep.FIELD_SAMPLE_SIZE, QFieldType.INTEGER).withLabel("Records and documents to sample per table")
|
||||
.withDefaultValue(config.getValidationSampleSize()))))
|
||||
.withStep(backend(ValidateIndexStep.class, AbstractDiagnosticToolStep.STEP_RUN, null))
|
||||
.withStep(result(AbstractDiagnosticToolStep.STEP_RESULT, "Validation report"));
|
||||
|
||||
if(StringUtils.hasContent(config.getSchedulerName()) && StringUtils.hasContent(config.getValidationCronExpression()))
|
||||
{
|
||||
process.withSchedule(new QScheduleMetaData()
|
||||
.withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(config.getValidationCronExpression())
|
||||
.withCronTimeZoneId(config.getValidationCronTimeZoneId())
|
||||
.withDescription("Quick Search index validation"));
|
||||
}
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect Search Document, a record action on a table (an indexed table,
|
||||
** or Quick Search Failed Event for the event's record).
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildInspectProcess(QuickSearchQBitConfig config, String tableName)
|
||||
{
|
||||
return (new QProcessMetaData()
|
||||
.withName(inspectProcessName(config, tableName))
|
||||
.withLabel("Inspect Search Document")
|
||||
.withIcon(new QIcon("manage_search"))
|
||||
.withTableName(tableName)
|
||||
.withMinInputRecords(1)
|
||||
.withMaxInputRecords(InspectRecordStep.MAX_RECORDS)
|
||||
.withPermissionRules(clone(config.getDebugPermissionRules()))
|
||||
.withStep(backend(InspectRecordStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, tableName))
|
||||
.withStep(form("Inspect", List.of(
|
||||
new QFieldMetaData(InspectRecordStep.FIELD_TERM, QFieldType.STRING).withLabel("Why doesn't it match... (optional search term)")
|
||||
.withMaxLength(RecordSearchAction.MAX_SEARCH_TERM_LENGTH))))
|
||||
.withStep(backend(InspectRecordStep.class, AbstractDiagnosticToolStep.STEP_RUN, null)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reindex in Search, a record action on a table (an indexed table, or
|
||||
** Quick Search Failed Event for the event's record). Writes to the index,
|
||||
** so it uses the admin permission rules.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData buildReindexRecordsProcess(QuickSearchQBitConfig config, String tableName)
|
||||
{
|
||||
return (new QProcessMetaData()
|
||||
.withName(reindexRecordsProcessName(config, tableName))
|
||||
.withLabel("Reindex in Search")
|
||||
.withIcon(new QIcon("sync"))
|
||||
.withTableName(tableName)
|
||||
.withMinInputRecords(1)
|
||||
.withMaxInputRecords(ReindexRecordsStep.MAX_RECORDS)
|
||||
.withPermissionRules(clone(config.getAdminPermissionRules()))
|
||||
.withStep(backend(ReindexRecordsStep.class, AbstractDiagnosticToolStep.STEP_PREPARE, tableName))
|
||||
.withStep(result(AbstractDiagnosticToolStep.STEP_CONFIRM, "Confirm"))
|
||||
.withStep(backend(ReindexRecordsStep.class, AbstractDiagnosticToolStep.STEP_RUN, null))
|
||||
.withStep(result(AbstractDiagnosticToolStep.STEP_RESULT, "Reindexed")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The fixed processes (on Quick Search Index and Quick Search Failed
|
||||
** Event).
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> buildFixedProcesses(QuickSearchQBitConfig config)
|
||||
{
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
processes.add(buildPlaygroundProcess(config));
|
||||
processes.add(buildValidateProcess(config));
|
||||
processes.add(buildAnalyzeProcess(config));
|
||||
processes.add(buildInspectProcess(config, config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)));
|
||||
processes.add(buildReindexRecordsProcess(config, config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)));
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A backend step of a tool; with recordsTableName, the step receives the
|
||||
** selected records of that table.
|
||||
*******************************************************************************/
|
||||
private static QBackendStepMetaData backend(Class<?> stepClass, String name, String recordsTableName)
|
||||
{
|
||||
QBackendStepMetaData step = new QBackendStepMetaData().withName(name).withCode(new QCodeReference(stepClass));
|
||||
if(recordsTableName != null)
|
||||
{
|
||||
step.withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName(recordsTableName)));
|
||||
}
|
||||
return (step);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The FORM step: fields above, the tool's HTML below.
|
||||
*******************************************************************************/
|
||||
private static QFrontendStepMetaData form(String label, List<QFieldMetaData> fields)
|
||||
{
|
||||
QFrontendStepMetaData step = new QFrontendStepMetaData()
|
||||
.withName(AbstractDiagnosticToolStep.STEP_FORM)
|
||||
.withLabel(label)
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML));
|
||||
fields.forEach(step::withFormField);
|
||||
return (step);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An HTML-only frontend step.
|
||||
*******************************************************************************/
|
||||
private static QFrontendStepMetaData result(String name, String label)
|
||||
{
|
||||
return (new QFrontendStepMetaData()
|
||||
.withName(name)
|
||||
.withLabel(label)
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules clone(QPermissionRules rules)
|
||||
{
|
||||
return (rules == null ? null : rules.clone());
|
||||
}
|
||||
|
||||
}
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the fixed diagnostic tool processes (Search Playground, Validate
|
||||
** Search Index, Test Search Analyzer, and Inspect / Reindex on Quick Search
|
||||
** Failed Event) when enableDiagnosticTools is true. The per-table record
|
||||
** actions are added by QuickSearchQBitProducer once the indexed tables are
|
||||
** known.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticsProcessesMetaDataProducer extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
|
||||
{
|
||||
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
|
||||
for(QProcessMetaData process : QuickSearchDiagnosticsProcessMetaDataHelper.buildFixedProcesses(getQBitConfig()))
|
||||
{
|
||||
output.add(process);
|
||||
}
|
||||
return (output);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** after tables, before the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (200);
|
||||
}
|
||||
|
||||
}
|
||||
-79
@@ -1,79 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDiagnosticsWidgetRenderer;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the Search Diagnostics widget (quickSearchDiagnostics) when
|
||||
** enableDiagnosticTools is true. The admin app shows it on its home; a host
|
||||
** with its own Quick Search home adds it there by name.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticsWidgetMetaDataProducer extends QBitComponentMetaDataProducer<QWidgetMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
public static final String WIDGET_NAME = "quickSearchDiagnostics";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QWidgetMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getQBitConfig();
|
||||
return (new QWidgetMetaData()
|
||||
.withName(config.applyPrefix(WIDGET_NAME))
|
||||
.withLabel("Search Diagnostics")
|
||||
.withIcon("troubleshoot")
|
||||
.withType(WidgetType.HTML.getType())
|
||||
.withGridColumns(12)
|
||||
.withIsCard(true)
|
||||
.withCodeReference(new QCodeReference(QuickSearchDiagnosticsWidgetRenderer.class))
|
||||
.withPermissionRules(config.getDebugPermissionRules() == null ? null : config.getDebugPermissionRules().clone()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** after tables, before the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (300);
|
||||
}
|
||||
|
||||
}
|
||||
-85
@@ -17,22 +17,16 @@
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -132,85 +126,6 @@ public class QuickSearchTableMetaDataHelper
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** quickSearchDiagnosticEvent: failed searches, diagnostic tool runs (the
|
||||
** audit trail) and refused runs. Written by the QBit only: insert, update
|
||||
** and delete are disabled for users and the API (core actions, which the
|
||||
** QBit uses, do not check capabilities).
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData buildDiagnosticEventTable(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(config.applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME))
|
||||
.withLabel("Quick Search Diagnostic Event")
|
||||
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
|
||||
.withIcon(new QIcon("bug_report"))
|
||||
.withBackendName(config.getBackendName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s %s")
|
||||
.withRecordLabelFields("eventType", "source")
|
||||
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("eventType", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("source", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("outcome", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("userName", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tableNames", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("searchTerm", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("durationMillis", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("errorClass", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("errorMessage", QFieldType.TEXT).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("detail", QFieldType.TEXT).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("hostName", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withSection(new QFieldSection("identity", "Event", new QIcon("bug_report"), Tier.T1, List.of("id", "createDate", "eventType", "source", "outcome")))
|
||||
.withSection(new QFieldSection("request", "Request", new QIcon("search"), Tier.T2, List.of("userName", "tableNames", "searchTerm", "durationMillis", "hostName")))
|
||||
.withSection(new QFieldSection("error", "Error and detail", new QIcon("error_outline"), Tier.T2, List.of("errorClass", "errorMessage", "detail"))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** quickSearchValidationRun: index validation reports (history). The
|
||||
** report is an HTML field, shown in full on the record view; only the
|
||||
** QBit writes rows (insert, update and delete are disabled for users and
|
||||
** the API), so the HTML is always the QBit's escaped output.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData buildValidationRunTable(QuickSearchQBitConfig config)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(config.applyPrefix(QuickSearchValidationRun.TABLE_NAME))
|
||||
.withLabel("Quick Search Validation Run")
|
||||
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE)
|
||||
.withIcon(new QIcon("fact_check"))
|
||||
.withBackendName(config.getBackendName())
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s %s")
|
||||
.withRecordLabelFields("status", "startTime")
|
||||
.withPermissionRules(config.getTablePermissionRules() == null ? null : config.getTablePermissionRules().clone())
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("status", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("triggerType", QFieldType.STRING).withLabel("Trigger").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("userName", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tableNames", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("sampleSize", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("tablesChecked", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("failCount", QFieldType.INTEGER).withLabel("Failed Checks").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("warnCount", QFieldType.INTEGER).withLabel("Warnings").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("summary", QFieldType.STRING).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("reportHtml", QFieldType.HTML).withLabel("Report").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("reportJson", QFieldType.TEXT).withLabel("Report JSON").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
|
||||
.withSection(new QFieldSection("identity", "Run", new QIcon("fact_check"), Tier.T1, List.of("id", "status", "summary", "triggerType", "startTime", "endTime")))
|
||||
.withSection(new QFieldSection("scope", "Scope", new QIcon("table_chart"), Tier.T2, List.of("userName", "tableNames", "sampleSize", "tablesChecked", "failCount", "warnCount", "createDate")))
|
||||
.withSection(new QFieldSection("report", "Report", new QIcon("description"), Tier.T2, List.of("reportHtml")))
|
||||
.withSection(new QFieldSection("data", "Report data (JSON)", new QIcon("data_object"), Tier.T3, List.of("reportJson"))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Possible value source over quickSearchIndex, so run rows show the table
|
||||
** name instead of an id.
|
||||
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.metadata;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces the quickSearchValidationRun table when enableDiagnosticTools is true.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchValidationRunTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, QuickSearchQBitConfig>
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData produce(QInstance qInstance) throws QException
|
||||
{
|
||||
return (QuickSearchTableMetaDataHelper.buildValidationRunTable(getQBitConfig()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (getQBitConfig() != null && Boolean.TRUE.equals(getQBitConfig().getEnableDiagnosticTools()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** tables before processes and the app
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int getSortOrder()
|
||||
{
|
||||
return (100);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,508 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.model;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QField;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A Quick Search diagnostic event: a failed search (OpenSearch error or timeout,
|
||||
** after which core record search falls back to its own search), a diagnostic
|
||||
** tool run (the audit trail of the admin tools), or a refused run (rate limit).
|
||||
** Written by DiagnosticEventRecorder; read-only for operators.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticEvent extends QRecordEntity
|
||||
{
|
||||
public static final String TABLE_NAME = "quickSearchDiagnosticEvent";
|
||||
|
||||
public static final String TYPE_SEARCH_FAILED = "SEARCH_FAILED";
|
||||
public static final String TYPE_TOOL_RUN = "TOOL_RUN";
|
||||
public static final String TYPE_REINDEX = "REINDEX";
|
||||
public static final String TYPE_RATE_LIMITED = "RATE_LIMITED";
|
||||
public static final String SOURCE_CORE_RECORD_SEARCH = "CORE_RECORD_SEARCH";
|
||||
public static final String SOURCE_QUICK_SEARCH_ACTION = "QUICK_SEARCH_ACTION";
|
||||
public static final String OUTCOME_OK = "OK";
|
||||
public static final String OUTCOME_ERROR = "ERROR";
|
||||
public static final String OUTCOME_REFUSED = "REFUSED";
|
||||
|
||||
@QField()
|
||||
private Integer id;
|
||||
|
||||
@QField()
|
||||
private String eventType;
|
||||
|
||||
@QField()
|
||||
private String source;
|
||||
|
||||
@QField()
|
||||
private String outcome;
|
||||
|
||||
@QField()
|
||||
private String userName;
|
||||
|
||||
@QField()
|
||||
private String tableNames;
|
||||
|
||||
@QField()
|
||||
private String searchTerm;
|
||||
|
||||
@QField()
|
||||
private Integer durationMillis;
|
||||
|
||||
@QField()
|
||||
private String errorClass;
|
||||
|
||||
@QField()
|
||||
private String errorMessage;
|
||||
|
||||
@QField()
|
||||
private String detail;
|
||||
|
||||
@QField()
|
||||
private String hostName;
|
||||
|
||||
@QField()
|
||||
private Instant createDate;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Default constructor
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor from a QRecord
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent(QRecord record) throws QException
|
||||
{
|
||||
populateFromQRecord(record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for id
|
||||
*******************************************************************************/
|
||||
public Integer getId()
|
||||
{
|
||||
return (this.id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for id
|
||||
*******************************************************************************/
|
||||
public void setId(Integer id)
|
||||
{
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for id
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withId(Integer id)
|
||||
{
|
||||
this.id = id;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for eventType
|
||||
*******************************************************************************/
|
||||
public String getEventType()
|
||||
{
|
||||
return (this.eventType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for eventType
|
||||
*******************************************************************************/
|
||||
public void setEventType(String eventType)
|
||||
{
|
||||
this.eventType = eventType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for eventType
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withEventType(String eventType)
|
||||
{
|
||||
this.eventType = eventType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for source
|
||||
*******************************************************************************/
|
||||
public String getSource()
|
||||
{
|
||||
return (this.source);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for source
|
||||
*******************************************************************************/
|
||||
public void setSource(String source)
|
||||
{
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for source
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withSource(String source)
|
||||
{
|
||||
this.source = source;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for outcome
|
||||
*******************************************************************************/
|
||||
public String getOutcome()
|
||||
{
|
||||
return (this.outcome);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for outcome
|
||||
*******************************************************************************/
|
||||
public void setOutcome(String outcome)
|
||||
{
|
||||
this.outcome = outcome;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for outcome
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withOutcome(String outcome)
|
||||
{
|
||||
this.outcome = outcome;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for userName
|
||||
*******************************************************************************/
|
||||
public String getUserName()
|
||||
{
|
||||
return (this.userName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for userName
|
||||
*******************************************************************************/
|
||||
public void setUserName(String userName)
|
||||
{
|
||||
this.userName = userName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for userName
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withUserName(String userName)
|
||||
{
|
||||
this.userName = userName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tableNames
|
||||
*******************************************************************************/
|
||||
public String getTableNames()
|
||||
{
|
||||
return (this.tableNames);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for tableNames
|
||||
*******************************************************************************/
|
||||
public void setTableNames(String tableNames)
|
||||
{
|
||||
this.tableNames = tableNames;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tableNames
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withTableNames(String tableNames)
|
||||
{
|
||||
this.tableNames = tableNames;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for searchTerm
|
||||
*******************************************************************************/
|
||||
public String getSearchTerm()
|
||||
{
|
||||
return (this.searchTerm);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for searchTerm
|
||||
*******************************************************************************/
|
||||
public void setSearchTerm(String searchTerm)
|
||||
{
|
||||
this.searchTerm = searchTerm;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for searchTerm
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withSearchTerm(String searchTerm)
|
||||
{
|
||||
this.searchTerm = searchTerm;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for durationMillis
|
||||
*******************************************************************************/
|
||||
public Integer getDurationMillis()
|
||||
{
|
||||
return (this.durationMillis);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for durationMillis
|
||||
*******************************************************************************/
|
||||
public void setDurationMillis(Integer durationMillis)
|
||||
{
|
||||
this.durationMillis = durationMillis;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for durationMillis
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withDurationMillis(Integer durationMillis)
|
||||
{
|
||||
this.durationMillis = durationMillis;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for errorClass
|
||||
*******************************************************************************/
|
||||
public String getErrorClass()
|
||||
{
|
||||
return (this.errorClass);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for errorClass
|
||||
*******************************************************************************/
|
||||
public void setErrorClass(String errorClass)
|
||||
{
|
||||
this.errorClass = errorClass;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for errorClass
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withErrorClass(String errorClass)
|
||||
{
|
||||
this.errorClass = errorClass;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for errorMessage
|
||||
*******************************************************************************/
|
||||
public String getErrorMessage()
|
||||
{
|
||||
return (this.errorMessage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for errorMessage
|
||||
*******************************************************************************/
|
||||
public void setErrorMessage(String errorMessage)
|
||||
{
|
||||
this.errorMessage = errorMessage;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for errorMessage
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withErrorMessage(String errorMessage)
|
||||
{
|
||||
this.errorMessage = errorMessage;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for detail
|
||||
*******************************************************************************/
|
||||
public String getDetail()
|
||||
{
|
||||
return (this.detail);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for detail
|
||||
*******************************************************************************/
|
||||
public void setDetail(String detail)
|
||||
{
|
||||
this.detail = detail;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for detail
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withDetail(String detail)
|
||||
{
|
||||
this.detail = detail;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for hostName
|
||||
*******************************************************************************/
|
||||
public String getHostName()
|
||||
{
|
||||
return (this.hostName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for hostName
|
||||
*******************************************************************************/
|
||||
public void setHostName(String hostName)
|
||||
{
|
||||
this.hostName = hostName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for hostName
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withHostName(String hostName)
|
||||
{
|
||||
this.hostName = hostName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for createDate
|
||||
*******************************************************************************/
|
||||
public Instant getCreateDate()
|
||||
{
|
||||
return (this.createDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for createDate
|
||||
*******************************************************************************/
|
||||
public void setCreateDate(Instant createDate)
|
||||
{
|
||||
this.createDate = createDate;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for createDate
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticEvent withCreateDate(Instant createDate)
|
||||
{
|
||||
this.createDate = createDate;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,572 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.model;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QField;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One run of the index validation: overall status, counts of failed and
|
||||
** warning checks, and the full report (HTML for the record view, JSON for
|
||||
** tools). Kept as history; purged after runHistoryRetentionDays.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchValidationRun extends QRecordEntity
|
||||
{
|
||||
public static final String TABLE_NAME = "quickSearchValidationRun";
|
||||
|
||||
public static final String STATUS_PASS = "PASS";
|
||||
public static final String STATUS_WARN = "WARN";
|
||||
public static final String STATUS_FAIL = "FAIL";
|
||||
public static final String STATUS_ERROR = "ERROR";
|
||||
public static final String TRIGGER_MANUAL = "MANUAL";
|
||||
public static final String TRIGGER_SCHEDULED = "SCHEDULED";
|
||||
|
||||
@QField()
|
||||
private Integer id;
|
||||
|
||||
@QField()
|
||||
private String status;
|
||||
|
||||
@QField()
|
||||
private String triggerType;
|
||||
|
||||
@QField()
|
||||
private String userName;
|
||||
|
||||
@QField()
|
||||
private String tableNames;
|
||||
|
||||
@QField()
|
||||
private Integer sampleSize;
|
||||
|
||||
@QField()
|
||||
private Integer tablesChecked;
|
||||
|
||||
@QField()
|
||||
private Integer failCount;
|
||||
|
||||
@QField()
|
||||
private Integer warnCount;
|
||||
|
||||
@QField()
|
||||
private String summary;
|
||||
|
||||
@QField()
|
||||
private String reportHtml;
|
||||
|
||||
@QField()
|
||||
private String reportJson;
|
||||
|
||||
@QField()
|
||||
private Instant startTime;
|
||||
|
||||
@QField()
|
||||
private Instant endTime;
|
||||
|
||||
@QField()
|
||||
private Instant createDate;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Default constructor
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor from a QRecord
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun(QRecord record) throws QException
|
||||
{
|
||||
populateFromQRecord(record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for id
|
||||
*******************************************************************************/
|
||||
public Integer getId()
|
||||
{
|
||||
return (this.id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for id
|
||||
*******************************************************************************/
|
||||
public void setId(Integer id)
|
||||
{
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for id
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withId(Integer id)
|
||||
{
|
||||
this.id = id;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for status
|
||||
*******************************************************************************/
|
||||
public String getStatus()
|
||||
{
|
||||
return (this.status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for status
|
||||
*******************************************************************************/
|
||||
public void setStatus(String status)
|
||||
{
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for status
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withStatus(String status)
|
||||
{
|
||||
this.status = status;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for triggerType
|
||||
*******************************************************************************/
|
||||
public String getTriggerType()
|
||||
{
|
||||
return (this.triggerType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for triggerType
|
||||
*******************************************************************************/
|
||||
public void setTriggerType(String triggerType)
|
||||
{
|
||||
this.triggerType = triggerType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for triggerType
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withTriggerType(String triggerType)
|
||||
{
|
||||
this.triggerType = triggerType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for userName
|
||||
*******************************************************************************/
|
||||
public String getUserName()
|
||||
{
|
||||
return (this.userName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for userName
|
||||
*******************************************************************************/
|
||||
public void setUserName(String userName)
|
||||
{
|
||||
this.userName = userName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for userName
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withUserName(String userName)
|
||||
{
|
||||
this.userName = userName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tableNames
|
||||
*******************************************************************************/
|
||||
public String getTableNames()
|
||||
{
|
||||
return (this.tableNames);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for tableNames
|
||||
*******************************************************************************/
|
||||
public void setTableNames(String tableNames)
|
||||
{
|
||||
this.tableNames = tableNames;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tableNames
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withTableNames(String tableNames)
|
||||
{
|
||||
this.tableNames = tableNames;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for sampleSize
|
||||
*******************************************************************************/
|
||||
public Integer getSampleSize()
|
||||
{
|
||||
return (this.sampleSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for sampleSize
|
||||
*******************************************************************************/
|
||||
public void setSampleSize(Integer sampleSize)
|
||||
{
|
||||
this.sampleSize = sampleSize;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for sampleSize
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withSampleSize(Integer sampleSize)
|
||||
{
|
||||
this.sampleSize = sampleSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tablesChecked
|
||||
*******************************************************************************/
|
||||
public Integer getTablesChecked()
|
||||
{
|
||||
return (this.tablesChecked);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for tablesChecked
|
||||
*******************************************************************************/
|
||||
public void setTablesChecked(Integer tablesChecked)
|
||||
{
|
||||
this.tablesChecked = tablesChecked;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tablesChecked
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withTablesChecked(Integer tablesChecked)
|
||||
{
|
||||
this.tablesChecked = tablesChecked;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for failCount
|
||||
*******************************************************************************/
|
||||
public Integer getFailCount()
|
||||
{
|
||||
return (this.failCount);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for failCount
|
||||
*******************************************************************************/
|
||||
public void setFailCount(Integer failCount)
|
||||
{
|
||||
this.failCount = failCount;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for failCount
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withFailCount(Integer failCount)
|
||||
{
|
||||
this.failCount = failCount;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for warnCount
|
||||
*******************************************************************************/
|
||||
public Integer getWarnCount()
|
||||
{
|
||||
return (this.warnCount);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for warnCount
|
||||
*******************************************************************************/
|
||||
public void setWarnCount(Integer warnCount)
|
||||
{
|
||||
this.warnCount = warnCount;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for warnCount
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withWarnCount(Integer warnCount)
|
||||
{
|
||||
this.warnCount = warnCount;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for summary
|
||||
*******************************************************************************/
|
||||
public String getSummary()
|
||||
{
|
||||
return (this.summary);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for summary
|
||||
*******************************************************************************/
|
||||
public void setSummary(String summary)
|
||||
{
|
||||
this.summary = summary;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for summary
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withSummary(String summary)
|
||||
{
|
||||
this.summary = summary;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for reportHtml
|
||||
*******************************************************************************/
|
||||
public String getReportHtml()
|
||||
{
|
||||
return (this.reportHtml);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for reportHtml
|
||||
*******************************************************************************/
|
||||
public void setReportHtml(String reportHtml)
|
||||
{
|
||||
this.reportHtml = reportHtml;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for reportHtml
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withReportHtml(String reportHtml)
|
||||
{
|
||||
this.reportHtml = reportHtml;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for reportJson
|
||||
*******************************************************************************/
|
||||
public String getReportJson()
|
||||
{
|
||||
return (this.reportJson);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for reportJson
|
||||
*******************************************************************************/
|
||||
public void setReportJson(String reportJson)
|
||||
{
|
||||
this.reportJson = reportJson;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for reportJson
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withReportJson(String reportJson)
|
||||
{
|
||||
this.reportJson = reportJson;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for startTime
|
||||
*******************************************************************************/
|
||||
public Instant getStartTime()
|
||||
{
|
||||
return (this.startTime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for startTime
|
||||
*******************************************************************************/
|
||||
public void setStartTime(Instant startTime)
|
||||
{
|
||||
this.startTime = startTime;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for startTime
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withStartTime(Instant startTime)
|
||||
{
|
||||
this.startTime = startTime;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for endTime
|
||||
*******************************************************************************/
|
||||
public Instant getEndTime()
|
||||
{
|
||||
return (this.endTime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for endTime
|
||||
*******************************************************************************/
|
||||
public void setEndTime(Instant endTime)
|
||||
{
|
||||
this.endTime = endTime;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for endTime
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withEndTime(Instant endTime)
|
||||
{
|
||||
this.endTime = endTime;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for createDate
|
||||
*******************************************************************************/
|
||||
public Instant getCreateDate()
|
||||
{
|
||||
return (this.createDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for createDate
|
||||
*******************************************************************************/
|
||||
public void setCreateDate(Instant createDate)
|
||||
{
|
||||
this.createDate = createDate;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for createDate
|
||||
*******************************************************************************/
|
||||
public QuickSearchValidationRun withCreateDate(Instant createDate)
|
||||
{
|
||||
this.createDate = createDate;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
-566
@@ -1,566 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.opensearch;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import jakarta.json.stream.JsonGenerator;
|
||||
import org.opensearch.client.json.JsonpMapper;
|
||||
import org.opensearch.client.json.JsonpSerializable;
|
||||
import org.opensearch.client.opensearch.OpenSearchClient;
|
||||
import org.opensearch.client.opensearch._types.OpenSearchException;
|
||||
import org.opensearch.client.opensearch._types.query_dsl.Query;
|
||||
import org.opensearch.client.opensearch.core.ExplainResponse;
|
||||
import org.opensearch.client.opensearch.core.SearchRequest;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.opensearch.client.opensearch.core.search.Hit;
|
||||
import org.opensearch.client.opensearch.core.search.HighlightField;
|
||||
import org.opensearch.client.opensearch.core.search.MatchedQueries;
|
||||
import org.opensearch.client.opensearch.core.search.TotalHitsRelation;
|
||||
import org.opensearch.client.opensearch.generic.Body;
|
||||
import org.opensearch.client.opensearch.generic.Requests;
|
||||
import org.opensearch.client.opensearch.generic.Response;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only OpenSearch operations behind the Quick Search admin diagnostics:
|
||||
** a probe search that reports scores, highlights and matched clauses with
|
||||
** its request and response JSON, explain for one document, document reads,
|
||||
** _analyze, random document sampling, and raw cluster / index / mapping /
|
||||
** settings reads. Nothing here writes to the cluster.
|
||||
** The probe uses exactly the query production search builds (same clauses,
|
||||
** operator, boosts and table filter); it only adds query names and
|
||||
** highlighting on the field values, which change neither matching nor
|
||||
** scoring.
|
||||
*******************************************************************************/
|
||||
public class QuickSearchDiagnosticsOperations
|
||||
{
|
||||
public static final int MAX_ANALYZE_TEXT_LENGTH = 10_000;
|
||||
public static final int MAX_PROBE_SIZE = 100;
|
||||
public static final int MAX_SAMPLE_SIZE = 1_000;
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private final QuickSearchOpenSearchClient quickSearchClient;
|
||||
private final OpenSearchClient client;
|
||||
private final String indexName;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One hit of a probe search.
|
||||
*******************************************************************************/
|
||||
public record ProbeHit(String recordId, String recordLabel, Double score, List<String> matchedClauses, Map<String, List<String>> highlights)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A probe search of one table: request and response JSON, server and
|
||||
** wall-clock time, total hits and the hits.
|
||||
*******************************************************************************/
|
||||
public record SearchProbe(String tableName, String requestJson, String responseJson, Long tookMillis, Long wallMillis, Long totalHits, Boolean totalIsLowerBound, List<ProbeHit> hits)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The result of explain for one document and the search query.
|
||||
*******************************************************************************/
|
||||
public record ExplainResult(Boolean documentFound, Boolean matched, Double score, String explanationText, String json)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One indexed document as stored: whether it exists, its version, the
|
||||
** physical index holding it and its source.
|
||||
*******************************************************************************/
|
||||
public record DocumentView(Boolean found, Long version, String physicalIndex, JsonNode source)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One token produced by _analyze.
|
||||
*******************************************************************************/
|
||||
public record AnalyzedToken(String token, Integer position, Integer startOffset, Integer endOffset, String type)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A raw JSON response: HTTP status and parsed body (null when empty).
|
||||
*******************************************************************************/
|
||||
public record RawResponse(Integer status, JsonNode body)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether the status is 2xx.
|
||||
*******************************************************************************/
|
||||
public boolean isOk()
|
||||
{
|
||||
return (status != null && status >= 200 && status < 300);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
*******************************************************************************/
|
||||
public QuickSearchDiagnosticsOperations(QuickSearchOpenSearchClient quickSearchClient)
|
||||
{
|
||||
this.quickSearchClient = quickSearchClient;
|
||||
this.client = quickSearchClient.getOpenSearchClient();
|
||||
this.indexName = quickSearchClient.getIndexName();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the alias name the operations read.
|
||||
*******************************************************************************/
|
||||
public String getIndexName()
|
||||
{
|
||||
return (indexName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run the production search query against one table with query names and
|
||||
** field highlighting, returning up to size hits (capped at MAX_PROBE_SIZE).
|
||||
*******************************************************************************/
|
||||
public SearchProbe probeSearch(String term, String tableName, int size, List<QuickSearchableTableConfig> tableConfigs) throws QException
|
||||
{
|
||||
int boundedSize = Math.max(1, Math.min(size, MAX_PROBE_SIZE));
|
||||
Query query = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, true);
|
||||
SearchRequest request = SearchRequest.of(r -> r
|
||||
.index(indexName)
|
||||
.query(query)
|
||||
.highlight(h -> h
|
||||
.requireFieldMatch(true)
|
||||
.fields("searchableText", HighlightField.of(hf -> hf))
|
||||
.fields("fieldValues.*", HighlightField.of(hf -> hf)))
|
||||
.trackTotalHits(t -> t.count(QuickSearchOpenSearchClient.MAX_RESULT_WINDOW))
|
||||
.size(boundedSize));
|
||||
|
||||
long start = System.nanoTime();
|
||||
try
|
||||
{
|
||||
SearchResponse<OpenSearchDocument> response = client.search(request, OpenSearchDocument.class);
|
||||
long wall = (System.nanoTime() - start) / 1_000_000;
|
||||
|
||||
List<ProbeHit> hits = new ArrayList<>();
|
||||
for(Hit<OpenSearchDocument> hit : response.hits().hits())
|
||||
{
|
||||
OpenSearchDocument source = hit.source();
|
||||
hits.add(new ProbeHit(
|
||||
source == null ? idFromDocumentId(hit.id()) : source.getRecordId(),
|
||||
source == null ? null : source.getRecordLabel(),
|
||||
hit.score(),
|
||||
matchedClauses(hit.matchedQueries()),
|
||||
hit.highlight() == null ? Map.of() : new LinkedHashMap<>(hit.highlight())));
|
||||
}
|
||||
|
||||
Long total = response.hits().total() == null ? (long) hits.size() : response.hits().total().value();
|
||||
Boolean lowerBound = response.hits().total() != null && response.hits().total().relation() == TotalHitsRelation.Gte;
|
||||
return (new SearchProbe(tableName, toJson(request), toJson(response), response.took(), wall, total, lowerBound, hits));
|
||||
}
|
||||
catch(OpenSearchException | IOException e)
|
||||
{
|
||||
throw (wrap("Probe search failed for table [" + tableName + "]", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Names of the clauses a hit matched (query names or, in newer servers,
|
||||
** the keys of the named-query scores).
|
||||
*******************************************************************************/
|
||||
static List<String> matchedClauses(MatchedQueries matchedQueries)
|
||||
{
|
||||
if(matchedQueries == null)
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
if(matchedQueries.isScores())
|
||||
{
|
||||
return (new ArrayList<>(matchedQueries.scores().keySet()));
|
||||
}
|
||||
return (new ArrayList<>(matchedQueries.names()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Explain how the production query scores one document of a table. A
|
||||
** missing document gives documentFound false (not an error).
|
||||
*******************************************************************************/
|
||||
public ExplainResult explain(String term, String tableName, String recordId, List<QuickSearchableTableConfig> tableConfigs) throws QException
|
||||
{
|
||||
Query query = quickSearchClient.buildSearchQuery(term, List.of(tableName), tableConfigs, false);
|
||||
String documentId = OpenSearchDocument.buildDocumentId(tableName, recordId);
|
||||
try
|
||||
{
|
||||
ExplainResponse<OpenSearchDocument> response = client.explain(e -> e.index(indexName).id(documentId).query(query), OpenSearchDocument.class);
|
||||
String json = toJson(response);
|
||||
JsonNode tree = MAPPER.readTree(json);
|
||||
JsonNode explanation = tree.path("explanation");
|
||||
Double score = explanation.isMissingNode() ? null : explanation.path("value").asDouble();
|
||||
return (new ExplainResult(true, response.matched(), score, formatExplanation(explanation), json));
|
||||
}
|
||||
catch(OpenSearchException e)
|
||||
{
|
||||
if(e.status() == 404)
|
||||
{
|
||||
return (new ExplainResult(false, false, null, null, null));
|
||||
}
|
||||
throw (wrap("Explain failed for document [" + documentId + "]", e));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (wrap("Explain failed for document [" + documentId + "]", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An explanation tree as indented text ("value description"), bounded to
|
||||
** 400 lines.
|
||||
*******************************************************************************/
|
||||
public static String formatExplanation(JsonNode explanation)
|
||||
{
|
||||
if(explanation == null || explanation.isMissingNode() || explanation.isNull())
|
||||
{
|
||||
return ("");
|
||||
}
|
||||
List<String> lines = new ArrayList<>();
|
||||
appendExplanation(explanation, 0, lines);
|
||||
return (String.join("\n", lines));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static void appendExplanation(JsonNode node, int depth, List<String> lines)
|
||||
{
|
||||
if(lines.size() >= 400)
|
||||
{
|
||||
return;
|
||||
}
|
||||
lines.add(" ".repeat(depth) + String.format("%.4f", node.path("value").asDouble()) + " " + node.path("description").asText(""));
|
||||
for(JsonNode detail : node.path("details"))
|
||||
{
|
||||
appendExplanation(detail, depth + 1, lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read one document of a table by record id.
|
||||
*******************************************************************************/
|
||||
public DocumentView getDocument(String tableName, String recordId) throws QException
|
||||
{
|
||||
RawResponse response = raw("GET", "/" + encode(indexName) + "/_doc/" + encode(OpenSearchDocument.buildDocumentId(tableName, recordId)), null);
|
||||
if(response.status() == 404)
|
||||
{
|
||||
return (new DocumentView(false, null, null, null));
|
||||
}
|
||||
requireOk(response, "Document read failed");
|
||||
JsonNode body = response.body();
|
||||
return (new DocumentView(body.path("found").asBoolean(false), body.has("_version") ? body.get("_version").asLong() : null,
|
||||
body.path("_index").asText(null), body.get("_source")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read many documents of a table by record id: record id to source, for
|
||||
** the documents that exist.
|
||||
*******************************************************************************/
|
||||
public Map<String, JsonNode> getDocuments(String tableName, Collection<String> recordIds) throws QException
|
||||
{
|
||||
Map<String, JsonNode> sources = new LinkedHashMap<>();
|
||||
if(recordIds == null || recordIds.isEmpty())
|
||||
{
|
||||
return (sources);
|
||||
}
|
||||
|
||||
ObjectNode body = JsonNodeFactory.instance.objectNode();
|
||||
ArrayNode ids = body.putArray("ids");
|
||||
for(String recordId : recordIds)
|
||||
{
|
||||
ids.add(OpenSearchDocument.buildDocumentId(tableName, recordId));
|
||||
}
|
||||
|
||||
RawResponse response = raw("POST", "/" + encode(indexName) + "/_mget", body.toString());
|
||||
requireOk(response, "Document read failed");
|
||||
for(JsonNode doc : response.body().path("docs"))
|
||||
{
|
||||
if(doc.path("found").asBoolean(false) && doc.has("_source"))
|
||||
{
|
||||
sources.put(idFromDocumentId(doc.path("_id").asText()), doc.get("_source"));
|
||||
}
|
||||
}
|
||||
return (sources);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run _analyze on the index with a named analyzer (an index analyzer or a
|
||||
** built-in one). Text is truncated to MAX_ANALYZE_TEXT_LENGTH.
|
||||
*******************************************************************************/
|
||||
public List<AnalyzedToken> analyze(String analyzer, String text) throws QException
|
||||
{
|
||||
String bounded = text == null ? "" : (text.length() > MAX_ANALYZE_TEXT_LENGTH ? text.substring(0, MAX_ANALYZE_TEXT_LENGTH) : text);
|
||||
ObjectNode body = JsonNodeFactory.instance.objectNode().put("analyzer", analyzer).put("text", bounded);
|
||||
|
||||
RawResponse response = raw("POST", "/" + encode(indexName) + "/_analyze", body.toString());
|
||||
requireOk(response, "Analyze failed for analyzer [" + analyzer + "]");
|
||||
|
||||
List<AnalyzedToken> tokens = new ArrayList<>();
|
||||
for(JsonNode token : response.body().path("tokens"))
|
||||
{
|
||||
tokens.add(new AnalyzedToken(token.path("token").asText(), token.path("position").asInt(), token.path("start_offset").asInt(),
|
||||
token.path("end_offset").asInt(), token.path("type").asText(null)));
|
||||
}
|
||||
return (tokens);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Up to n record ids of a table's documents, chosen at random.
|
||||
*******************************************************************************/
|
||||
public List<String> sampleRecordIds(String tableName, int n) throws QException
|
||||
{
|
||||
int size = Math.max(1, Math.min(n, MAX_SAMPLE_SIZE));
|
||||
ObjectNode body = JsonNodeFactory.instance.objectNode();
|
||||
body.put("size", size);
|
||||
body.putArray("_source").add("recordId");
|
||||
ObjectNode functionScore = body.putObject("query").putObject("function_score");
|
||||
functionScore.putObject("query").putObject("term").put("sourceTable", tableName);
|
||||
functionScore.putObject("random_score");
|
||||
|
||||
RawResponse response = raw("POST", "/" + encode(indexName) + "/_search", body.toString());
|
||||
requireOk(response, "Sampling documents failed for table [" + tableName + "]");
|
||||
|
||||
List<String> recordIds = new ArrayList<>();
|
||||
for(JsonNode hit : response.body().path("hits").path("hits"))
|
||||
{
|
||||
String recordId = hit.path("_source").path("recordId").asText(null);
|
||||
recordIds.add(recordId != null ? recordId : idFromDocumentId(hit.path("_id").asText()));
|
||||
}
|
||||
return (recordIds);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cluster health, with the alias's indexes (level=indices).
|
||||
*******************************************************************************/
|
||||
public RawResponse clusterHealth() throws QException
|
||||
{
|
||||
return (raw("GET", "/_cluster/health/" + encode(indexName) + "?level=indices", null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live mapping of the alias's index(es).
|
||||
*******************************************************************************/
|
||||
public RawResponse mapping() throws QException
|
||||
{
|
||||
return (raw("GET", "/" + encode(indexName) + "/_mapping", null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live settings of the alias's index(es).
|
||||
*******************************************************************************/
|
||||
public RawResponse settings() throws QException
|
||||
{
|
||||
return (raw("GET", "/" + encode(indexName) + "/_settings", null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The physical indexes named like this alias's ({alias}-v*), with health,
|
||||
** status, document counts and size, from _cat/indices.
|
||||
*******************************************************************************/
|
||||
public RawResponse physicalIndexes() throws QException
|
||||
{
|
||||
return (raw("GET", "/_cat/indices/" + encode(indexName) + "-v*?format=json&h=index,health,status,docs.count,docs.deleted,store.size&expand_wildcards=all", null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record id part of a document id ("table:id").
|
||||
*******************************************************************************/
|
||||
static String idFromDocumentId(String documentId)
|
||||
{
|
||||
if(documentId == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
int colon = documentId.indexOf(':');
|
||||
return (colon < 0 ? documentId : documentId.substring(colon + 1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Send a raw request and parse the JSON body. HTTP errors are returned
|
||||
** (status and body), not thrown; transport errors throw.
|
||||
*******************************************************************************/
|
||||
public RawResponse raw(String method, String endpoint, String jsonBody) throws QException
|
||||
{
|
||||
Requests.JsonBodyBuilder builder = Requests.builder().method(method).endpoint(endpoint);
|
||||
if(jsonBody != null)
|
||||
{
|
||||
builder.json(jsonBody);
|
||||
}
|
||||
|
||||
try(Response response = client.generic().execute(builder.build()))
|
||||
{
|
||||
JsonNode body = null;
|
||||
if(response.getBody().isPresent())
|
||||
{
|
||||
try(Body responseBody = response.getBody().get())
|
||||
{
|
||||
String text = responseBody.bodyAsString();
|
||||
body = text == null || text.isBlank() ? null : MAPPER.readTree(text);
|
||||
}
|
||||
}
|
||||
return (new RawResponse(response.getStatus(), body));
|
||||
}
|
||||
catch(OpenSearchException | IOException e)
|
||||
{
|
||||
throw (wrap(method + " " + endpoint.replaceAll("\\?.*", "") + " failed", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Throw a QException with OpenSearch's reason when a raw response is not
|
||||
** 2xx.
|
||||
*******************************************************************************/
|
||||
public static void requireOk(RawResponse response, String message) throws QException
|
||||
{
|
||||
if(!response.isOk())
|
||||
{
|
||||
throw (new QException(message + ": OpenSearch returned status " + response.status() + ": " + errorReason(response.body())));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** OpenSearch's error reason from an error body, or a short form of it.
|
||||
*******************************************************************************/
|
||||
public static String errorReason(JsonNode body)
|
||||
{
|
||||
if(body == null)
|
||||
{
|
||||
return ("no response body");
|
||||
}
|
||||
JsonNode error = body.path("error");
|
||||
if(error.isTextual())
|
||||
{
|
||||
return (error.asText());
|
||||
}
|
||||
String reason = error.path("reason").asText(null);
|
||||
if(reason != null)
|
||||
{
|
||||
return (reason);
|
||||
}
|
||||
String text = body.toString();
|
||||
return (text.length() > 300 ? text.substring(0, 300) + "..." : text);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize a request or response with the transport's mapper (which knows
|
||||
** OpenSearchDocument; the default mapper of toJsonString does not).
|
||||
*******************************************************************************/
|
||||
private String toJson(JsonpSerializable value)
|
||||
{
|
||||
JsonpMapper mapper = client._transport().jsonpMapper();
|
||||
StringWriter writer = new StringWriter();
|
||||
try(JsonGenerator generator = mapper.jsonProvider().createGenerator(writer))
|
||||
{
|
||||
value.serialize(generator, mapper);
|
||||
}
|
||||
return (writer.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** URL-encode one path segment.
|
||||
*******************************************************************************/
|
||||
static String encode(String segment)
|
||||
{
|
||||
return (URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QException wrap(String message, Exception e)
|
||||
{
|
||||
if(e instanceof OpenSearchException openSearchException)
|
||||
{
|
||||
String reason = openSearchException.error() == null ? openSearchException.getMessage() : openSearchException.error().reason();
|
||||
return (new QException(message + ": OpenSearch returned status " + openSearchException.status() + ": " + reason, e));
|
||||
}
|
||||
return (new QException(message + ": " + e.getMessage(), e));
|
||||
}
|
||||
|
||||
}
|
||||
+20
-170
@@ -21,14 +21,9 @@ import java.io.Closeable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
@@ -37,7 +32,6 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import org.opensearch.client.json.JsonData;
|
||||
import org.opensearch.client.opensearch.OpenSearchAsyncClient;
|
||||
import org.opensearch.client.opensearch.OpenSearchClient;
|
||||
import org.opensearch.client.opensearch._types.Conflicts;
|
||||
import org.opensearch.client.opensearch._types.FieldValue;
|
||||
@@ -57,17 +51,12 @@ import org.opensearch.client.opensearch.core.DeleteByQueryRequest;
|
||||
import org.opensearch.client.opensearch.core.DeleteByQueryResponse;
|
||||
import org.opensearch.client.opensearch.core.DeleteRequest;
|
||||
import org.opensearch.client.opensearch.core.IndexRequest;
|
||||
import org.opensearch.client.opensearch.core.MsearchRequest;
|
||||
import org.opensearch.client.opensearch.core.MsearchResponse;
|
||||
import org.opensearch.client.opensearch.core.SearchRequest;
|
||||
import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.opensearch.client.opensearch.core.bulk.BulkOperation;
|
||||
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
|
||||
import org.opensearch.client.opensearch.core.bulk.DeleteOperation;
|
||||
import org.opensearch.client.opensearch.core.bulk.IndexOperation;
|
||||
import org.opensearch.client.opensearch.core.msearch.MultiSearchResponseItem;
|
||||
import org.opensearch.client.opensearch.core.msearch.RequestItem;
|
||||
import org.opensearch.client.opensearch.core.search.Hit;
|
||||
import org.opensearch.client.opensearch.core.search.Highlight;
|
||||
import org.opensearch.client.opensearch.core.search.HighlightField;
|
||||
import org.opensearch.client.opensearch.indices.CreateIndexRequest;
|
||||
@@ -102,15 +91,12 @@ public class QuickSearchOpenSearchClient implements Closeable
|
||||
public static final String INDEX_ANALYZER = "quick_search_analyzer";
|
||||
public static final String SEARCH_ANALYZER = "quick_search_search_analyzer";
|
||||
public static final int MAX_RESULT_WINDOW = 10_000;
|
||||
public static final String MATCH_CLAUSE_NAME = "searchableText";
|
||||
public static final String FIELD_CLAUSE_PREFIX = "field:";
|
||||
|
||||
private final OpenSearchClient client;
|
||||
private final OpenSearchAsyncClient asyncClient;
|
||||
private final OpenSearchTransport transport;
|
||||
private final String indexName;
|
||||
private final int maxBulkRequestBytes;
|
||||
private final ObjectMapper sizingMapper;
|
||||
private final OpenSearchClient client;
|
||||
private final OpenSearchTransport transport;
|
||||
private final String indexName;
|
||||
private final int maxBulkRequestBytes;
|
||||
private final ObjectMapper sizingMapper;
|
||||
|
||||
private volatile Boolean mappingOutdated = false;
|
||||
private Boolean closed = false;
|
||||
@@ -124,7 +110,6 @@ public class QuickSearchOpenSearchClient implements Closeable
|
||||
{
|
||||
this.transport = OpenSearchTransportFactory.build(config);
|
||||
this.client = new OpenSearchClient(transport);
|
||||
this.asyncClient = new OpenSearchAsyncClient(transport);
|
||||
this.indexName = config.getOpensearchIndexName();
|
||||
this.maxBulkRequestBytes = config.getMaxBulkRequestBytes() == null ? 5 * 1024 * 1024 : config.getMaxBulkRequestBytes();
|
||||
this.sizingMapper = new ObjectMapper().registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
|
||||
@@ -667,7 +652,14 @@ public class QuickSearchOpenSearchClient implements Closeable
|
||||
{
|
||||
try
|
||||
{
|
||||
Query finalQuery = buildSearchQuery(searchTerm, allowedTables, tableConfigs);
|
||||
List<FieldValue> allowed = allowedTables.stream().map(FieldValue::of).toList();
|
||||
|
||||
BoolQuery.Builder boolBuilder = new BoolQuery.Builder()
|
||||
.must(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm -> mm.query(searchTerm).fields("searchableText").operator(Operator.And)))))
|
||||
.filter(Query.of(q -> q.terms(t -> t.field("sourceTable").terms(tv -> tv.value(allowed)))))
|
||||
.should(buildFieldBoostQueries(searchTerm, tableConfigs, allowedTables));
|
||||
|
||||
Query finalQuery = Query.of(q -> q.bool(boolBuilder.build()));
|
||||
Highlight highlight = Highlight.of(h -> h.fields("searchableText", HighlightField.of(hf -> hf)));
|
||||
|
||||
SearchRequest request = SearchRequest.of(r -> r
|
||||
@@ -689,138 +681,10 @@ public class QuickSearchOpenSearchClient implements Closeable
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record ids of the best matches in each table, best first, for core record
|
||||
** search. One _msearch request with one search per table (the same query and
|
||||
** ranking as {@link #search}, restricted to that table), so every table gets
|
||||
** its own top idsPerTable hits. Only sourceTable and recordId are fetched.
|
||||
**
|
||||
** The whole request is bounded by timeoutMillis of wall-clock time (and the
|
||||
** same server-side search timeout): when it runs out, the request is
|
||||
** cancelled and a QException is thrown. A failed search for any table fails
|
||||
** the call. Returns every requested table as a key, in the given order.
|
||||
*******************************************************************************/
|
||||
public Map<String, List<String>> searchRecordIdsPerTable(String searchTerm, List<String> tableNames, int idsPerTable, List<QuickSearchableTableConfig> tableConfigs, int timeoutMillis) throws QException
|
||||
{
|
||||
Map<String, List<String>> idsByTable = new LinkedHashMap<>();
|
||||
if(tableNames == null || tableNames.isEmpty())
|
||||
{
|
||||
return (idsByTable);
|
||||
}
|
||||
|
||||
List<RequestItem> searches = new ArrayList<>();
|
||||
for(String tableName : tableNames)
|
||||
{
|
||||
Query query = buildSearchQuery(searchTerm, List.of(tableName), tableConfigs);
|
||||
searches.add(RequestItem.of(ri -> ri
|
||||
.header(h -> h.index(indexName))
|
||||
.body(b -> b
|
||||
.query(query)
|
||||
.size(idsPerTable)
|
||||
.trackTotalHits(t -> t.enabled(false))
|
||||
.source(sc -> sc.filter(f -> f.includes("sourceTable", "recordId")))
|
||||
.timeout(timeoutMillis + "ms"))));
|
||||
}
|
||||
|
||||
CompletableFuture<MsearchResponse<OpenSearchDocument>> future = null;
|
||||
try
|
||||
{
|
||||
future = asyncClient.msearch(MsearchRequest.of(r -> r.searches(searches)), OpenSearchDocument.class);
|
||||
MsearchResponse<OpenSearchDocument> response = future.get(timeoutMillis, TimeUnit.MILLISECONDS);
|
||||
|
||||
List<MultiSearchResponseItem<OpenSearchDocument>> items = response.responses();
|
||||
if(items == null || items.size() != tableNames.size())
|
||||
{
|
||||
throw (new QException("Record search failed: OpenSearch returned " + (items == null ? 0 : items.size()) + " responses for " + tableNames.size() + " searches"));
|
||||
}
|
||||
|
||||
for(int i = 0; i < tableNames.size(); i++)
|
||||
{
|
||||
MultiSearchResponseItem<OpenSearchDocument> item = items.get(i);
|
||||
if(item.isFailure())
|
||||
{
|
||||
String reason = item.failure() != null && item.failure().error() != null ? item.failure().error().reason() : "unknown error";
|
||||
throw (new QException("Record search failed for table [" + tableNames.get(i) + "]: " + reason));
|
||||
}
|
||||
|
||||
List<String> ids = new ArrayList<>();
|
||||
for(Hit<OpenSearchDocument> hit : item.result().hits().hits())
|
||||
{
|
||||
if(hit.source() != null && hit.source().getRecordId() != null)
|
||||
{
|
||||
ids.add(hit.source().getRecordId());
|
||||
}
|
||||
}
|
||||
idsByTable.put(tableNames.get(i), ids);
|
||||
}
|
||||
return (idsByTable);
|
||||
}
|
||||
catch(TimeoutException e)
|
||||
{
|
||||
future.cancel(true);
|
||||
throw (new QException("Record search timed out after " + timeoutMillis + " ms", e));
|
||||
}
|
||||
catch(InterruptedException e)
|
||||
{
|
||||
if(future != null)
|
||||
{
|
||||
future.cancel(true);
|
||||
}
|
||||
Thread.currentThread().interrupt();
|
||||
throw (new QException("Record search was interrupted", e));
|
||||
}
|
||||
catch(ExecutionException e)
|
||||
{
|
||||
throw wrap("Record search failed", e.getCause() instanceof Exception cause ? cause : e);
|
||||
}
|
||||
catch(OpenSearchException | java.io.IOException e)
|
||||
{
|
||||
throw wrap("Record search failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The search query: every term of the query (operator AND) must match
|
||||
** searchableText, only the allowed tables match (an empty set matches
|
||||
** nothing), and per-field should clauses boost by configured weight.
|
||||
*******************************************************************************/
|
||||
private Query buildSearchQuery(String searchTerm, Collection<String> allowedTables, List<QuickSearchableTableConfig> tableConfigs)
|
||||
{
|
||||
return (buildSearchQuery(searchTerm, allowedTables, tableConfigs, false));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The search query; with named true each clause carries a query name
|
||||
** (MATCH_CLAUSE_NAME for the required searchableText match, the field name
|
||||
** for each boost clause) so a hit reports which clauses matched. Names do
|
||||
** not change matching or scoring.
|
||||
*******************************************************************************/
|
||||
Query buildSearchQuery(String searchTerm, Collection<String> allowedTables, List<QuickSearchableTableConfig> tableConfigs, boolean named)
|
||||
{
|
||||
List<FieldValue> allowed = allowedTables.stream().map(FieldValue::of).toList();
|
||||
|
||||
BoolQuery.Builder boolBuilder = new BoolQuery.Builder()
|
||||
.must(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
|
||||
{
|
||||
mm.query(searchTerm).fields("searchableText").operator(Operator.And);
|
||||
return (named ? mm.queryName(MATCH_CLAUSE_NAME) : mm);
|
||||
}))))
|
||||
.filter(Query.of(q -> q.terms(t -> t.field("sourceTable").terms(tv -> tv.value(allowed)))))
|
||||
.should(buildFieldBoostQueries(searchTerm, tableConfigs, allowedTables, named));
|
||||
|
||||
return (Query.of(q -> q.bool(boolBuilder.build())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<Query> buildFieldBoostQueries(String searchTerm, List<QuickSearchableTableConfig> tableConfigs, Collection<String> allowedTables, boolean named)
|
||||
private List<Query> buildFieldBoostQueries(String searchTerm, List<QuickSearchableTableConfig> tableConfigs, Collection<String> allowedTables)
|
||||
{
|
||||
List<Query> queries = new ArrayList<>();
|
||||
if(tableConfigs == null)
|
||||
@@ -840,15 +704,12 @@ public class QuickSearchOpenSearchClient implements Closeable
|
||||
for(String field : fields)
|
||||
{
|
||||
float boost = (weights == null || weights.get(field) == null) ? 1f : weights.get(field).floatValue();
|
||||
queries.add(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm ->
|
||||
{
|
||||
mm.query(searchTerm)
|
||||
.fields("fieldValues." + field)
|
||||
.operator(Operator.And)
|
||||
.lenient(true)
|
||||
.boost(boost);
|
||||
return (named ? mm.queryName(FIELD_CLAUSE_PREFIX + field) : mm);
|
||||
}))));
|
||||
queries.add(Query.of(q -> q.multiMatch(MultiMatchQuery.of(mm -> mm
|
||||
.query(searchTerm)
|
||||
.fields("fieldValues." + field)
|
||||
.operator(Operator.And)
|
||||
.lenient(true)
|
||||
.boost(boost)))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -891,17 +752,6 @@ public class QuickSearchOpenSearchClient implements Closeable
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The underlying OpenSearch client, for the read-only diagnostics
|
||||
** (QuickSearchDiagnosticsOperations).
|
||||
*******************************************************************************/
|
||||
OpenSearchClient getOpenSearchClient()
|
||||
{
|
||||
return (client);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the configured alias name.
|
||||
*******************************************************************************/
|
||||
|
||||
+10
-111
@@ -23,16 +23,13 @@ import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
@@ -49,9 +46,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
@@ -381,64 +376,6 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh a FULL_REINDEX run record's modifyDate: the heartbeat by which
|
||||
** other nodes tell a full reindex that is still going from one that
|
||||
** stopped (see FullReindexStep.recoverDeadRun). Nothing reads it for other
|
||||
** run types. Best effort: a failure is logged and never stops the run (one
|
||||
** judged stopped as a result fails safely before its swap).
|
||||
*******************************************************************************/
|
||||
protected void heartbeat(QuickSearchIndexRun run)
|
||||
{
|
||||
if(run == null || run.getId() == null || !FullReindexStep.RUN_TYPE.equals(run.getRunType()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", run.getId()).withValue("modifyDate", Instant.now())));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not refresh the full reindex heartbeat", e, logPair("runId", run.getId()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record IDs, among the rows' recordIds, that exist in the source table.
|
||||
*******************************************************************************/
|
||||
Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
|
||||
{
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
return (Set.of());
|
||||
}
|
||||
|
||||
String primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
|
||||
List<Serializable> recordIds = new ArrayList<>();
|
||||
rows.forEach(row -> recordIds.add(row.getValueString("recordId")));
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, recordIds)));
|
||||
|
||||
Set<String> existingIds = new HashSet<>();
|
||||
for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
existingIds.add(record.getValueString(primaryKeyField));
|
||||
}
|
||||
return (existingIds);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Update fields on a quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
@@ -492,10 +429,9 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
/*******************************************************************************
|
||||
** Index every source record matching the criteria into targetIndex (null
|
||||
** for the alias), paging by primary key so each record is read once even
|
||||
** while rows are inserted or deleted. The run's heartbeat is refreshed
|
||||
** after every page.
|
||||
** while rows are inserted or deleted.
|
||||
*******************************************************************************/
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String tableName = tableConfig.getTableName();
|
||||
@@ -555,8 +491,6 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
{
|
||||
throw new QException("Source record has no value in primary key field [" + primaryKeyField + "]");
|
||||
}
|
||||
|
||||
heartbeat(run);
|
||||
}
|
||||
|
||||
return (new IndexCounts(processed, indexed, errors, skipped, firstError));
|
||||
@@ -568,37 +502,19 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
** Delete run records older than runHistoryRetentionDays. Never throws.
|
||||
*******************************************************************************/
|
||||
protected void purgeOldRuns()
|
||||
{
|
||||
Integer retentionDays = getConfigOrNull() == null ? null : getConfigOrNull().getRunHistoryRetentionDays();
|
||||
if(retentionDays == null || retentionDays <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Instant cutoff = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
|
||||
purgeOlderThan(getConfigOrNull().getQuickSearchIndexRunTableName(), "startTime", cutoff, retentionDays);
|
||||
purgeOlderThan(getConfigOrNull().applyPrefix(QuickSearchDiagnosticEvent.TABLE_NAME), "createDate", cutoff, retentionDays);
|
||||
purgeOlderThan(getConfigOrNull().applyPrefix(QuickSearchValidationRun.TABLE_NAME), "createDate", cutoff, retentionDays);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Delete up to 5000 rows of a history table older than the cutoff. A table
|
||||
** not in the instance is skipped; failures are logged.
|
||||
*******************************************************************************/
|
||||
private void purgeOlderThan(String tableName, String dateField, Instant cutoff, Integer retentionDays)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(QContext.getQInstance() == null || QContext.getQInstance().getTable(tableName) == null)
|
||||
Integer retentionDays = getConfig().getRunHistoryRetentionDays();
|
||||
if(retentionDays == null || retentionDays <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Instant cutoff = Instant.now().minus(retentionDays, ChronoUnit.DAYS);
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(dateField, QCriteriaOperator.LESS_THAN, cutoff)).withLimit(5000));
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("startTime", QCriteriaOperator.LESS_THAN, cutoff)).withLimit(5000));
|
||||
List<QRecord> old = new QueryAction().execute(queryInput).getRecords();
|
||||
if(CollectionUtils.nullSafeIsEmpty(old))
|
||||
{
|
||||
@@ -612,31 +528,14 @@ public abstract class AbstractIndexingStep implements BackendStep
|
||||
}
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(tableName);
|
||||
deleteInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
deleteInput.setPrimaryKeys(ids);
|
||||
new DeleteAction().execute(deleteInput);
|
||||
LOG.info("Purged old Quick Search history records", logPair("tableName", tableName), logPair("count", ids.size()), logPair("olderThanDays", retentionDays));
|
||||
LOG.info("Purged old Quick Search run records", logPair("count", ids.size()), logPair("olderThanDays", retentionDays));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Failed to purge old Quick Search history records", e, logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The config, or null when it cannot be resolved (purging is best effort).
|
||||
*******************************************************************************/
|
||||
private QuickSearchQBitConfig getConfigOrNull()
|
||||
{
|
||||
try
|
||||
{
|
||||
return (getConfig());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
return (null);
|
||||
LOG.warn("Failed to purge old run records", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
@@ -56,10 +55,8 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
/*******************************************************************************
|
||||
** Scheduled catch-up. For each enabled table that is due, re-indexes records
|
||||
** whose basepull timestamp is at or after the previous run's start minus the
|
||||
** configured overlap, paging by primary key. First recovers what a full
|
||||
** reindex that stopped without finishing left behind (so its captured deletes
|
||||
** are replayed in the same run) and replays failed real-time events, and
|
||||
** afterwards purges old run history.
|
||||
** configured overlap, paging by primary key. First replays failed real-time
|
||||
** events, and afterwards purges old run history.
|
||||
**
|
||||
** The watermark stored on the quickSearchIndex row is the run's start time,
|
||||
** captured before the first query, so rows modified during the run are read
|
||||
@@ -86,16 +83,6 @@ public class BasepullIndexStep extends AbstractIndexingStep
|
||||
{
|
||||
List<String> failures = new ArrayList<>();
|
||||
|
||||
try
|
||||
{
|
||||
new FullReindexStep().recoverDeadRun(true);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Recovering from a stopped full reindex did not complete", e);
|
||||
failures.add("full reindex recovery: " + e.getMessage());
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
replayFailedEvents(output);
|
||||
@@ -224,7 +211,7 @@ public class BasepullIndexStep extends AbstractIndexingStep
|
||||
criteria.add(new QFilterCriteria(timestampField, QCriteriaOperator.GREATER_THAN_OR_EQUALS, since));
|
||||
}
|
||||
|
||||
IndexCounts counts = indexAllRecords(client, tableConfig, criteria, null, run);
|
||||
IndexCounts counts = indexAllRecords(client, tableConfig, criteria, null);
|
||||
|
||||
if(counts.errors() == 0)
|
||||
{
|
||||
@@ -269,10 +256,8 @@ public class BasepullIndexStep extends AbstractIndexingStep
|
||||
|
||||
/*******************************************************************************
|
||||
** Replay PENDING failed real-time events: re-read and index records for
|
||||
** INDEX events, delete documents for DELETE events. A DELETE whose record
|
||||
** exists again (re-created with the same key since) keeps its document and
|
||||
** counts as replayed. Successes are removed; failures bump attempts and
|
||||
** become EXHAUSTED at the limit.
|
||||
** INDEX events, delete documents for DELETE events. Successes are removed;
|
||||
** failures bump attempts and become EXHAUSTED at the limit.
|
||||
*******************************************************************************/
|
||||
void replayFailedEvents(RunBackendStepOutput output) throws QException
|
||||
{
|
||||
@@ -307,37 +292,19 @@ public class BasepullIndexStep extends AbstractIndexingStep
|
||||
|
||||
for(Map.Entry<String, List<QRecord>> entry : deleteByTable.entrySet())
|
||||
{
|
||||
List<QRecord> toDelete = entry.getValue();
|
||||
List<String> documentIds = new ArrayList<>();
|
||||
for(QRecord event : entry.getValue())
|
||||
{
|
||||
documentIds.add(OpenSearchDocument.buildDocumentId(entry.getKey(), event.getValueString("recordId")));
|
||||
}
|
||||
try
|
||||
{
|
||||
Set<String> existingIds = queryExistingRecordIds(entry.getKey(), entry.getValue());
|
||||
toDelete = new ArrayList<>();
|
||||
for(QRecord event : entry.getValue())
|
||||
{
|
||||
if(existingIds.contains(event.getValueString("recordId")))
|
||||
{
|
||||
succeeded.add(event.getValue("id"));
|
||||
}
|
||||
else
|
||||
{
|
||||
toDelete.add(event);
|
||||
}
|
||||
}
|
||||
|
||||
if(!toDelete.isEmpty())
|
||||
{
|
||||
List<String> documentIds = new ArrayList<>();
|
||||
for(QRecord event : toDelete)
|
||||
{
|
||||
documentIds.add(OpenSearchDocument.buildDocumentId(entry.getKey(), event.getValueString("recordId")));
|
||||
}
|
||||
BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
|
||||
markReplayOutcome(toDelete, result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
|
||||
}
|
||||
BulkIndexResult result = client.deleteDocuments(documentIds, config.getBulkBatchSize());
|
||||
markReplayOutcome(entry.getValue(), result.getFailureCount() == 0, result.getErrors(), succeeded, failed);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
markReplayOutcome(toDelete, false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
|
||||
markReplayOutcome(entry.getValue(), false, List.of(e.getMessage() == null ? e.toString() : e.getMessage()), succeeded, failed);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,15 +19,12 @@ package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
@@ -74,14 +71,6 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** AWAITING_REINDEX failed event. Once the rebuild's own writes are done and
|
||||
** the alias is swapped, those deletes are applied through the alias.
|
||||
**
|
||||
** Only one full reindex runs at a time. Each table's run record is RUNNING
|
||||
** before any table is marked REBUILDING, and its modifyDate is refreshed
|
||||
** after every page; a run whose records show no sign of life for
|
||||
** fullReindexStaleMinutes has stopped (killed JVM, lost node). Every full
|
||||
** reindex, and every basepull, first recovers what such a run left behind
|
||||
** (see recoverDeadRun); a full reindex then refuses to start while another
|
||||
** one is alive.
|
||||
**
|
||||
** With a "tableName" input, that table is rebuilt in place with the reconcile
|
||||
** algorithm (index everything, then delete what was not re-indexed).
|
||||
*******************************************************************************/
|
||||
@@ -99,18 +88,12 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
String tableNameFilter = input.getValueString("tableName");
|
||||
boolean allTables = tableNameFilter == null || tableNameFilter.isBlank();
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// an all-tables rebuild applies the captured deletes itself after its swap //
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
refuseIfAlive(recoverDeadRun(!allTables));
|
||||
String tableNameFilter = input.getValueString("tableName");
|
||||
|
||||
List<QuickSearchableTableConfig> tables = new ArrayList<>();
|
||||
for(QuickSearchableTableConfig tableConfig : getDiscoveredTables())
|
||||
{
|
||||
if(!allTables && !tableNameFilter.equals(tableConfig.getTableName()))
|
||||
if(tableNameFilter != null && !tableNameFilter.isBlank() && !tableNameFilter.equals(tableConfig.getTableName()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -125,7 +108,7 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
}
|
||||
}
|
||||
|
||||
if(!allTables)
|
||||
if(tableNameFilter != null && !tableNameFilter.isBlank())
|
||||
{
|
||||
Integer recordsIndexed = 0;
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
@@ -151,42 +134,29 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
QuickSearchOpenSearchClient client = getClient();
|
||||
String physicalIndex = client.newPhysicalIndexName();
|
||||
|
||||
client.createPhysicalIndex(physicalIndex);
|
||||
LOG.info("Rebuilding Quick Search index", logPair("physicalIndex", physicalIndex), logPair("tables", tables.size()));
|
||||
|
||||
Map<QuickSearchableTableConfig, QuickSearchIndexRun> runs = new LinkedHashMap<>();
|
||||
Map<String, IndexCounts> counts = new LinkedHashMap<>();
|
||||
Map<String, Instant> startTimes = new LinkedHashMap<>();
|
||||
Map<String, String> priorStatus = new LinkedHashMap<>();
|
||||
Integer indexed = 0;
|
||||
|
||||
boolean created = false;
|
||||
boolean marked = false;
|
||||
boolean swapped = false;
|
||||
|
||||
try
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// the run records exist before any table is REBUILDING, so whoever //
|
||||
// sees a table REBUILDING also sees this run alive; then make sure no //
|
||||
// other run started at the same moment //
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
runs.put(tableConfig, createRunRecord(queryIndexId(tableConfig.getTableName()), RUN_TYPE));
|
||||
}
|
||||
refuseIfAlive(queryLiveRunsOtherThan(runs.values()));
|
||||
|
||||
client.createPhysicalIndex(physicalIndex);
|
||||
created = true;
|
||||
LOG.info("Rebuilding Quick Search index", logPair("physicalIndex", physicalIndex), logPair("tables", tables.size()));
|
||||
|
||||
marked = true;
|
||||
markRebuilding(tables, priorStatus);
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
String tableName = tableConfig.getTableName();
|
||||
QuickSearchIndexRun run = createRunRecord(queryIndexId(tableName), RUN_TYPE);
|
||||
runs.put(tableConfig, run);
|
||||
startTimes.put(tableName, Instant.now());
|
||||
|
||||
IndexCounts tableCounts = indexAllRecords(client, tableConfig, List.of(), physicalIndex, runs.get(tableConfig));
|
||||
IndexCounts tableCounts = indexAllRecords(client, tableConfig, List.of(), physicalIndex);
|
||||
counts.put(tableName, tableCounts);
|
||||
indexed = indexed + tableCounts.indexed();
|
||||
|
||||
@@ -197,8 +167,6 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
}
|
||||
|
||||
client.refreshIndex(physicalIndex);
|
||||
keepAlive(runs.values());
|
||||
verifyStillOwned(tables, runs.values());
|
||||
client.swapAliasTo(physicalIndex);
|
||||
swapped = true;
|
||||
|
||||
@@ -206,7 +174,6 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
{
|
||||
String tableName = tableConfig.getTableName();
|
||||
IndexCounts tableCounts = counts.get(tableName);
|
||||
keepAlive(runs.values());
|
||||
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("lastFullReindexTime", Instant.now());
|
||||
@@ -221,31 +188,30 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
values.put("status", STATUS_ACTIVE);
|
||||
updateIndexRow(runs.get(tableConfig).getQuickSearchIndexId(), values);
|
||||
priorStatus.remove(tableName);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
// the run records stay RUNNING, refreshed per batch, until the captured //
|
||||
// deletes are applied; if a recovery takes the run over anyway, the drain //
|
||||
// stops, and it never reads rows captured after it started, which belong //
|
||||
// to a newer run //
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
applyCapturedDeletes(client, tables, runs.values());
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
IndexCounts tableCounts = counts.get(tableConfig.getTableName());
|
||||
completeRunRecord(runs.get(tableConfig), RUN_COMPLETED, tableCounts.processed(), tableCounts.indexed(), 0, null);
|
||||
}
|
||||
|
||||
applyCapturedDeletes(client, tables);
|
||||
|
||||
LOG.info("Full reindex complete", logPair("physicalIndex", physicalIndex), logPair("recordsIndexed", indexed));
|
||||
return (indexed);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
boolean takenOver = marked && isTakenOverOrUnknown(runs.values());
|
||||
LOG.warn(swapped ? "Full reindex failed after the alias swap; the new index stays in service" : "Full reindex failed; the previous index stays in service", e, logPair("physicalIndex", physicalIndex));
|
||||
|
||||
if(created && !swapped)
|
||||
for(Map.Entry<QuickSearchableTableConfig, QuickSearchIndexRun> entry : runs.entrySet())
|
||||
{
|
||||
IndexCounts tableCounts = counts.get(entry.getKey().getTableName());
|
||||
completeRunRecord(entry.getValue(), RUN_FAILED,
|
||||
tableCounts == null ? 0 : tableCounts.processed(),
|
||||
tableCounts == null ? 0 : tableCounts.indexed(),
|
||||
tableCounts == null ? 1 : Math.max(1, tableCounts.errors()),
|
||||
e.getMessage());
|
||||
}
|
||||
|
||||
if(!swapped)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -257,29 +223,8 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// a recovery that took this run over owns its tables, captured deletes //
|
||||
// and run records (a newer run may own the first two by now); when that //
|
||||
// cannot be read, the run records stay RUNNING for the next recovery //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
if(!takenOver)
|
||||
{
|
||||
if(marked)
|
||||
{
|
||||
restoreStatus(priorStatus);
|
||||
applyCapturedDeletes(client, tables, runs.values());
|
||||
}
|
||||
|
||||
for(Map.Entry<QuickSearchableTableConfig, QuickSearchIndexRun> entry : runs.entrySet())
|
||||
{
|
||||
IndexCounts tableCounts = counts.get(entry.getKey().getTableName());
|
||||
completeRunRecord(entry.getValue(), RUN_FAILED,
|
||||
tableCounts == null ? 0 : tableCounts.processed(),
|
||||
tableCounts == null ? 0 : tableCounts.indexed(),
|
||||
tableCounts == null ? 1 : Math.max(1, tableCounts.errors()),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
restoreStatus(priorStatus);
|
||||
applyCapturedDeletes(client, tables);
|
||||
|
||||
if(e instanceof QException qException)
|
||||
{
|
||||
@@ -330,421 +275,20 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recover what a full reindex that stopped without finishing (killed JVM,
|
||||
** lost node) left behind, unless a full reindex is alive. Returns the live
|
||||
** RUNNING FULL_REINDEX run records, and changes nothing when there are any.
|
||||
**
|
||||
** A run is alive while the startTime or modifyDate (its heartbeat) of one of
|
||||
** its RUNNING records is within fullReindexStaleMinutes. When none is
|
||||
** alive: each REBUILDING table goes back to ACTIVE (the status it had
|
||||
** before is not stored; drift detection marks NEEDS_REINDEX again where it
|
||||
** applies), with its lastBasepullTime moved back to the stopped run's
|
||||
** start, so basepull re-reads updates that a swap may have left out; with
|
||||
** handCapturedDeletesToBasepull, every AWAITING_REINDEX row up to the
|
||||
** newest one read here becomes PENDING; and only then is each stale RUNNING
|
||||
** record marked FAILED, so a recovery that fails partway leaves them for
|
||||
** the next one, run start included. Unlike completeRunRecord, that write
|
||||
** fails the recovery (and so the full reindex calling it) when any record
|
||||
** is not updated. Replaying those deletes is harmless
|
||||
** whether or not the stopped run swapped: before a swap the alias points at
|
||||
** the old index, which the listener already deleted from (a missing
|
||||
** document counts as success); after one, it removes the copy the run read
|
||||
** before the delete.
|
||||
**
|
||||
** The REBUILDING rows and captured rows are read before the run records. A
|
||||
** run creates its records before it marks a table, and the listener
|
||||
** captures a delete only once it sees REBUILDING, so anything read here
|
||||
** that belongs to a live run comes with a live record, and nothing of a
|
||||
** live run is touched. A run that starts after the records are read, or a
|
||||
** live run misjudged as stopped, can have a table put back to ACTIVE or its
|
||||
** records marked FAILED; it checks both right after a heartbeat, before its
|
||||
** swap, and fails without swapping. A misjudged run that already swapped
|
||||
** stops applying its captured deletes once its records are FAILED, and
|
||||
** never reads rows captured after its drain started.
|
||||
*******************************************************************************/
|
||||
List<QRecord> recoverDeadRun(boolean handCapturedDeletesToBasepull) throws QException
|
||||
{
|
||||
List<QRecord> rebuildingRows = queryRebuildingIndexRows();
|
||||
Serializable lastCapturedId = handCapturedDeletesToBasepull ? queryLastCapturedDeleteId() : null;
|
||||
List<QRecord> runningRuns = queryRunningRuns();
|
||||
|
||||
Instant cutoff = staleCutoff();
|
||||
List<QRecord> alive = runningRuns.stream().filter(run -> isAlive(run, cutoff)).toList();
|
||||
if(!alive.isEmpty() || (rebuildingRows.isEmpty() && runningRuns.isEmpty() && lastCapturedId == null))
|
||||
{
|
||||
return (alive);
|
||||
}
|
||||
|
||||
LOG.warn("Recovering what a full reindex that stopped without finishing left behind",
|
||||
logPair("staleRunIds", runningRuns.stream().map(run -> run.getValue("id")).toList()),
|
||||
logPair("tableNames", rebuildingRows.stream().map(row -> row.getValueString("tableName")).toList()));
|
||||
|
||||
for(QRecord row : rebuildingRows)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("status", STATUS_ACTIVE);
|
||||
|
||||
Instant runStart = earliestStart(runningRuns, row.getValueInteger("id"));
|
||||
Instant lastBasepullTime = row.getValueInstant("lastBasepullTime");
|
||||
if(runStart != null && lastBasepullTime != null && runStart.isBefore(lastBasepullTime))
|
||||
{
|
||||
values.put("lastBasepullTime", runStart);
|
||||
}
|
||||
updateIndexRow(row.getValueInteger("id"), values);
|
||||
}
|
||||
|
||||
if(lastCapturedId != null)
|
||||
{
|
||||
handCapturedDeletesToBasepull(lastCapturedId);
|
||||
}
|
||||
|
||||
failDeadRuns(runningRuns);
|
||||
return (alive);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Mark the stale run records FAILED, failing when any of them is not
|
||||
** updated: a live run misjudged as stopped learns it was taken over only
|
||||
** from these records, so recovery must not finish, and no new full reindex
|
||||
** may start, until they are written. The status is mirrored onto each
|
||||
** index row, as completeRunRecord does.
|
||||
*******************************************************************************/
|
||||
private void failDeadRuns(List<QRecord> runningRuns) throws QException
|
||||
{
|
||||
if(runningRuns.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Instant endTime = Instant.now();
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
for(QRecord runRecord : runningRuns)
|
||||
{
|
||||
updates.add(new QRecord()
|
||||
.withValue("id", runRecord.getValue("id"))
|
||||
.withValue("status", RUN_FAILED)
|
||||
.withValue("endTime", endTime)
|
||||
.withValue("recordsProcessed", 0)
|
||||
.withValue("recordsIndexed", 0)
|
||||
.withValue("errorCount", 1)
|
||||
.withValue("errorMessage", truncate(staleRunMessage(runRecord), 4000)));
|
||||
}
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
updateInput.setRecords(updates);
|
||||
List<QRecord> results = CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords());
|
||||
List<QRecord> written = results.stream().filter(result -> CollectionUtils.nullSafeIsEmpty(result.getErrors())).toList();
|
||||
if(written.size() < updates.size())
|
||||
{
|
||||
List<QRecord> notWritten = results.stream().filter(result -> CollectionUtils.nullSafeHasContents(result.getErrors())).toList();
|
||||
throw (new QException("Could not mark " + (updates.size() - written.size()) + " stopped full reindex run record(s) FAILED"
|
||||
+ (notWritten.isEmpty() ? "" : " (first error: " + notWritten.get(0).getErrors() + ")")
|
||||
+ "; recovery did not finish, and no full reindex starts until it does"));
|
||||
}
|
||||
|
||||
for(QRecord runRecord : runningRuns)
|
||||
{
|
||||
Integer indexId = runRecord.getValueInteger("quickSearchIndexId");
|
||||
if(indexId != null)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put("lastRunStatus", RUN_TYPE + " " + RUN_FAILED);
|
||||
values.put("lastErrorMessage", truncate(staleRunMessage(runRecord), 4000));
|
||||
updateIndexRow(indexId, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The error message recorded on a run that stopped without finishing.
|
||||
*******************************************************************************/
|
||||
private static String staleRunMessage(QRecord runRecord)
|
||||
{
|
||||
return ("Full reindex stopped without finishing (no heartbeat since " + lastSeen(runRecord) + "); recovered automatically");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Make the AWAITING_REINDEX rows up to lastCapturedId PENDING, for basepull
|
||||
** to replay. Rows are read in id order past the previous batch, so a row
|
||||
** that could not be updated is not read again.
|
||||
*******************************************************************************/
|
||||
private void handCapturedDeletesToBasepull(Serializable lastCapturedId) throws QException
|
||||
{
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
|
||||
Serializable afterId = null;
|
||||
Integer handedOver = 0;
|
||||
|
||||
while(true)
|
||||
{
|
||||
QQueryFilter filter = new QQueryFilter(
|
||||
new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX),
|
||||
new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, lastCapturedId))
|
||||
.withOrderBy(new QFilterOrderBy("id", true))
|
||||
.withLimit(config.getSourceBatchSize());
|
||||
if(afterId != null)
|
||||
{
|
||||
filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, afterId));
|
||||
}
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(failedEventTable);
|
||||
queryInput.setFilter(filter);
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
if(CollectionUtils.nullSafeIsEmpty(rows))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
afterId = rows.get(rows.size() - 1).getValue("id");
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
rows.forEach(row -> updates.add(new QRecord().withValue("id", row.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_PENDING)));
|
||||
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(failedEventTable);
|
||||
updateInput.setRecords(updates);
|
||||
new UpdateAction().execute(updateInput);
|
||||
handedOver = handedOver + rows.size();
|
||||
}
|
||||
|
||||
LOG.info("Handed deletes captured during a stopped full reindex to basepull", logPair("count", handedOver));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fail when another full reindex is alive.
|
||||
*******************************************************************************/
|
||||
private void refuseIfAlive(List<QRecord> alive) throws QException
|
||||
{
|
||||
if(alive.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
QRecord run = alive.get(0);
|
||||
throw (new QException("A full reindex is already running (run " + run.getValue("id") + ", last heartbeat " + lastSeen(run) + "); try again once it finishes. "
|
||||
+ "A run with no heartbeat for " + getConfig().getEffectiveFullReindexStaleMinutes() + " minutes is treated as stopped and recovered automatically."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Just before the swap, fail unless this run still owns its tables: its
|
||||
** run records are still RUNNING (a recovery that judged it stopped marks
|
||||
** them FAILED) and every table is still REBUILDING (otherwise a delete
|
||||
** since then may have reached only the old index, uncaptured).
|
||||
*******************************************************************************/
|
||||
private void verifyStillOwned(List<QuickSearchableTableConfig> tables, Collection<QuickSearchIndexRun> runs) throws QException
|
||||
{
|
||||
if(isTakenOver(runs))
|
||||
{
|
||||
throw (new QException("Full reindex was recovered as stopped (no heartbeat for " + getConfig().getEffectiveFullReindexStaleMinutes() + " minutes) while still running; the alias was not swapped"));
|
||||
}
|
||||
|
||||
for(QuickSearchableTableConfig tableConfig : tables)
|
||||
{
|
||||
String status = queryIndexRow(tableConfig.getTableName()).getValueString("status");
|
||||
if(!STATUS_REBUILDING.equals(status))
|
||||
{
|
||||
throw (new QException("Table [" + tableConfig.getTableName() + "] is no longer REBUILDING (now " + status + "), so a delete since then may not have been captured; the alias was not swapped"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh one of the run's records, which keeps the whole run alive.
|
||||
*******************************************************************************/
|
||||
private void keepAlive(Collection<QuickSearchIndexRun> runs)
|
||||
{
|
||||
runs.stream().findFirst().ifPresent(this::heartbeat);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether a recovery has marked any of the run records finished.
|
||||
*******************************************************************************/
|
||||
private boolean isTakenOver(Collection<QuickSearchIndexRun> runs) throws QException
|
||||
{
|
||||
if(runs.isEmpty())
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
runs.forEach(run -> ids.add(run.getId()));
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, ids)));
|
||||
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()).stream()
|
||||
.anyMatch(run -> !RUN_RUNNING.equals(run.getValueString("status"))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** isTakenOver for the failure path: never throws, and answers true when it
|
||||
** cannot tell, which leaves the tables to the next recovery.
|
||||
*******************************************************************************/
|
||||
private boolean isTakenOverOrUnknown(Collection<QuickSearchIndexRun> runs)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (isTakenOver(runs));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not read whether a recovery took over the failed full reindex; leaving its tables to the next recovery", e);
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The live RUNNING FULL_REINDEX run records other than the given ones.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryLiveRunsOtherThan(Collection<QuickSearchIndexRun> own) throws QException
|
||||
{
|
||||
Set<Integer> ownIds = new HashSet<>();
|
||||
own.forEach(run -> ownIds.add(run.getId()));
|
||||
|
||||
Instant cutoff = staleCutoff();
|
||||
return (queryRunningRuns().stream()
|
||||
.filter(run -> !ownIds.contains(run.getValueInteger("id")) && isAlive(run, cutoff))
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every RUNNING FULL_REINDEX run record, oldest first.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryRunningRuns() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexRunTableName());
|
||||
queryInput.setFilter(new QQueryFilter(
|
||||
new QFilterCriteria("runType", QCriteriaOperator.EQUALS, RUN_TYPE),
|
||||
new QFilterCriteria("status", QCriteriaOperator.EQUALS, RUN_RUNNING))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)));
|
||||
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every quickSearchIndex row that is REBUILDING.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryRebuildingIndexRows() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().getQuickSearchIndexTableName());
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, STATUS_REBUILDING)));
|
||||
return (CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The id of the newest AWAITING_REINDEX row, or null. Ids grow in insert
|
||||
** order, so a row captured after this read is beyond it.
|
||||
*******************************************************************************/
|
||||
private Serializable queryLastCapturedDeleteId() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX))
|
||||
.withOrderBy(new QFilterOrderBy("id", false))
|
||||
.withLimit(1));
|
||||
List<QRecord> rows = new QueryAction().execute(queryInput).getRecords();
|
||||
return (CollectionUtils.nullSafeIsEmpty(rows) ? null : rows.get(0).getValue("id"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run records last seen before this are stale.
|
||||
*******************************************************************************/
|
||||
private Instant staleCutoff() throws QException
|
||||
{
|
||||
return (Instant.now().minus(getConfig().getEffectiveFullReindexStaleMinutes(), ChronoUnit.MINUTES));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether a run record was last seen after the cutoff.
|
||||
*******************************************************************************/
|
||||
private static boolean isAlive(QRecord run, Instant cutoff)
|
||||
{
|
||||
Instant lastSeen = lastSeen(run);
|
||||
return (lastSeen != null && lastSeen.isAfter(cutoff));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A run record's latest sign of life: its startTime or its heartbeat
|
||||
** (modifyDate), whichever is later; null when it has neither.
|
||||
*******************************************************************************/
|
||||
private static Instant lastSeen(QRecord run)
|
||||
{
|
||||
Instant startTime = run.getValueInstant("startTime");
|
||||
Instant modifyDate = run.getValueInstant("modifyDate");
|
||||
if(startTime == null || modifyDate == null)
|
||||
{
|
||||
return (startTime == null ? modifyDate : startTime);
|
||||
}
|
||||
return (startTime.isAfter(modifyDate) ? startTime : modifyDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The earliest startTime among the run records of an index row, or null.
|
||||
*******************************************************************************/
|
||||
private static Instant earliestStart(List<QRecord> runs, Integer indexId)
|
||||
{
|
||||
return (runs.stream()
|
||||
.filter(run -> Objects.equals(indexId, run.getValueInteger("quickSearchIndexId")))
|
||||
.map(run -> run.getValueInstant("startTime"))
|
||||
.filter(Objects::nonNull)
|
||||
.min(Instant::compareTo)
|
||||
.orElse(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply the deletes the listener captured during the rebuild (failed-event
|
||||
** rows with status AWAITING_REINDEX) through the alias, after the rebuild's
|
||||
** own writes, so an in-flight page cannot bring a deleted record back.
|
||||
** Called once the tables are no longer REBUILDING, so the listener stops
|
||||
** adding rows (one that races the status change hands its own rows to
|
||||
** basepull). Only rows up to the newest one read at the start are applied,
|
||||
** since a later row belongs to a newer run. The run is kept alive before
|
||||
** every batch, and the drain stops once a recovery has taken it over.
|
||||
** basepull).
|
||||
**
|
||||
** Rows are read in id order, past the previous batch, so a batch whose rows
|
||||
** could not be removed or updated (QQQ reports those as per-row errors, not
|
||||
** exceptions) is not read again; such rows stay AWAITING_REINDEX until the
|
||||
** next full reindex, or a basepull that finds none running, picks them up.
|
||||
** Never throws.
|
||||
** exceptions) is not read again; such rows stay AWAITING_REINDEX for the next
|
||||
** full reindex. Never throws.
|
||||
*******************************************************************************/
|
||||
void applyCapturedDeletes(QuickSearchOpenSearchClient client, List<QuickSearchableTableConfig> tables, Collection<QuickSearchIndexRun> runs)
|
||||
void applyCapturedDeletes(QuickSearchOpenSearchClient client, List<QuickSearchableTableConfig> tables)
|
||||
{
|
||||
List<Serializable> tableNames = new ArrayList<>();
|
||||
tables.forEach(tableConfig -> tableNames.add(tableConfig.getTableName()));
|
||||
@@ -758,22 +302,13 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
{
|
||||
QuickSearchQBitConfig config = getConfig();
|
||||
String failedEventTable = config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME);
|
||||
Serializable lastCapturedId = queryLastCapturedDeleteId();
|
||||
Serializable afterId = null;
|
||||
|
||||
while(lastCapturedId != null)
|
||||
while(true)
|
||||
{
|
||||
if(isTakenOver(runs))
|
||||
{
|
||||
LOG.warn("A recovery took this full reindex over while it applied its captured deletes; leaving the rest to the recovery", logPair("applied", applied));
|
||||
break;
|
||||
}
|
||||
keepAlive(runs);
|
||||
|
||||
QQueryFilter filter = new QQueryFilter(
|
||||
new QFilterCriteria("status", QCriteriaOperator.EQUALS, QuickSearchFailedEvent.STATUS_AWAITING_REINDEX),
|
||||
new QFilterCriteria("tableName", QCriteriaOperator.IN, tableNames),
|
||||
new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, lastCapturedId))
|
||||
new QFilterCriteria("tableName", QCriteriaOperator.IN, tableNames))
|
||||
.withOrderBy(new QFilterOrderBy("id", true))
|
||||
.withLimit(config.getSourceBatchSize());
|
||||
if(afterId != null)
|
||||
@@ -914,6 +449,35 @@ public class FullReindexStep extends AbstractIndexingStep
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record IDs, among the rows' recordIds, that exist in the source table.
|
||||
*******************************************************************************/
|
||||
Set<String> queryExistingRecordIds(String tableName, List<QRecord> rows) throws QException
|
||||
{
|
||||
QuickSearchableTableConfig tableConfig = getTableConfig(tableName);
|
||||
if(tableConfig == null)
|
||||
{
|
||||
return (Set.of());
|
||||
}
|
||||
|
||||
String primaryKeyField = tableConfig.getPrimaryKeyField() == null ? "id" : tableConfig.getPrimaryKeyField();
|
||||
List<Serializable> recordIds = new ArrayList<>();
|
||||
rows.forEach(row -> recordIds.add(row.getValueString("recordId")));
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, recordIds)));
|
||||
|
||||
Set<String> existingIds = new HashSet<>();
|
||||
for(QRecord record : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
existingIds.add(record.getValueString(primaryKeyField));
|
||||
}
|
||||
return (existingIds);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Rebuild one table in place (kept for callers that used this directly).
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -132,7 +132,7 @@ public class ReconcileIndexStep extends AbstractIndexingStep
|
||||
try
|
||||
{
|
||||
Instant reconcileStart = Instant.now();
|
||||
IndexCounts counts = indexAllRecords(client, tableConfig, List.of(), null, run);
|
||||
IndexCounts counts = indexAllRecords(client, tableConfig, List.of(), null);
|
||||
|
||||
Long documentsRemoved = 0L;
|
||||
if(counts.errors() == 0)
|
||||
|
||||
-162
@@ -1,162 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Base of the diagnostic tool steps. Each tool's process has the steps
|
||||
** PREPARE (backend), FORM or CONFIRM (frontend: fields and an HTML block),
|
||||
** RUN (backend) and, for one-shot tools, RESULT (frontend: HTML). One step
|
||||
** class serves a tool's backend steps, switching on the step name.
|
||||
** Looping tools (playground, analyzer, inspect) end RUN by pointing the
|
||||
** process back at PREPARE, so the next step is FORM again with the results
|
||||
** below the fields: change the input, run again, without restarting.
|
||||
** HTML goes in the process value "{step}.html", which the HTML component of
|
||||
** that frontend step renders.
|
||||
*******************************************************************************/
|
||||
public abstract class AbstractDiagnosticToolStep implements BackendStep
|
||||
{
|
||||
public static final String STEP_PREPARE = "prepare";
|
||||
public static final String STEP_FORM = "form";
|
||||
public static final String STEP_CONFIRM = "confirm";
|
||||
public static final String STEP_RUN = "run";
|
||||
public static final String STEP_RESULT = "result";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One record a tool works on.
|
||||
*******************************************************************************/
|
||||
public record Target(String tableName, String recordId)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** "table:recordId"
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return (tableName + ":" + recordId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse "table:recordId" (null when malformed).
|
||||
*******************************************************************************/
|
||||
public static Target parse(String value)
|
||||
{
|
||||
int colon = value == null ? -1 : value.indexOf(':');
|
||||
return (colon <= 0 || colon == value.length() - 1 ? null : new Target(value.substring(0, colon), value.substring(colon + 1)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the HTML shown by a frontend step.
|
||||
*******************************************************************************/
|
||||
protected static void setHtml(RunBackendStepOutput output, String frontendStepName, String html)
|
||||
{
|
||||
output.addValue(frontendStepName + ".html", html);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** After RUN, show FORM again (the step after PREPARE).
|
||||
*******************************************************************************/
|
||||
protected static void loopBackToForm(RunBackendStepOutput output)
|
||||
{
|
||||
output.setOverrideLastStepName(STEP_PREPARE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A failure shown in the step's HTML instead of ending the process, so the
|
||||
** user can change the input and run again.
|
||||
*******************************************************************************/
|
||||
protected static String failureHtml(String what, Exception e)
|
||||
{
|
||||
return (new DiagnosticsHtml().open().status(DiagnosticsHtml.Level.FAIL, what + ": " + e.getMessage()).close().toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The records the process was started on, as targets: for a source table,
|
||||
** its primary keys; for the failed event table, each row's table and
|
||||
** record id (only tables Quick Search indexes). Read from the records the
|
||||
** backend loaded into the process state (never from client values), so
|
||||
** every step works on exactly the records the user selected and may read.
|
||||
** At most max; more is a user-facing error.
|
||||
*******************************************************************************/
|
||||
protected static List<Target> targetsFromInput(RunBackendStepInput input, QuickSearchRuntime runtime, int max) throws QException
|
||||
{
|
||||
QProcessMetaData process = QContext.getQInstance().getProcess(input.getProcessName());
|
||||
String tableName = process == null ? null : process.getTableName();
|
||||
QTableMetaData table = tableName == null ? null : QContext.getQInstance().getTable(tableName);
|
||||
if(table == null)
|
||||
{
|
||||
throw (new QUserFacingException("This tool runs on records of a table."));
|
||||
}
|
||||
|
||||
List<QRecord> records = CollectionUtils.nonNullList(input.getRecords());
|
||||
if(records.isEmpty())
|
||||
{
|
||||
throw (new QUserFacingException("Select at least one record."));
|
||||
}
|
||||
if(records.size() > max)
|
||||
{
|
||||
throw (new QUserFacingException("Select at most " + max + " records (" + records.size() + "+ selected)."));
|
||||
}
|
||||
|
||||
boolean failedEvents = tableName.equals(runtime.getConfig().applyPrefix(QuickSearchFailedEvent.TABLE_NAME));
|
||||
Set<Target> targets = new LinkedHashSet<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
String targetTable = failedEvents ? record.getValueString("tableName") : tableName;
|
||||
String recordId = failedEvents ? record.getValueString("recordId") : record.getValueString(table.getPrimaryKeyField());
|
||||
if(StringUtils.hasContent(targetTable) && StringUtils.hasContent(recordId) && runtime.getTableConfig(targetTable) != null)
|
||||
{
|
||||
targets.add(new Target(targetTable, recordId));
|
||||
}
|
||||
}
|
||||
return (new ArrayList<>(targets));
|
||||
}
|
||||
|
||||
}
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes.diagnostics;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.AnalyzerTester;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Steps of the Test Search Analyzer process (a table action on Quick Search
|
||||
** Index): run the index's analyzers on text, and check a query against it.
|
||||
** Loops back to the form.
|
||||
*******************************************************************************/
|
||||
public class AnalyzeTextStep extends AbstractDiagnosticToolStep
|
||||
{
|
||||
public static final String FIELD_TEXT = "analyzeText";
|
||||
public static final String FIELD_QUERY = "analyzeQuery";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = DiagnosticsSupport.requireRuntime();
|
||||
if(STEP_PREPARE.equals(input.getStepName()))
|
||||
{
|
||||
setHtml(output, STEP_FORM, new DiagnosticsHtml().open()
|
||||
.paragraph("Enter text as it would be indexed (a field value) and, optionally, a search term. You get the tokens each of the index's analyzers produces "
|
||||
+ "and, with a term, whether every term token is among the text's indexed tokens: that is how a document's searchable text matches. Nothing is changed.")
|
||||
.close().toString());
|
||||
return;
|
||||
}
|
||||
|
||||
loopBackToForm(output);
|
||||
long start = System.currentTimeMillis();
|
||||
try
|
||||
{
|
||||
DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_ANALYZE_TEXT);
|
||||
}
|
||||
catch(QUserFacingException e)
|
||||
{
|
||||
setHtml(output, STEP_FORM, failureHtml("Not run", e));
|
||||
return;
|
||||
}
|
||||
|
||||
String query = input.getValueString(FIELD_QUERY);
|
||||
try
|
||||
{
|
||||
setHtml(output, STEP_FORM, AnalyzerTester.render(new AnalyzerTester().analyze(runtime, input.getValueString(FIELD_TEXT), query)));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_ANALYZE_TEXT, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, null, query, start, null, null);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
setHtml(output, STEP_FORM, failureHtml("Analyzing failed (is OpenSearch reachable and the index created?)", e));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_ANALYZE_TEXT, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, null, query, start, e, null);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-92
@@ -1,92 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.RecordInspector;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Steps of Inspect Search Document, a record action on every indexed table
|
||||
** (and on Quick Search Failed Event, for the event's record). PREPARE shows
|
||||
** each selected record's indexed document next to the database; RUN adds
|
||||
** the match analysis for the entered term and loops back to the form.
|
||||
** Read-only.
|
||||
*******************************************************************************/
|
||||
public class InspectRecordStep extends AbstractDiagnosticToolStep
|
||||
{
|
||||
public static final String FIELD_TERM = "searchTerm";
|
||||
public static final int MAX_RECORDS = 10;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = DiagnosticsSupport.requireRuntime();
|
||||
if(!STEP_PREPARE.equals(input.getStepName()))
|
||||
{
|
||||
loopBackToForm(output);
|
||||
}
|
||||
List<Target> targets = targetsFromInput(input, runtime, MAX_RECORDS);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
String term = STEP_PREPARE.equals(input.getStepName()) ? null : input.getValueString(FIELD_TERM);
|
||||
try
|
||||
{
|
||||
DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_INSPECT_RECORD);
|
||||
}
|
||||
catch(QUserFacingException e)
|
||||
{
|
||||
setHtml(output, STEP_FORM, failureHtml("Not run", e));
|
||||
return;
|
||||
}
|
||||
|
||||
Set<String> tables = new LinkedHashSet<>();
|
||||
try
|
||||
{
|
||||
List<RecordInspector.Inspection> inspections = new ArrayList<>();
|
||||
for(Target target : targets)
|
||||
{
|
||||
tables.add(target.tableName());
|
||||
inspections.add(new RecordInspector().inspect(runtime, target.tableName(), target.recordId(), term, input));
|
||||
}
|
||||
setHtml(output, STEP_FORM, RecordInspector.render(inspections, term));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_INSPECT_RECORD, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, tables, term, start, null,
|
||||
"records: " + String.join(", ", targets.stream().map(Target::toString).toList()));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
setHtml(output, STEP_FORM, failureHtml("Inspecting failed (is OpenSearch reachable?)", e));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_INSPECT_RECORD, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, tables, term, start, e, null);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-109
@@ -1,109 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.RecordReindexer;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Steps of Reindex in Search, a record action on every indexed table (and
|
||||
** on Quick Search Failed Event, for the event's record): PREPARE asks to
|
||||
** confirm, RUN rebuilds the selected records' documents (RecordReindexer)
|
||||
** and shows the outcome. The only diagnostic action that writes to the
|
||||
** index; it needs the Quick Search admin permission.
|
||||
*******************************************************************************/
|
||||
public class ReindexRecordsStep extends AbstractDiagnosticToolStep
|
||||
{
|
||||
public static final int MAX_RECORDS = 500;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = DiagnosticsSupport.requireRuntime();
|
||||
if(STEP_PREPARE.equals(input.getStepName()))
|
||||
{
|
||||
List<Target> targets = targetsFromInput(input, runtime, MAX_RECORDS);
|
||||
Map<String, List<String>> byTable = groupByTable(targets);
|
||||
List<String> lines = new ArrayList<>();
|
||||
byTable.forEach((table, ids) -> lines.add(ids.size() + " record(s) of " + table));
|
||||
setHtml(output, STEP_CONFIRM, new DiagnosticsHtml().open()
|
||||
.paragraph("Rebuild the search documents of " + String.join(", ", lines) + " from the database now? Records that no longer exist have their documents removed. "
|
||||
+ "Records are not changed. The run is recorded in Quick Search Diagnostic Event.")
|
||||
.close().toString());
|
||||
return;
|
||||
}
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_REINDEX_RECORDS);
|
||||
|
||||
StringBuilder html = new StringBuilder();
|
||||
for(Map.Entry<String, List<String>> entry : groupByTable(targetsFromInput(input, runtime, MAX_RECORDS)).entrySet())
|
||||
{
|
||||
try
|
||||
{
|
||||
List<RecordReindexer.Outcome> outcomes = new RecordReindexer().reindex(runtime, entry.getKey(), entry.getValue());
|
||||
html.append(RecordReindexer.render(entry.getKey(), outcomes));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_REINDEX_RECORDS, QuickSearchDiagnosticEvent.TYPE_REINDEX, List.of(entry.getKey()), null, start, null,
|
||||
"records: " + String.join(", ", entry.getValue()) + "; outcomes: " + String.join(", ", outcomes.stream().map(o -> o.recordId() + "=" + o.outcome()).toList()));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
html.append(failureHtml("Reindexing " + entry.getKey() + " failed", e));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_REINDEX_RECORDS, QuickSearchDiagnosticEvent.TYPE_REINDEX, List.of(entry.getKey()), null, start, e,
|
||||
"records: " + String.join(", ", entry.getValue()));
|
||||
}
|
||||
}
|
||||
setHtml(output, STEP_RESULT, html.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record ids grouped by table, in selection order.
|
||||
*******************************************************************************/
|
||||
static Map<String, List<String>> groupByTable(List<Target> targets) throws QUserFacingException
|
||||
{
|
||||
if(targets.isEmpty())
|
||||
{
|
||||
throw (new QUserFacingException("No records to reindex."));
|
||||
}
|
||||
Map<String, List<String>> byTable = new LinkedHashMap<>();
|
||||
for(Target target : targets)
|
||||
{
|
||||
byTable.computeIfAbsent(target.tableName(), k -> new ArrayList<>()).add(target.recordId());
|
||||
}
|
||||
return (byTable);
|
||||
}
|
||||
|
||||
}
|
||||
-169
@@ -1,169 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes.diagnostics;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.SearchPathDiagnosis;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.SearchPlayground;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Steps of the Search Playground process (a table action on Quick Search
|
||||
** Index). PREPARE shows which tables are indexed and the provider's state,
|
||||
** and pre-fills the tables from the selected Quick Search Index rows; RUN
|
||||
** runs SearchPlayground as the current user and loops back to the form.
|
||||
*******************************************************************************/
|
||||
public class SearchPlaygroundStep extends AbstractDiagnosticToolStep
|
||||
{
|
||||
public static final String FIELD_TERM = "searchTerm";
|
||||
public static final String FIELD_TABLES = "tableNames";
|
||||
public static final String FIELD_LIMIT_PER_TABLE = "limitPerTable";
|
||||
public static final String FIELD_EXPLAIN_HIT = "explainHit";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = DiagnosticsSupport.requireRuntime();
|
||||
if(STEP_PREPARE.equals(input.getStepName()))
|
||||
{
|
||||
prepare(input, output, runtime);
|
||||
return;
|
||||
}
|
||||
|
||||
loopBackToForm(output);
|
||||
long start = System.currentTimeMillis();
|
||||
try
|
||||
{
|
||||
DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_PLAYGROUND);
|
||||
}
|
||||
catch(QUserFacingException e)
|
||||
{
|
||||
setHtml(output, STEP_FORM, failureHtml("Not run", e));
|
||||
return;
|
||||
}
|
||||
|
||||
String term = input.getValueString(FIELD_TERM);
|
||||
List<String> tables = parseTables(input.getValueString(FIELD_TABLES));
|
||||
try
|
||||
{
|
||||
SearchPlayground.Report report = new SearchPlayground().run(runtime,
|
||||
new SearchPlayground.Input(term, tables, input.getValueInteger(FIELD_LIMIT_PER_TABLE), input.getValueString(FIELD_EXPLAIN_HIT)), input);
|
||||
setHtml(output, STEP_FORM, SearchPlayground.render(report));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_PLAYGROUND, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, tables, term, start, null,
|
||||
report.coreError() != null ? "global search failed: " + report.coreError() : (report.providerError() != null ? "provider failed: " + report.providerError() : null));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
setHtml(output, STEP_FORM, failureHtml("The playground failed", e));
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_PLAYGROUND, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, tables, term, start, e, null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Intro HTML and table pre-fill.
|
||||
*******************************************************************************/
|
||||
private void prepare(RunBackendStepInput input, RunBackendStepOutput output, QuickSearchRuntime runtime) throws QException
|
||||
{
|
||||
List<String> selected = selectedIndexTables(input, runtime);
|
||||
if(!selected.isEmpty())
|
||||
{
|
||||
output.addValue(FIELD_TABLES, String.join(", ", selected));
|
||||
}
|
||||
|
||||
List<String> indexed = new ArrayList<>();
|
||||
runtime.getDiscoveredTables().forEach(t -> indexed.add(t.getTableName()));
|
||||
|
||||
SearchPathDiagnosis.ProviderState state = SearchPathDiagnosis.providerState(QContext.getQInstance(), runtime);
|
||||
DiagnosticsHtml html = new DiagnosticsHtml().open()
|
||||
.paragraph("Type a term and run it. The playground runs it the way the global search does, as you, and shows the path each table takes, what you would see, "
|
||||
+ "and the OpenSearch query, scores, matched fields and raw response behind it. Leave the tables blank to search every searchable table. "
|
||||
+ "To explain a hit, copy its document id (table:id) into Explain hit. Nothing is changed.")
|
||||
.facts(List.of(
|
||||
new String[] { "Indexed tables", indexed.isEmpty() ? null : String.join(", ", indexed) },
|
||||
new String[] { "Provider", Boolean.TRUE.equals(state.quickSearchProvider()) ? "Quick Search serves the global search" : "Quick Search does not serve the global search" },
|
||||
new String[] { "Backing off", state.backoffRemainingMillis() > 0 ? (state.backoffRemainingMillis() / 1000) + " s left after a failure" : "no" }))
|
||||
.close();
|
||||
setHtml(output, STEP_FORM, html.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Table names of the Quick Search Index rows the process was started on
|
||||
** (the recordIds value), if any.
|
||||
*******************************************************************************/
|
||||
static List<String> selectedIndexTables(RunBackendStepInput input, QuickSearchRuntime runtime) throws QException
|
||||
{
|
||||
String recordIds = input.getValueString("recordIds");
|
||||
String table = runtime.getConfig().getQuickSearchIndexTableName();
|
||||
if(!StringUtils.hasContent(recordIds) || QContext.getQInstance().getTable(table) == null)
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(table);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, Arrays.stream(recordIds.split(",")).map(String::trim).filter(s -> !s.isEmpty()).toList())));
|
||||
List<String> tables = new ArrayList<>();
|
||||
for(QRecord row : CollectionUtils.nonNullList(new QueryAction().execute(queryInput).getRecords()))
|
||||
{
|
||||
tables.add(row.getValueString("tableName"));
|
||||
}
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Table names from a comma- or space-separated list.
|
||||
*******************************************************************************/
|
||||
public static List<String> parseTables(String value)
|
||||
{
|
||||
if(!StringUtils.hasContent(value))
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
return (Arrays.stream(value.split("[,\\s]+")).map(String::trim).filter(s -> !s.isEmpty()).distinct().toList());
|
||||
}
|
||||
|
||||
}
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.processes.diagnostics;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsSupport;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.IndexValidator;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Steps of Validate Search Index (a table action on Quick Search Index, and
|
||||
** schedulable with validationCronExpression). PREPARE picks the tables (the
|
||||
** selected Quick Search Index rows, else all); RUN validates them
|
||||
** (IndexValidator), saves the report as a Quick Search Validation Run row
|
||||
** and shows it. One validation runs at a time per server. A scheduled run
|
||||
** (frontend steps skipped) validates every table with the configured sample
|
||||
** size and is not rate-limited.
|
||||
*******************************************************************************/
|
||||
public class ValidateIndexStep extends AbstractDiagnosticToolStep
|
||||
{
|
||||
public static final String FIELD_SAMPLE_SIZE = "sampleSize";
|
||||
public static final String FIELD_TABLES = "validateTableNames";
|
||||
public static final String FIELD_RUN_ID = "validationRunId";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = DiagnosticsSupport.requireRuntime();
|
||||
QuickSearchQBitConfig config = runtime.getConfig();
|
||||
if(STEP_PREPARE.equals(input.getStepName()))
|
||||
{
|
||||
List<String> tables = SearchPlaygroundStep.selectedIndexTables(input, runtime);
|
||||
output.addValue(FIELD_TABLES, String.join(",", tables));
|
||||
if(input.getValue(FIELD_SAMPLE_SIZE) == null)
|
||||
{
|
||||
output.addValue(FIELD_SAMPLE_SIZE, config.getValidationSampleSize());
|
||||
}
|
||||
setHtml(output, STEP_FORM, new DiagnosticsHtml().open()
|
||||
.paragraph("Validate " + (tables.isEmpty() ? "every indexed table" : String.join(", ", tables)) + ": cluster and index health, the mapping and analyzers, "
|
||||
+ "database rows vs indexed documents, and a sample of records and documents checked for missing, stale and orphaned documents. "
|
||||
+ "Read-only; the report is saved as a Quick Search Validation Run.")
|
||||
.close().toString());
|
||||
return;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// a scheduled run skips the frontend and has no user; an API caller //
|
||||
// asking to skip the frontend is still a user, so still rate-limited //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
QSession session = QContext.getQSession();
|
||||
boolean scheduled = input.getFrontendStepBehavior() == RunProcessInput.FrontendStepBehavior.SKIP
|
||||
&& (session == null || session instanceof QSystemUserSession || session.getUser() == null);
|
||||
long start = System.currentTimeMillis();
|
||||
if(!scheduled)
|
||||
{
|
||||
DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_VALIDATE_INDEX);
|
||||
}
|
||||
if(!runtime.beginValidation())
|
||||
{
|
||||
throw (new QUserFacingException("A validation is already running on this server. Try again when it finishes."));
|
||||
}
|
||||
|
||||
List<String> tables = SearchPlaygroundStep.parseTables(input.getValueString(FIELD_TABLES));
|
||||
Integer sample = input.getValueInteger(FIELD_SAMPLE_SIZE);
|
||||
try
|
||||
{
|
||||
IndexValidator validator = new IndexValidator();
|
||||
IndexValidator.Report report = validator.validate(runtime, tables, sample == null ? config.getValidationSampleSize() : sample);
|
||||
Integer runId = validator.persist(runtime, report, scheduled ? QuickSearchValidationRun.TRIGGER_SCHEDULED : QuickSearchValidationRun.TRIGGER_MANUAL,
|
||||
DiagnosticsSupport.currentUserName());
|
||||
|
||||
String link = runId == null
|
||||
? "<p><em>The report was not saved (the validation run table is not configured or the insert failed; see the log).</em></p>"
|
||||
: "<p>Saved as <a href=\"/app/" + DiagnosticsHtml.escape(config.applyPrefix(QuickSearchValidationRun.TABLE_NAME)) + "/" + runId + "\">validation run #" + runId + "</a>.</p>";
|
||||
setHtml(output, STEP_RESULT, IndexValidator.render(report) + link);
|
||||
output.addValue(FIELD_RUN_ID, runId);
|
||||
output.addValue("status", report.status());
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_VALIDATE_INDEX, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, report.tableNames(), null, start, null,
|
||||
IndexValidator.summary(report) + (runId == null ? "" : " run #" + runId));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
DiagnosticsSupport.recordToolRun(runtime, DiagnosticsSupport.TOOL_VALIDATE_INDEX, QuickSearchDiagnosticEvent.TYPE_TOOL_RUN, tables, null, start, e, null);
|
||||
throw (e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
runtime.endValidation();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,466 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.developer.DeveloperPanelsAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanel;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.AnalyzerTester;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Check;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.DiagnosticsHtml.Level;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.IndexValidator;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.QuickSearchDeveloperPanelProvider;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.RecordInspector;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.RecordReindexer;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.SearchPathDiagnosis;
|
||||
import com.kingsrook.qbits.quicksearch.diagnostics.SearchPlayground;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchDiagnosticsOperations;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.AbstractDiagnosticToolStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.InspectRecordStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The diagnostic tools against a real OpenSearch (Testcontainers): the
|
||||
** playground (paths, provider ids, global search results, scores, matched
|
||||
** clauses, highlights, query DSL, explain, term tokens), the analyzer
|
||||
** tester, record inspection (current, stale, missing, and why a term does
|
||||
** or does not match), index validation (counts, missing, stale and orphaned
|
||||
** documents, mapping, analyzers, leftover physical index, the saved
|
||||
** report), reindexing records, and the tool processes end to end.
|
||||
** Real-time indexing is off so the tests can make the index drift.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class DiagnosticsIntegrationTest
|
||||
{
|
||||
private static final String BACKEND = "memory";
|
||||
private static final String PRODUCT = "diagProduct";
|
||||
private static final String INDEX = "quick_search_diagnostics_it";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
private static QuickSearchQBitConfig config;
|
||||
private static QuickSearchRuntime runtime;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Instance with one indexed product table; four products indexed.
|
||||
*******************************************************************************/
|
||||
@BeforeAll
|
||||
static void setUpAll() throws Exception
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(PRODUCT)
|
||||
.withLabel("Product")
|
||||
.withBackendName(BACKEND)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s")
|
||||
.withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("description", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
|
||||
config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND)
|
||||
.withOpensearchHost(opensearch.getHost())
|
||||
.withOpensearchPort(opensearch.getMappedPort(9200))
|
||||
.withOpensearchIndexName(INDEX)
|
||||
.withEnableRealTimeIndexing(false)
|
||||
.withDiagnosticRunsPerMinute(1000)
|
||||
.withSearchableTable(new SearchableTableConfig(PRODUCT, List.of(
|
||||
new SearchableFieldConfig("name").withWeight(3),
|
||||
new SearchableFieldConfig("description"))));
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
QContext.init(qInstance, new QSession());
|
||||
runtime = config.getRuntime();
|
||||
|
||||
new InsertAction().execute(new InsertInput(PRODUCT).withRecords(List.of(
|
||||
product(1, "Blue Widget", "the original"),
|
||||
product(2, "Widget", "comes in blue and green"),
|
||||
product(3, "Red Widget", "not that colour"),
|
||||
product(4, "Gadget", "no match here"))));
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
runtime.getClient().refreshIndex();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@AfterAll
|
||||
static void tearDownAll()
|
||||
{
|
||||
if(runtime != null)
|
||||
{
|
||||
runtime.close();
|
||||
}
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QRecord product(int id, String name, String description)
|
||||
{
|
||||
return (new QRecord().withValue("id", id).withValue("name", name).withValue("description", description).withValue("modifyDate", Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QuickSearchDiagnosticsOperations ops() throws Exception
|
||||
{
|
||||
return (new QuickSearchDiagnosticsOperations(runtime.getClient()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static Check check(List<Check> checks, String name)
|
||||
{
|
||||
return (checks.stream().filter(c -> c.name().equals(name)).findFirst().orElseThrow(() -> new AssertionError("no check " + name + " in " + checks)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(1)
|
||||
void testPlayground_fullPicture() throws Exception
|
||||
{
|
||||
SearchPlayground.Report report = new SearchPlayground().run(runtime, new SearchPlayground.Input("blue widget", List.of(PRODUCT), 5, PRODUCT + ":1"), new QueryInput());
|
||||
|
||||
assertThat(report.paths()).singleElement().satisfies(p -> assertThat(p.path()).isEqualTo(SearchPathDiagnosis.PATH_OPENSEARCH));
|
||||
assertThat(report.providerError()).isNull();
|
||||
assertThat(report.providerIds().get(PRODUCT)).containsExactlyInAnyOrder("1", "2");
|
||||
assertThat(report.coreError()).isNull();
|
||||
assertThat(report.coreResults().get(PRODUCT)).extracting(r -> String.valueOf(r.getRecordId())).containsExactlyInAnyOrder("1", "2");
|
||||
assertThat(report.termTokens()).extracting(QuickSearchDiagnosticsOperations.AnalyzedToken::token).containsExactly("blue", "widget");
|
||||
|
||||
SearchPlayground.TableProbe probe = report.probes().get(0);
|
||||
assertThat(probe.error()).isNull();
|
||||
assertThat(probe.hiddenCount()).isZero();
|
||||
assertThat(report.providerHiddenCounts().get(PRODUCT)).isZero();
|
||||
assertThat(probe.probe().hits()).hasSize(2);
|
||||
QuickSearchDiagnosticsOperations.ProbeHit top = probe.probe().hits().get(0);
|
||||
assertThat(top.recordId()).isEqualTo("1");
|
||||
assertThat(top.matchedClauses()).contains("searchableText", "field:name");
|
||||
assertThat(top.highlights().values().stream().flatMap(List::stream).toList()).anyMatch(f -> f.contains("<em>"));
|
||||
assertThat(probe.readableIds()).contains("1", "2");
|
||||
assertThat(probe.shownIds()).contains("1", "2");
|
||||
assertThat(probe.probe().requestJson()).contains("\"_name\"").contains("searchableText");
|
||||
assertThat(probe.probe().responseJson()).contains("\"hits\"");
|
||||
|
||||
assertThat(report.explain().documentFound()).isTrue();
|
||||
assertThat(report.explain().matched()).isTrue();
|
||||
assertThat(report.explain().score()).isPositive();
|
||||
assertThat(report.explain().explanationText()).isNotBlank();
|
||||
|
||||
String html = SearchPlayground.render(report);
|
||||
assertThat(html).contains("Query DSL").contains("Raw response").contains("shown in global search").contains("<mark>").contains("Matched: searchableText, name");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(2)
|
||||
void testAnalyzer_tokensAndQueryMatch() throws Exception
|
||||
{
|
||||
AnalyzerTester.Result result = new AnalyzerTester().analyze(runtime, "Blue Widgets", "wid x");
|
||||
assertThat(result.indexTokens()).extracting(QuickSearchDiagnosticsOperations.AnalyzedToken::token).contains("bl", "blue", "wi", "wid", "widgets");
|
||||
assertThat(result.textAsQueryTokens()).extracting(QuickSearchDiagnosticsOperations.AnalyzedToken::token).containsExactly("blue", "widgets");
|
||||
assertThat(result.queryTokens()).extracting(QuickSearchDiagnosticsOperations.AnalyzedToken::token).containsExactly("wid", "x");
|
||||
|
||||
String html = AnalyzerTester.render(result);
|
||||
assertThat(html).contains("never indexed (n-grams start at 2)").contains("At least one query token is missing");
|
||||
assertThat(AnalyzerTester.render(new AnalyzerTester().analyze(runtime, "Blue Widgets", "wid blu"))).contains("Every query token is found");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(3)
|
||||
void testInspect_currentRecordAndMatchReasons() throws Exception
|
||||
{
|
||||
RecordInspector.Inspection current = new RecordInspector().inspect(runtime, PRODUCT, "1", null, new QueryInput());
|
||||
assertThat(current.verdict()).isEqualTo(RecordInspector.VERDICT_CURRENT);
|
||||
assertThat(current.diffs()).allMatch(RecordInspector.FieldDiff::isSame);
|
||||
assertThat(current.indexed().version()).isNotNull();
|
||||
|
||||
RecordInspector.Inspection matched = new RecordInspector().inspect(runtime, PRODUCT, "1", "blue", new QueryInput());
|
||||
assertThat(check(matched.matchChecks(), "Token \"blue\"").level()).isEqualTo(Level.OK);
|
||||
assertThat(check(matched.matchChecks(), "OpenSearch explain").level()).isEqualTo(Level.OK);
|
||||
assertThat(check(matched.matchChecks(), "Rank").level()).isEqualTo(Level.OK);
|
||||
|
||||
RecordInspector.Inspection oneChar = new RecordInspector().inspect(runtime, PRODUCT, "1", "blue x", new QueryInput());
|
||||
assertThat(check(oneChar.matchChecks(), "Token \"x\"").message()).contains("One character");
|
||||
assertThat(check(oneChar.matchChecks(), "OpenSearch explain").level()).isEqualTo(Level.FAIL);
|
||||
|
||||
RecordInspector.Inspection tooLong = new RecordInspector().inspect(runtime, PRODUCT, "1", "abcdefghijklmnopqrstuvwxy", new QueryInput());
|
||||
assertThat(check(tooLong.matchChecks(), "Token \"abcdefghijklmnopqrstuvwxy\"").message()).contains("Longer than 20");
|
||||
|
||||
RecordInspector.Inspection absent = new RecordInspector().inspect(runtime, PRODUCT, "1", "gadget", new QueryInput());
|
||||
assertThat(check(absent.matchChecks(), "Token \"gadget\"").message()).contains("Not in this record");
|
||||
|
||||
assertThat(RecordInspector.render(List.of(matched), "blue")).contains("Indexed and current").contains("Why does (or doesn't) this record match");
|
||||
|
||||
RecordInspector.Inspection none = new RecordInspector().inspect(runtime, PRODUCT, "999", null, new QueryInput());
|
||||
assertThat(none.verdict()).isEqualTo(RecordInspector.VERDICT_NO_RECORD);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(3)
|
||||
void testDeveloperPanel_currentRecordIsIndexed() throws Exception
|
||||
{
|
||||
QSession previous = QContext.getQSession();
|
||||
try
|
||||
{
|
||||
QContext.setQSession(new QSession()
|
||||
.withPermission(PermissionsHelper.getTablePermissionName(runtime.getConfig().getQuickSearchIndexTableName(), TablePermissionSubType.READ))
|
||||
.withPermission(QuickSearchQBitConfig.DEFAULT_DEBUG_PERMISSION_BASE_NAME + ".hasAccess"));
|
||||
|
||||
DeveloperPanelContext context = new DeveloperPanelContext()
|
||||
.withTable(QContext.getQInstance().getTable(PRODUCT))
|
||||
.withRecord(new QRecord().withValue("id", 1));
|
||||
DeveloperPanel panel = new DeveloperPanelsAction().execute(context).stream()
|
||||
.filter(p -> QuickSearchDeveloperPanelProvider.ID.equals(p.getId())).findFirst().orElseThrow();
|
||||
|
||||
assertThat(panel.getError()).isNull();
|
||||
assertThat(panel.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_INDEXED);
|
||||
assertThat(panel.getBlocks()).anyMatch(block -> "Database vs index".equals(block.getLabel()) && block.getRows().stream().allMatch(row -> "yes".equals(row.get(3))));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.setQSession(previous);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(4)
|
||||
void testInspect_staleAndMissing() throws Exception
|
||||
{
|
||||
new UpdateAction().execute(new UpdateInput(PRODUCT).withRecord(new QRecord().withValue("id", 3).withValue("name", "Crimson Widget").withValue("modifyDate", Instant.now())));
|
||||
new InsertAction().execute(new InsertInput(PRODUCT).withRecord(product(5, "Purple Widget", "brand new")));
|
||||
|
||||
RecordInspector.Inspection stale = new RecordInspector().inspect(runtime, PRODUCT, "3", "crimson", new QueryInput());
|
||||
assertThat(stale.verdict()).isEqualTo(RecordInspector.VERDICT_STALE);
|
||||
assertThat(stale.diffs()).filteredOn(d -> !d.isSame()).extracting(RecordInspector.FieldDiff::name).contains("fieldValues.name", "searchableText", "recordLabel");
|
||||
assertThat(check(stale.matchChecks(), "Token \"crimson\"").level()).isEqualTo(Level.WARN);
|
||||
assertThat(check(stale.matchChecks(), "Indexed document").level()).isEqualTo(Level.WARN);
|
||||
|
||||
RecordInspector.Inspection missing = new RecordInspector().inspect(runtime, PRODUCT, "5", "purple", new QueryInput());
|
||||
assertThat(missing.verdict()).isEqualTo(RecordInspector.VERDICT_MISSING);
|
||||
assertThat(check(missing.matchChecks(), "Indexed document").level()).isEqualTo(Level.FAIL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(5)
|
||||
void testValidate_findsDriftOrphansAndLeftovers() throws Exception
|
||||
{
|
||||
runtime.getClient().indexDocuments(List.of(new OpenSearchDocument().withSourceTable(PRODUCT).withRecordId("99").withSearchableText("orphan")
|
||||
.withFieldValues(Map.of("name", "orphan")).withIndexedAt(Instant.now())), 10);
|
||||
runtime.getClient().refreshIndex();
|
||||
QuickSearchDiagnosticsOperations.RawResponse created = ops().raw("PUT", "/" + INDEX + "-v2-1", "{}");
|
||||
assertThat(created.isOk()).isTrue();
|
||||
|
||||
try
|
||||
{
|
||||
IndexValidator validator = new IndexValidator();
|
||||
IndexValidator.Report report = validator.validate(runtime, List.of(), 50);
|
||||
assertThat(report.status()).isEqualTo(QuickSearchValidationRun.STATUS_FAIL);
|
||||
|
||||
IndexValidator.TableStats stats = report.tableStats().get(0);
|
||||
assertThat(stats.databaseCount()).isEqualTo(5);
|
||||
assertThat(stats.indexCount()).isEqualTo(5L);
|
||||
assertThat(stats.missingIds()).containsExactly("5");
|
||||
assertThat(stats.staleFields()).containsKey("3");
|
||||
assertThat(stats.extraIds()).containsExactly("99");
|
||||
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Mapping version") && f.level() == Level.OK);
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Field mappings") && f.level() == Level.OK);
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Analysis settings") && f.level() == Level.OK);
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Alias") && f.level() == Level.OK);
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Leftover index") && f.message().contains(INDEX + "-v2-1"));
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Sampled documents") && f.level() == Level.FAIL && f.message().contains("99"));
|
||||
|
||||
Integer runId = validator.persist(runtime, report, QuickSearchValidationRun.TRIGGER_MANUAL, "tester");
|
||||
assertThat(runId).isNotNull();
|
||||
QRecord row = new QueryAction().execute(new QueryInput(QuickSearchValidationRun.TABLE_NAME)).getRecords().get(0);
|
||||
assertThat(row.getValueString("status")).isEqualTo(QuickSearchValidationRun.STATUS_FAIL);
|
||||
assertThat(row.getValueString("reportHtml")).contains("Sampled records").contains("Stale fields in " + PRODUCT);
|
||||
assertThat(row.getValueString("reportJson")).contains("\"missingIds\":[\"5\"]");
|
||||
assertThat(row.getValueInteger("failCount")).isPositive();
|
||||
}
|
||||
finally
|
||||
{
|
||||
ops().raw("DELETE", "/" + INDEX + "-v2-1", null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(6)
|
||||
void testReindex_fixesStaleMissingAndOrphan() throws Exception
|
||||
{
|
||||
List<RecordReindexer.Outcome> outcomes = new RecordReindexer().reindex(runtime, PRODUCT, List.of("3", "5", "99", "abc"));
|
||||
assertThat(outcomes).extracting(RecordReindexer.Outcome::outcome)
|
||||
.containsExactly(RecordReindexer.OUTCOME_INDEXED, RecordReindexer.OUTCOME_INDEXED, RecordReindexer.OUTCOME_DELETED, RecordReindexer.OUTCOME_SKIPPED);
|
||||
|
||||
assertThat(new RecordInspector().inspect(runtime, PRODUCT, "3", null, new QueryInput()).verdict()).isEqualTo(RecordInspector.VERDICT_CURRENT);
|
||||
assertThat(new RecordInspector().inspect(runtime, PRODUCT, "5", null, new QueryInput()).verdict()).isEqualTo(RecordInspector.VERDICT_CURRENT);
|
||||
assertThat(ops().getDocument(PRODUCT, "99").found()).isFalse();
|
||||
|
||||
IndexValidator.Report report = new IndexValidator().validate(runtime, List.of(PRODUCT), 50);
|
||||
assertThat(report.findings()).noneMatch(f -> f.level() == Level.FAIL);
|
||||
assertThat(report.status()).isIn(QuickSearchValidationRun.STATUS_PASS, QuickSearchValidationRun.STATUS_WARN);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record actions and the validation as processes, through
|
||||
** RunProcessAction (what the UI calls).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(7)
|
||||
void testProcesses_endToEnd() throws Exception
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// inspect: stops at the form with the document already compared //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
RunProcessInput inspect = new RunProcessInput();
|
||||
inspect.setProcessName(QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, PRODUCT));
|
||||
inspect.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
inspect.addValue("recordIds", "1");
|
||||
RunProcessOutput inspected = new RunProcessAction().execute(inspect);
|
||||
assertThat(inspected.getProcessState().getNextStepName()).hasValue(AbstractDiagnosticToolStep.STEP_FORM);
|
||||
assertThat(inspected.getValues().get("form.html").toString()).contains("Indexed and current");
|
||||
|
||||
RunProcessInput inspectTerm = new RunProcessInput();
|
||||
inspectTerm.setProcessName(inspect.getProcessName());
|
||||
inspectTerm.setProcessUUID(inspected.getProcessUUID());
|
||||
inspectTerm.setStartAfterStep(AbstractDiagnosticToolStep.STEP_FORM);
|
||||
inspectTerm.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
inspectTerm.addValue(InspectRecordStep.FIELD_TERM, "gadget");
|
||||
RunProcessOutput withTerm = new RunProcessAction().execute(inspectTerm);
|
||||
assertThat(withTerm.getProcessState().getNextStepName()).hasValue(AbstractDiagnosticToolStep.STEP_FORM);
|
||||
assertThat(withTerm.getValues().get("form.html").toString()).contains("Not in this record");
|
||||
|
||||
////////////////////////////////////////
|
||||
// reindex: confirm, then the outcome //
|
||||
////////////////////////////////////////
|
||||
RunProcessInput reindex = new RunProcessInput();
|
||||
reindex.setProcessName(QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config, PRODUCT));
|
||||
reindex.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
reindex.addValue("recordIds", "2");
|
||||
RunProcessOutput confirm = new RunProcessAction().execute(reindex);
|
||||
assertThat(confirm.getProcessState().getNextStepName()).hasValue(AbstractDiagnosticToolStep.STEP_CONFIRM);
|
||||
assertThat(confirm.getValues().get("confirm.html").toString()).contains("1 record(s) of " + PRODUCT);
|
||||
|
||||
RunProcessInput reindexRun = new RunProcessInput();
|
||||
reindexRun.setProcessName(reindex.getProcessName());
|
||||
reindexRun.setProcessUUID(confirm.getProcessUUID());
|
||||
reindexRun.setStartAfterStep(AbstractDiagnosticToolStep.STEP_CONFIRM);
|
||||
reindexRun.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
RunProcessOutput reindexed = new RunProcessAction().execute(reindexRun);
|
||||
assertThat(reindexed.getValues().get("result.html").toString()).contains("1 reindexed");
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// validation as the scheduler runs it (frontend skipped) //
|
||||
/////////////////////////////////////////////////////////////
|
||||
RunProcessInput validate = new RunProcessInput();
|
||||
validate.setProcessName(QuickSearchDiagnosticsProcessMetaDataHelper.VALIDATE_PROCESS_NAME);
|
||||
validate.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
|
||||
RunProcessOutput validated = new RunProcessAction().execute(validate);
|
||||
assertThat(validated.getValues().get("validationRunId")).isNotNull();
|
||||
List<QRecord> runs = new QueryAction().execute(new QueryInput(QuickSearchValidationRun.TABLE_NAME)).getRecords();
|
||||
assertThat(runs).anyMatch(r -> QuickSearchValidationRun.TRIGGER_SCHEDULED.equals(r.getValueString("triggerType")));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsWidgetMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The metadata the diagnostic tools add: the two history tables, the fixed
|
||||
** processes on Quick Search Index and Quick Search Failed Event, the
|
||||
** per-table record actions, the widget on the admin app, their permission
|
||||
** rules, scheduling, the opt-out flags, and that the produced instance
|
||||
** passes QInstanceValidator.
|
||||
*******************************************************************************/
|
||||
class DiagnosticsMetaDataTest
|
||||
{
|
||||
private static final String BACKEND = "memory";
|
||||
private static final String SOURCE = "product";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QInstance buildInstance()
|
||||
{
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(SOURCE)
|
||||
.withBackendName(BACKEND)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
QContext.init(qInstance, new QSession());
|
||||
return (qInstance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QuickSearchQBitConfig config()
|
||||
{
|
||||
return (new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("diag-meta")
|
||||
.withSearchableTable(SOURCE, List.of(new SearchableFieldConfig("name"))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_addsTablesProcessesAndWidget() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
new QuickSearchQBitProducer().withConfig(config()).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getTable(QuickSearchDiagnosticEvent.TABLE_NAME)).isNotNull();
|
||||
assertThat(qInstance.getTable(QuickSearchValidationRun.TABLE_NAME)).isNotNull();
|
||||
assertThat(qInstance.getTable(QuickSearchValidationRun.TABLE_NAME).getField("reportHtml").getType()).isEqualTo(QFieldType.HTML);
|
||||
for(String history : List.of(QuickSearchDiagnosticEvent.TABLE_NAME, QuickSearchValidationRun.TABLE_NAME))
|
||||
{
|
||||
assertThat(qInstance.getTable(history).getDisabledCapabilities()).as(history)
|
||||
.contains(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
}
|
||||
|
||||
for(String name : List.of(QuickSearchDiagnosticsProcessMetaDataHelper.PLAYGROUND_PROCESS_NAME, QuickSearchDiagnosticsProcessMetaDataHelper.VALIDATE_PROCESS_NAME,
|
||||
QuickSearchDiagnosticsProcessMetaDataHelper.ANALYZE_PROCESS_NAME))
|
||||
{
|
||||
QProcessMetaData process = qInstance.getProcess(name);
|
||||
assertThat(process).as(name).isNotNull();
|
||||
assertThat(process.getTableName()).isEqualTo(QuickSearchIndex.TABLE_NAME);
|
||||
assertThat(process.getPermissionRules().getLevel()).isEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
|
||||
assertThat(process.getPermissionRules().getPermissionBaseName()).isEqualTo(QuickSearchQBitConfig.DEFAULT_DEBUG_PERMISSION_BASE_NAME);
|
||||
}
|
||||
|
||||
QProcessMetaData inspect = qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config(), SOURCE));
|
||||
QProcessMetaData reindex = qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config(), SOURCE));
|
||||
assertThat(inspect.getTableName()).isEqualTo(SOURCE);
|
||||
assertThat(inspect.getPermissionRules().getPermissionBaseName()).isEqualTo(QuickSearchQBitConfig.DEFAULT_DEBUG_PERMISSION_BASE_NAME);
|
||||
assertThat(inspect.getSourceQBitName()).isNotBlank();
|
||||
assertThat(reindex.getTableName()).isEqualTo(SOURCE);
|
||||
assertThat(reindex.getPermissionRules().getPermissionBaseName()).isEqualTo(QuickSearchQBitConfig.DEFAULT_ADMIN_PERMISSION_BASE_NAME);
|
||||
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config(), QuickSearchFailedEvent.TABLE_NAME)).getTableName())
|
||||
.isEqualTo(QuickSearchFailedEvent.TABLE_NAME);
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config(), QuickSearchFailedEvent.TABLE_NAME))).isNotNull();
|
||||
|
||||
assertThat(qInstance.getWidget(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME)).isNotNull();
|
||||
assertThat(qInstance.getWidget(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME).getPermissionRules().getPermissionBaseName())
|
||||
.isEqualTo(QuickSearchQBitConfig.DEFAULT_DEBUG_PERMISSION_BASE_NAME);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The tools are actions, never nav entries: no diagnostic process is a
|
||||
** child of the admin app; the history tables and the widget are.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testProduce_toolsAreNotInTheAppNav() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
new QuickSearchQBitProducer().withConfig(config()).produce(qInstance);
|
||||
|
||||
QAppMetaData app = qInstance.getApp(QuickSearchQBitProducer.APP_NAME);
|
||||
List<String> children = app.getChildren().stream().map(c -> c.getName()).toList();
|
||||
assertThat(children).contains(QuickSearchDiagnosticEvent.TABLE_NAME, QuickSearchValidationRun.TABLE_NAME);
|
||||
assertThat(children).noneMatch(n -> n.startsWith("quickSearchPlayground") || n.startsWith("quickSearchInspect") || n.startsWith("quickSearchReindexRecords")
|
||||
|| n.startsWith("quickSearchValidateIndex") || n.startsWith("quickSearchAnalyzeText"));
|
||||
assertThat(app.getWidgets()).containsExactly(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_instancePassesValidation() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
new QuickSearchQBitProducer().withConfig(config()).produce(qInstance);
|
||||
new QInstanceValidator().revalidate(qInstance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_diagnosticsOff_addsNothing() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
new QuickSearchQBitProducer().withConfig(config().withEnableDiagnosticTools(false)).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getTable(QuickSearchDiagnosticEvent.TABLE_NAME)).isNull();
|
||||
assertThat(qInstance.getTable(QuickSearchValidationRun.TABLE_NAME)).isNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.PLAYGROUND_PROCESS_NAME)).isNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config(), SOURCE))).isNull();
|
||||
assertThat(qInstance.getWidget(QuickSearchDiagnosticsWidgetMetaDataProducer.WIDGET_NAME)).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_recordActionsOff_keepsFixedTools() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
new QuickSearchQBitProducer().withConfig(config().withEnableRecordDiagnosticActions(false)).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.PLAYGROUND_PROCESS_NAME)).isNotNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config(), SOURCE))).isNull();
|
||||
assertThat(qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config(), SOURCE))).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_validationCron_schedulesTheValidation() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
QSchedulerMetaData scheduler = new QuartzSchedulerMetaData().withName("simple");
|
||||
qInstance.addScheduler(scheduler);
|
||||
new QuickSearchQBitProducer().withConfig(config().withSchedulerName("simple").withValidationCronExpression("0 0 4 * * ?").withValidationCronTimeZoneId("UTC")).produce(qInstance);
|
||||
|
||||
QProcessMetaData validate = qInstance.getProcess(QuickSearchDiagnosticsProcessMetaDataHelper.VALIDATE_PROCESS_NAME);
|
||||
assertThat(validate.getSchedule()).isNotNull();
|
||||
assertThat(validate.getSchedule().getCronExpression()).isEqualTo("0 0 4 * * ?");
|
||||
assertThat(validate.getSchedule().getCronTimeZoneId()).isEqualTo("UTC");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testValidate_validationCronNeedsSchedulerAndTools()
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withValidationCronExpression("0 0 4 * * ?")).produce(qInstance))
|
||||
.hasMessageContaining("validationCronExpression requires schedulerName");
|
||||
|
||||
QInstance second = buildInstance();
|
||||
second.addScheduler(new QuartzSchedulerMetaData().withName("simple"));
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withSchedulerName("simple").withValidationCronExpression("0 0 4 * * ?").withValidationCronTimeZoneId("UTC")
|
||||
.withEnableDiagnosticTools(false))
|
||||
.produce(second))
|
||||
.hasMessageContaining("validationCronExpression requires enableDiagnosticTools");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The validation schedule follows the same rules as the reconcile one: a
|
||||
** cron-capable scheduler, a valid Quartz expression and a time zone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testValidate_validationCronRules()
|
||||
{
|
||||
QInstance simple = buildInstance();
|
||||
simple.addScheduler(new SimpleSchedulerMetaData().withName("simple"));
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withSchedulerName("simple").withValidationCronExpression("0 0 4 * * ?").withValidationCronTimeZoneId("UTC"))
|
||||
.produce(simple))
|
||||
.hasMessageContaining("validationCronExpression requires a scheduler that supports cron schedules");
|
||||
|
||||
QInstance quartz = buildInstance();
|
||||
quartz.addScheduler(new QuartzSchedulerMetaData().withName("quartz"));
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withSchedulerName("quartz").withValidationCronExpression("0 0 4 * *")).produce(quartz))
|
||||
.hasMessageContaining("validationCronExpression [0 0 4 * *] is not a valid Quartz cron expression")
|
||||
.hasMessageContaining("validationCronExpression requires validationCronTimeZoneId");
|
||||
|
||||
QInstance zone = buildInstance();
|
||||
zone.addScheduler(new QuartzSchedulerMetaData().withName("quartz"));
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withSchedulerName("quartz").withValidationCronExpression("0 0 4 * * ?").withValidationCronTimeZoneId("Mars/Base"))
|
||||
.produce(zone))
|
||||
.hasMessageContaining("validationCronTimeZoneId [Mars/Base] is not a recognized time zone ID");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testValidate_sampleSizeAndRateBounds()
|
||||
{
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withValidationSampleSize(0)).produce(buildInstance()))
|
||||
.hasMessageContaining("validationSampleSize must be positive");
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withValidationSampleSize(5000)).produce(buildInstance()))
|
||||
.hasMessageContaining("validationSampleSize must be at most");
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(config().withDiagnosticRunsPerMinute(-1)).produce(buildInstance()))
|
||||
.hasMessageContaining("diagnosticRunsPerMinute must be positive");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_prefixAppliesToToolsAndTables() throws QException
|
||||
{
|
||||
QInstance qInstance = buildInstance();
|
||||
QuickSearchQBitConfig config = config().withTableNamePrefix("myApp_");
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getTable("myApp_" + QuickSearchDiagnosticEvent.TABLE_NAME)).isNotNull();
|
||||
assertThat(qInstance.getTable("myApp_" + QuickSearchValidationRun.TABLE_NAME)).isNotNull();
|
||||
QProcessMetaData playground = qInstance.getProcess("myApp_" + QuickSearchDiagnosticsProcessMetaDataHelper.PLAYGROUND_PROCESS_NAME);
|
||||
assertThat(playground.getTableName()).isEqualTo("myApp_" + QuickSearchIndex.TABLE_NAME);
|
||||
assertThat(qInstance.getProcess("myApp_quickSearchInspect_" + SOURCE)).isNotNull();
|
||||
}
|
||||
|
||||
}
|
||||
+2
-3
@@ -47,7 +47,6 @@ import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
@@ -177,9 +176,9 @@ class FullReindexDeleteReplayIntegrationTest
|
||||
// while the alias still points at the old one //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex, run);
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex);
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
|
||||
-321
@@ -1,321 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Integration test, against a real OpenSearch, that a full reindex killed
|
||||
** right after its alias swap (table left REBUILDING, RUNNING run record, a
|
||||
** captured delete never applied, so the deleted record is back in search) is
|
||||
** recovered by the scheduled basepull once its run record goes stale.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
class FullReindexRecoveryIntegrationTest
|
||||
{
|
||||
private static final String BACKEND_NAME = "reindexRecoveryMemoryBackend";
|
||||
private static final String TABLE_NAME = "recoveryCustomer";
|
||||
private static final String INDEX_NAME = "quick_search_reindex_recovery_test";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Entity annotated for quick search indexing.
|
||||
*******************************************************************************/
|
||||
@QuickSearchable(tableName = TABLE_NAME)
|
||||
static class RecoveryCustomer
|
||||
{
|
||||
@QuickSearchField(weight = 2)
|
||||
private String name;
|
||||
|
||||
@QuickSearchField
|
||||
private String city;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Thrown to stop a full reindex the way a killed JVM does: it is not an
|
||||
** Exception, so none of the step's failure handling runs.
|
||||
*******************************************************************************/
|
||||
static class SimulatedKill extends Error
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the instance, produce the QBit with real-time indexing on and the
|
||||
** record-security re-read off (search returns whatever the index holds),
|
||||
** and initialize QContext.
|
||||
*******************************************************************************/
|
||||
@BeforeAll
|
||||
static void setUpAll() throws Exception
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData()
|
||||
.withName(BACKEND_NAME)
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("city", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost(opensearch.getHost())
|
||||
.withOpensearchPort(opensearch.getMappedPort(9200))
|
||||
.withOpensearchIndexName(INDEX_NAME)
|
||||
.withSearchableEntityClasses(List.of(RecoveryCustomer.class))
|
||||
.withEnableRealTimeIndexing(true)
|
||||
.withApplyRecordSecurityLocks(false);
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clean up QContext and QBit state after all tests complete.
|
||||
*******************************************************************************/
|
||||
@AfterAll
|
||||
static void tearDownAll()
|
||||
{
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a record deleted during a full reindex that is then killed right
|
||||
** after its swap is back in search (the new index holds the copy read
|
||||
** before the delete, and the captured delete was never applied). While the
|
||||
** dead run's record is fresh it blocks a new full reindex; once it is
|
||||
** stale, basepull recovers the table and replays the delete, and the
|
||||
** record is gone from search.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testKilledFullReindex_isRecoveredByBasepull() throws Exception
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("name", "Gwendolyn").withValue("city", "Stalledton"),
|
||||
new QRecord().withValue("id", 2).withValue("name", "Stanford").withValue("city", "Stalledton")));
|
||||
new InsertAction().execute(insertInput);
|
||||
assertThat(searchRecordIds("stalledton")).containsExactlyInAnyOrder("1", "2");
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep()
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// delete a record once its document is in the new physical index //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex, run);
|
||||
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
deleteInput.setTableName(TABLE_NAME);
|
||||
deleteInput.setPrimaryKeys(List.<Serializable>of(1));
|
||||
new DeleteAction().execute(deleteInput);
|
||||
|
||||
return (counts);
|
||||
}
|
||||
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// die right after the swap, before any table is finished //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
@Override
|
||||
protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
|
||||
{
|
||||
if(values.containsKey("lastFullReindexTime"))
|
||||
{
|
||||
throw (new SimulatedKill());
|
||||
}
|
||||
super.updateIndexRow(indexId, values);
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(SimulatedKill.class);
|
||||
|
||||
assertThat(indexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).singleElement()
|
||||
.satisfies(run -> assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_RUNNING));
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).singleElement()
|
||||
.satisfies(row -> assertThat(row.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_AWAITING_REINDEX));
|
||||
assertThat(searchRecordIds("stalledton"))
|
||||
.as("the swapped-in index still holds the deleted record")
|
||||
.containsExactlyInAnyOrder("1", "2");
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.as("the dead run's record is still fresh, so it counts as running")
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("already running");
|
||||
|
||||
backdateRunRecords();
|
||||
new BasepullIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
assertThat(searchRecordIds("stalledton")).containsExactly("2");
|
||||
assertThat(indexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME))
|
||||
.filteredOn(run -> FullReindexStep.RUN_TYPE.equals(run.getValueString("runType")))
|
||||
.singleElement()
|
||||
.satisfies(run ->
|
||||
{
|
||||
assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED);
|
||||
assertThat(run.getValueString("errorMessage")).contains("stopped without finishing");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Move the start and heartbeat of every RUNNING run record two hours into
|
||||
** the past, as if its run had stopped refreshing it.
|
||||
*******************************************************************************/
|
||||
private void backdateRunRecords() throws QException
|
||||
{
|
||||
Instant twoHoursAgo = Instant.now().minus(2, ChronoUnit.HOURS);
|
||||
for(QRecord run : queryAll(QuickSearchIndexRun.TABLE_NAME))
|
||||
{
|
||||
if(AbstractIndexingStep.RUN_RUNNING.equals(run.getValueString("status")))
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndexRun.TABLE_NAME);
|
||||
updateInput.setOmitModifyDateUpdate(true);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", run.getValue("id")).withValue("startTime", twoHoursAgo).withValue("modifyDate", twoHoursAgo)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The table's quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
private QRecord indexRow() throws QException
|
||||
{
|
||||
return (queryAll(QuickSearchIndex.TABLE_NAME).stream()
|
||||
.filter(row -> TABLE_NAME.equals(row.getValueString("tableName")))
|
||||
.findFirst().orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All rows of a table.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> queryAll(String tableName) throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
return (new QueryAction().execute(queryInput).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the record IDs search returns for the term.
|
||||
*******************************************************************************/
|
||||
private List<String> searchRecordIds(String term) throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getClient().refreshIndex();
|
||||
return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME))
|
||||
.getResults().stream()
|
||||
.map(QuickSearchResult::getRecordId)
|
||||
.toList());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -65,9 +65,8 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Integration tests, against a real OpenSearch, that real-time indexing keeps
|
||||
** the index in step with the source table through inserts (with display
|
||||
** values), updates and deletes, and
|
||||
** that the reconcile process repairs an index that has drifted.
|
||||
** the index in step with the source table through inserts, updates and
|
||||
** deletes, and that the reconcile process repairs an index that has drifted.
|
||||
**
|
||||
** Each test uses its own record IDs and search terms, so tests are independent.
|
||||
*******************************************************************************/
|
||||
@@ -77,7 +76,7 @@ class IndexDriftIntegrationTest
|
||||
{
|
||||
private static final String BACKEND_NAME = "driftMemoryBackend";
|
||||
private static final String TABLE_NAME = "driftCustomer";
|
||||
private static final String REGION_TABLE = "driftRegion";
|
||||
private static final String COMPANY_NAME = "driftCompany";
|
||||
private static final String INDEX_NAME = "quick_search_drift_test";
|
||||
|
||||
@Container
|
||||
@@ -103,7 +102,7 @@ class IndexDriftIntegrationTest
|
||||
private String city;
|
||||
|
||||
@QuickSearchField
|
||||
private Integer regionId;
|
||||
private Integer companyId;
|
||||
}
|
||||
|
||||
|
||||
@@ -124,25 +123,23 @@ class IndexDriftIntegrationTest
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous")
|
||||
.withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(COMPANY_NAME)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormatAndFields("%s", "name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(COMPANY_NAME));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormatAndFields("%s (%s)", "name", "companyId")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("city", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("regionId", QFieldType.INTEGER).withPossibleValueSourceName(REGION_TABLE))
|
||||
.withRecordLabelFormat("%s - %s")
|
||||
.withRecordLabelFields("name", "regionId"));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(REGION_TABLE)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s")
|
||||
.withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(REGION_TABLE));
|
||||
.withField(new QFieldMetaData("companyId", QFieldType.INTEGER).withPossibleValueSourceName(COMPANY_NAME)));
|
||||
|
||||
QuickSearchQBitConfig config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND_NAME)
|
||||
@@ -173,28 +170,22 @@ class IndexDriftIntegrationTest
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a just-inserted record's document carries display values: the
|
||||
** possible-value label of a foreign key is searchable and is in the record
|
||||
** label, with no basepull in between (#28).
|
||||
** Test: a just-inserted record is indexed with its possible-value labels, in
|
||||
** its searchable text and its record label, not with the raw foreign key.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testInsert_documentCarriesDisplayValuesAndPossibleValueLabel() throws Exception
|
||||
void testInsert_indexesPossibleValueLabels() throws Exception
|
||||
{
|
||||
InsertInput regionInsert = new InsertInput();
|
||||
regionInsert.setTableName(REGION_TABLE);
|
||||
regionInsert.setRecords(List.of(new QRecord().withValue("id", 7).withValue("name", "Northwind Territory")));
|
||||
new InsertAction().execute(regionInsert);
|
||||
InsertInput companyInsert = new InsertInput();
|
||||
companyInsert.setTableName(COMPANY_NAME);
|
||||
companyInsert.setRecords(List.of(new QRecord().withValue("id", 1).withValue("name", "Rocketworks")));
|
||||
new InsertAction().execute(companyInsert);
|
||||
|
||||
insert(new QRecord().withValue("id", 31).withValue("name", "Rhoda").withValue("city", "Labelburg").withValue("regionId", 7));
|
||||
insert(new QRecord().withValue("id", 31).withValue("name", "Cyrus").withValue("city", "Labelton").withValue("companyId", 1));
|
||||
|
||||
client().refreshIndex();
|
||||
List<QuickSearchResult> results = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("northwind").withTableName(TABLE_NAME)).getResults();
|
||||
assertThat(results).as("the region's label, not its key, is indexed").extracting(QuickSearchResult::getRecordId).containsExactly("31");
|
||||
assertThat(results.get(0).getRecordLabel()).isEqualTo("Rhoda - Northwind Territory");
|
||||
|
||||
Map<String, Object> fieldValues = client().search("labelburg", List.of(TABLE_NAME), 10, 0, QuickSearchQBitContext.getDiscoveredTables())
|
||||
.hits().hits().get(0).source().getFieldValues();
|
||||
assertThat(fieldValues).containsEntry("regionId", "Northwind Territory");
|
||||
List<QuickSearchResult> results = search("rocketworks");
|
||||
assertThat(results).extracting(QuickSearchResult::getRecordId).containsExactly("31");
|
||||
assertThat(results.get(0).getRecordLabel()).isEqualTo("Cyrus (Rocketworks)");
|
||||
}
|
||||
|
||||
|
||||
@@ -313,15 +304,24 @@ class IndexDriftIntegrationTest
|
||||
*******************************************************************************/
|
||||
private List<String> searchRecordIds(String term) throws QException
|
||||
{
|
||||
client().refreshIndex();
|
||||
return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME))
|
||||
.getResults().stream()
|
||||
return (search(term).stream()
|
||||
.map(QuickSearchResult::getRecordId)
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the search results matching the term.
|
||||
*******************************************************************************/
|
||||
private List<QuickSearchResult> search(String term) throws QException
|
||||
{
|
||||
client().refreshIndex();
|
||||
return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME)).getResults());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh the index, then return the record IDs of the documents matching
|
||||
** the term, straight from OpenSearch (no record-security post-filter).
|
||||
|
||||
+3
-86
@@ -23,7 +23,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
@@ -247,18 +246,18 @@ class QuickSearchQBitConfigValidationTest
|
||||
void testScheduling_schedulerMustExist_andCronNeedsScheduler()
|
||||
{
|
||||
QInstance qInstance = instanceWithBackendAndCustomer();
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName("sched"));
|
||||
qInstance.addScheduler(new SimpleSchedulerMetaData().withName("sched"));
|
||||
|
||||
List<String> errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("missing").validate(qInstance, errors);
|
||||
assertThat(errors).anyMatch(e -> e.contains("Scheduler not found: missing"));
|
||||
|
||||
errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId("America/Chicago").validate(qInstance, errors);
|
||||
baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").validate(qInstance, errors);
|
||||
assertThat(errors).isEmpty();
|
||||
|
||||
errors = new ArrayList<>();
|
||||
baseConfig().withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId("UTC").validate(qInstance, errors);
|
||||
baseConfig().withReconcileCronExpression("0 0 3 * * ?").validate(qInstance, errors);
|
||||
assertThat(errors).anyMatch(e -> e.contains("requires schedulerName"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
@@ -269,72 +268,6 @@ class QuickSearchQBitConfigValidationTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ's instance validator rejects a cron schedule without a time zone and
|
||||
** has no default, so the QBit rejects it at produce time, naming its fields.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testScheduling_cronWithoutTimeZone_addsError()
|
||||
{
|
||||
QInstance qInstance = instanceWithBackendAndCustomer();
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName("sched"));
|
||||
|
||||
List<String> errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").validate(qInstance, errors);
|
||||
assertThat(errors).containsExactly("reconcileCronExpression requires reconcileCronTimeZoneId (a time zone ID such as UTC or America/Chicago)");
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// without a scheduler in the instance the rule still applies //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
assertThat(validate(baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId(" ")))
|
||||
.anyMatch(e -> e.startsWith("reconcileCronExpression requires reconcileCronTimeZoneId"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The cron expression and the zone id are checked the way QQQ's instance
|
||||
** validator checks them: Quartz cron syntax and a known TimeZone id.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testScheduling_invalidCronOrTimeZone_addsErrors()
|
||||
{
|
||||
List<String> errors = validate(baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * *").withReconcileCronTimeZoneId("UTC"));
|
||||
assertThat(errors).singleElement().satisfies(e -> assertThat(e).startsWith("reconcileCronExpression [0 0 3 * *] is not a valid Quartz cron expression"));
|
||||
|
||||
errors = validate(baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId("Mars/Olympus_Mons"));
|
||||
assertThat(errors).containsExactly("reconcileCronTimeZoneId [Mars/Olympus_Mons] is not a recognized time zone ID");
|
||||
|
||||
errors = validate(baseConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId("UTC"));
|
||||
assertThat(errors).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ's simple scheduler only runs fixed intervals; a cron schedule on it
|
||||
** fails when the schedule manager starts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testScheduling_cronOnSchedulerWithoutCronSupport_addsError()
|
||||
{
|
||||
QInstance qInstance = instanceWithBackendAndCustomer();
|
||||
qInstance.addScheduler(new SimpleSchedulerMetaData().withName("simple"));
|
||||
|
||||
List<String> errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("simple").withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId("UTC").validate(qInstance, errors);
|
||||
assertThat(errors).containsExactly("reconcileCronExpression requires a scheduler that supports cron schedules, such as a QuartzSchedulerMetaData; scheduler [simple] is of type simple");
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// basepull's fixed interval runs on either //
|
||||
////////////////////////////////////////////////
|
||||
errors = new ArrayList<>();
|
||||
baseConfig().withSchedulerName("simple").validate(qInstance, errors);
|
||||
assertThat(errors).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testNumbers_mustBePositive()
|
||||
{
|
||||
@@ -343,37 +276,22 @@ class QuickSearchQBitConfigValidationTest
|
||||
.withResponseTimeoutMillis(-1)
|
||||
.withMaxConnections(0)
|
||||
.withMaxSearchLimit(0)
|
||||
.withRecordSearchTimeoutMillis(0)
|
||||
.withBasepullRepeatSeconds(0)
|
||||
.withBasepullOverlapSeconds(-5)
|
||||
.withRunHistoryRetentionDays(0)
|
||||
.withFullReindexStaleMinutes(0)
|
||||
.withMaxFieldLength(0));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("connectTimeoutMillis"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("responseTimeoutMillis"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("maxConnections"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("maxSearchLimit"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("recordSearchTimeoutMillis"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("basepullRepeatSeconds"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("basepullOverlapSeconds"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("runHistoryRetentionDays"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("fullReindexStaleMinutes"));
|
||||
assertThat(errors).anyMatch(e -> e.startsWith("maxFieldLength"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testFullReindexStaleMinutes_nullIsValid_andMeansTheDefault()
|
||||
{
|
||||
QuickSearchQBitConfig config = baseConfig().withFullReindexStaleMinutes(null);
|
||||
assertThat(validate(config)).isEmpty();
|
||||
assertThat(config.getEffectiveFullReindexStaleMinutes()).isEqualTo(QuickSearchQBitConfig.DEFAULT_FULL_REINDEX_STALE_MINUTES);
|
||||
assertThat(config.withFullReindexStaleMinutes(5).getEffectiveFullReindexStaleMinutes()).isEqualTo(5);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testValidate_nullInstance_isSafe()
|
||||
{
|
||||
@@ -419,7 +337,6 @@ class QuickSearchQBitConfigValidationTest
|
||||
assertThat(config.getResponseTimeoutMillis()).isEqualTo(60000);
|
||||
assertThat(config.getMaxSearchLimit()).isEqualTo(100);
|
||||
assertThat(config.getBasepullOverlapSeconds()).isEqualTo(300);
|
||||
assertThat(config.getFullReindexStaleMinutes()).isEqualTo(30);
|
||||
assertThat(config.getAdminPermissionRules().getPermissionBaseName()).isEqualTo(QuickSearchQBitConfig.DEFAULT_ADMIN_PERMISSION_BASE_NAME);
|
||||
assertThat(config.getApplyRecordSecurityLocks()).isTrue();
|
||||
assertThat(config.getDefaultBackendNameForTables()).isNull();
|
||||
|
||||
@@ -16,18 +16,10 @@
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.CollectedLogMessage;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
@@ -42,11 +34,8 @@ import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField;
|
||||
import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -110,43 +99,6 @@ class QuickSearchQBitProducerTest
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Test entity with fields core search must not get: hidden, password and
|
||||
** non-text, plus an integer field it may get.
|
||||
***************************************************************************/
|
||||
@QuickSearchable(tableName = "account")
|
||||
static class AccountEntity
|
||||
{
|
||||
@QuickSearchField
|
||||
private String name;
|
||||
|
||||
@QuickSearchField
|
||||
private Integer accountNumber;
|
||||
|
||||
@QuickSearchField
|
||||
private String secretNote;
|
||||
|
||||
@QuickSearchField
|
||||
private String password;
|
||||
|
||||
@QuickSearchField
|
||||
private Instant openedAt;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Test entity whose only searchable field is a date.
|
||||
***************************************************************************/
|
||||
@QuickSearchable(tableName = "event")
|
||||
static class EventEntity
|
||||
{
|
||||
@QuickSearchField
|
||||
private Instant occurredAt;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@QuickSearchable(tableName = "person")
|
||||
static class BadFieldEntity
|
||||
{
|
||||
@@ -156,18 +108,6 @@ class QuickSearchQBitProducerTest
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Annotated entity on a table without the default modifyDate field.
|
||||
***************************************************************************/
|
||||
@QuickSearchable(tableName = "tag")
|
||||
static class TagEntity
|
||||
{
|
||||
@QuickSearchField
|
||||
private String body;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
@@ -349,141 +289,6 @@ class QuickSearchQBitProducerTest
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** By default the producer registers the core record-search provider and
|
||||
** gives indexed tables core searchFields from their searchable fields.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_registersRecordSearchProviderAndSearchFields() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig()).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getRecordSearchProvider()).isNotNull();
|
||||
assertThat(qInstance.getRecordSearchProvider().getName()).isEqualTo(QuickSearchRecordSearchProvider.class.getName());
|
||||
assertThat(qInstance.getTable(SOURCE_TABLE).getSearchFields()).containsExactly("firstName", "lastName", "email");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** serveCoreRecordSearch false leaves core search alone.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_serveCoreRecordSearchOff_noProviderNoSearchFields() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withServeCoreRecordSearch(false)).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getRecordSearchProvider()).isNull();
|
||||
assertThat(qInstance.getTable(SOURCE_TABLE).getSearchFields()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** searchFields the host set are kept, and a provider the host registered
|
||||
** is not replaced.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_keepsHostSearchFieldsAndProvider() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
qInstance.getTable(SOURCE_TABLE).withSearchFields("email");
|
||||
QCodeReference hostProvider = new QCodeReference(HostRecordSearchProvider.class);
|
||||
qInstance.withRecordSearchProvider(hostProvider);
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig()).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getTable(SOURCE_TABLE).getSearchFields()).containsExactly("email");
|
||||
assertThat(qInstance.getRecordSearchProvider()).isSameAs(hostProvider);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Producing twice (same provider already registered) keeps one provider.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testRegisterRecordSearchProvider_idempotent()
|
||||
{
|
||||
QInstance qInstance = new QInstance();
|
||||
QuickSearchQBitProducer.registerRecordSearchProvider(qInstance);
|
||||
QuickSearchQBitProducer.registerRecordSearchProvider(qInstance);
|
||||
assertThat(qInstance.getRecordSearchProvider().getName()).isEqualTo(QuickSearchRecordSearchProvider.class.getName());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Hidden and password fields are never indexed; core searchFields only get
|
||||
** visible string or integer fields; a table with none gets no searchFields.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_searchFieldsAndIndexing_skipHiddenPasswordAndNonText() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName("account")
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("accountNumber", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("secretNote", QFieldType.STRING).withIsHidden(true))
|
||||
.withField(new QFieldMetaData("password", QFieldType.PASSWORD))
|
||||
.withField(new QFieldMetaData("openedAt", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName("event")
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("occurredAt", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = buildValidConfig().withSearchableEntityClasses(List.of(AccountEntity.class, EventEntity.class));
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
|
||||
QuickSearchableTableConfig account = config.getRuntime().getTableConfig("account");
|
||||
assertThat(account.getSearchableFields()).containsExactly("name", "accountNumber", "openedAt");
|
||||
assertThat(qInstance.getTable("account").getSearchFields()).containsExactly("name", "accountNumber");
|
||||
assertThat(qInstance.getTable("event").getSearchFields()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** A config-driven table that is not in the instance yet, or whose backend
|
||||
** uses variants, gets no searchFields (and production does not fail).
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testDefaultCoreSearchFields_missingTableAndVariantBackend()
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName("variants").withBackendType(MemoryBackendModule.class).withUsesVariants(true));
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName("variantTable")
|
||||
.withBackendName("variants")
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
|
||||
QuickSearchQBitProducer.defaultCoreSearchFields(qInstance, List.of(
|
||||
new QuickSearchableTableConfig().withTableName("notThere").withSearchableFields(List.of("name")),
|
||||
new QuickSearchableTableConfig().withTableName("variantTable").withSearchableFields(List.of("name")),
|
||||
new QuickSearchableTableConfig().withTableName(SOURCE_TABLE).withSearchableFields(List.of("lastName", "nope", "lastName"))));
|
||||
|
||||
assertThat(qInstance.getTable("variantTable").getSearchFields()).isNull();
|
||||
assertThat(qInstance.getTable(SOURCE_TABLE).getSearchFields()).containsExactly("lastName");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_realTimeIndexingDisabled_noListener() throws QException
|
||||
{
|
||||
@@ -724,37 +529,18 @@ class QuickSearchQBitProducerTest
|
||||
void testProduce_schedulesWhenSchedulerConfigured() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName("sched"));
|
||||
qInstance.addScheduler(new SimpleSchedulerMetaData().withName("sched"));
|
||||
QContext.init(qInstance, new QSession());
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withSchedulerName("sched").withBasepullRepeatSeconds(120).withReconcileCronExpression("0 0 3 * * ?").withReconcileCronTimeZoneId("America/Chicago")).produce(qInstance);
|
||||
new QuickSearchQBitProducer().withConfig(buildValidConfig().withSchedulerName("sched").withBasepullRepeatSeconds(120).withReconcileCronExpression("0 0 3 * * ?")).produce(qInstance);
|
||||
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME).getSchedule().getSchedulerName()).isEqualTo("sched");
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME).getSchedule().getRepeatSeconds()).isEqualTo(120);
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.RECONCILE_PROCESS_NAME).getSchedule().getCronExpression()).isEqualTo("0 0 3 * * ?");
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.RECONCILE_PROCESS_NAME).getSchedule().getCronTimeZoneId()).isEqualTo("America/Chicago");
|
||||
assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME).getSchedule()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A reconcile cron without a time zone used to pass produce() and then fail
|
||||
** QQQ's instance validation at startup; now produce() rejects it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testProduce_cronWithoutTimeZone_failsProduce()
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName("sched"));
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
assertThatThrownBy(() -> new QuickSearchQBitProducer().withConfig(buildValidConfig().withSchedulerName("sched").withReconcileCronExpression("0 0 3 * * ?")).produce(qInstance))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("reconcileCronExpression requires reconcileCronTimeZoneId");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProduce_noScheduler_processesUnscheduled() throws QException
|
||||
{
|
||||
@@ -1067,67 +853,6 @@ class QuickSearchQBitProducerTest
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** A table without its basepull timestamp field gets the same startup
|
||||
** warning, once, whether it is annotated or config-driven; a config-driven
|
||||
** table that sets the field to null has opted out and is not warned about.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void testProduce_missingTimestampField_warnsOncePerTableForBothSources() throws QException
|
||||
{
|
||||
QInstance qInstance = buildQInstance();
|
||||
addTableWithoutTimestamp(qInstance, "tag");
|
||||
addTableWithoutTimestamp(qInstance, "note");
|
||||
addTableWithoutTimestamp(qInstance, "memo");
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
QuickSearchQBitConfig config = buildValidConfig()
|
||||
.withSearchableEntityClasses(List.of(TagEntity.class))
|
||||
.withSearchableTable("note", List.of(new SearchableFieldConfig("body")))
|
||||
.withSearchableTable(new SearchableTableConfig("memo", List.of(new SearchableFieldConfig("body"))).withBasepullTimestampField(null));
|
||||
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchQBitProducer.class);
|
||||
try
|
||||
{
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(QuickSearchQBitProducer.class);
|
||||
}
|
||||
|
||||
List<JSONObject> warnings = new ArrayList<>();
|
||||
for(CollectedLogMessage message : collectingLogger.getCollectedMessages())
|
||||
{
|
||||
if(Level.WARN.equals(message.getLevel()) && message.getMessage().contains("incremental basepull is disabled"))
|
||||
{
|
||||
warnings.add(message.getMessageAsJSONObject());
|
||||
}
|
||||
}
|
||||
|
||||
assertThat(warnings).extracting(w -> w.getString("tableName")).containsExactly("tag", "note");
|
||||
assertThat(warnings).extracting(w -> w.getString("timestampField")).containsOnly("modifyDate");
|
||||
assertThat(warnings).extracting(w -> w.getString("message")).containsOnly(warnings.get(0).getString("message"));
|
||||
assertThat(QuickSearchRuntime.get().getDiscoveredTables()).allSatisfy(t -> assertThat(t.getBasepullTimestampField()).isNull());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Add a table with an id and a body field, but no modifyDate.
|
||||
***************************************************************************/
|
||||
private void addTableWithoutTimestamp(QInstance qInstance, String tableName)
|
||||
{
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(tableName)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("body", QFieldType.STRING)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Build a base QInstance with a memory backend and source tables.
|
||||
***************************************************************************/
|
||||
@@ -1189,26 +914,4 @@ class QuickSearchQBitProducerTest
|
||||
.withSearchableEntityClasses(List.of(PersonEntity.class));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** A record search provider a host registered itself.
|
||||
***************************************************************************/
|
||||
public static class HostRecordSearchProvider implements RecordSearchProviderInterface
|
||||
{
|
||||
@Override
|
||||
public boolean claimsTable(QTableMetaData table)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public RecordSearchOutput search(RecordSearchInput input)
|
||||
{
|
||||
return (new RecordSearchOutput());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-281
@@ -1,281 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchResult;
|
||||
import com.kingsrook.qbits.quicksearch.processes.FullReindexStep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** End-to-end test of core record search (RecordSearchAction, the code behind
|
||||
** POST /qqq/v1/search) served by QuickSearchRecordSearchProvider from a real
|
||||
** OpenSearch (Testcontainers): the producer registers the provider and
|
||||
** searchFields, hits come back in Quick Search's relevance order and are
|
||||
** re-read by primary key (stale ids drop out, limitPerTable applies),
|
||||
** unindexed tables keep core's own search, and with OpenSearch stopped core
|
||||
** falls back to its own search without failing.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class RecordSearchProviderIntegrationTest
|
||||
{
|
||||
private static final String BACKEND_NAME = "testMemoryBackend";
|
||||
private static final String PRODUCT_TABLE = "rsProduct";
|
||||
private static final String NOTE_TABLE = "rsNote";
|
||||
private static final String INDEX_NAME = "quick_search_record_search_it";
|
||||
|
||||
@Container
|
||||
static GenericContainer<?> opensearch = new GenericContainer<>(System.getProperty("opensearch.test.image", "opensearchproject/opensearch:2.19.6"))
|
||||
.withExposedPorts(9200)
|
||||
.withEnv("discovery.type", "single-node")
|
||||
.withEnv("DISABLE_SECURITY_PLUGIN", "true")
|
||||
.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
|
||||
.waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200));
|
||||
|
||||
private static QuickSearchQBitConfig config;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the instance (an indexed product table and an unindexed note
|
||||
** table with its own searchFields), produce the QBit, index the products.
|
||||
*******************************************************************************/
|
||||
@BeforeAll
|
||||
static void setUpAll() throws Exception
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName(BACKEND_NAME).withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(PRODUCT_TABLE)
|
||||
.withLabel("Product")
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s")
|
||||
.withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("description", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(NOTE_TABLE)
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("text", QFieldType.STRING))
|
||||
.withSearchFields("text"));
|
||||
|
||||
config = new QuickSearchQBitConfig()
|
||||
.withBackendName(BACKEND_NAME)
|
||||
.withOpensearchHost(opensearch.getHost())
|
||||
.withOpensearchPort(opensearch.getMappedPort(9200))
|
||||
.withOpensearchIndexName(INDEX_NAME)
|
||||
.withSearchableTable(new SearchableTableConfig(PRODUCT_TABLE, List.of(
|
||||
new SearchableFieldConfig("name").withWeight(3),
|
||||
new SearchableFieldConfig("description"))))
|
||||
.withEnableRealTimeIndexing(false);
|
||||
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
new InsertAction().execute(new InsertInput(PRODUCT_TABLE).withRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("name", "Blue Widget").withValue("description", "the original").withValue("modifyDate", Instant.now()),
|
||||
new QRecord().withValue("id", 2).withValue("name", "Widget").withValue("description", "comes in blue and green").withValue("modifyDate", Instant.now()),
|
||||
new QRecord().withValue("id", 3).withValue("name", "Red Widget").withValue("description", "not that colour").withValue("modifyDate", Instant.now()),
|
||||
new QRecord().withValue("id", 4).withValue("name", "Gadget").withValue("description", "no match here").withValue("modifyDate", Instant.now()))));
|
||||
|
||||
new InsertAction().execute(new InsertInput(NOTE_TABLE).withRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("text", "a blue widget note"))));
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
config.getRuntime().getClient().refreshIndex();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterAll
|
||||
static void tearDownAll()
|
||||
{
|
||||
if(config != null && config.getRuntime() != null)
|
||||
{
|
||||
config.getRuntime().close();
|
||||
}
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static RecordSearchOutput coreSearch(String term, Integer limitPerTable) throws Exception
|
||||
{
|
||||
return (new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm(term).withLimitPerTable(limitPerTable)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<Serializable> idsFor(RecordSearchOutput output, String tableName)
|
||||
{
|
||||
return (output.getResults().stream().filter(r -> tableName.equals(r.getTableName())).map(RecordSearchResult::getRecordId).toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(1)
|
||||
void testProducerRegisteredProviderAndSearchFields()
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
assertThat(qInstance.getRecordSearchProvider().getName()).isEqualTo(QuickSearchRecordSearchProvider.class.getName());
|
||||
assertThat(qInstance.getTable(PRODUCT_TABLE).getSearchFields()).containsExactly("name", "description");
|
||||
assertThat(qInstance.getTable(NOTE_TABLE).getSearchFields()).containsExactly("text");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** "blue widget" matches products 1 and 2 in OpenSearch (all terms, any
|
||||
** field); core's contains-search would only find product 1. The order is
|
||||
** Quick Search's relevance order; labels come from core's re-read.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(2)
|
||||
void testCoreSearchServedFromOpenSearch() throws Exception
|
||||
{
|
||||
RecordSearchOutput output = coreSearch("blue widget", 5);
|
||||
|
||||
List<String> quickSearchOrder = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("blue widget").withTableNames(List.of(PRODUCT_TABLE)))
|
||||
.getResults().stream().map(QuickSearchResult::getRecordId).toList();
|
||||
assertThat(quickSearchOrder).containsExactlyInAnyOrder("1", "2");
|
||||
|
||||
assertThat(idsFor(output, PRODUCT_TABLE)).containsExactlyElementsOf(quickSearchOrder.stream().map(Integer::valueOf).toList());
|
||||
assertThat(output.getResults()).filteredOn(r -> PRODUCT_TABLE.equals(r.getTableName())).extracting(RecordSearchResult::getRecordLabel).contains("Blue Widget", "Widget");
|
||||
assertThat(output.getResults()).filteredOn(r -> PRODUCT_TABLE.equals(r.getTableName())).extracting(RecordSearchResult::getTableLabel).containsOnly("Product");
|
||||
|
||||
////////////////////////////////////////////////
|
||||
// the unindexed table keeps core's own search //
|
||||
////////////////////////////////////////////////
|
||||
assertThat(idsFor(output, NOTE_TABLE)).containsExactly(1);
|
||||
assertThat(config.getRuntime().isRecordSearchBackingOff()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
@Order(3)
|
||||
void testLimitPerTable() throws Exception
|
||||
{
|
||||
assertThat(idsFor(coreSearch("widget", 1), PRODUCT_TABLE)).hasSize(1);
|
||||
assertThat(idsFor(coreSearch("widget", 5), PRODUCT_TABLE)).containsExactlyInAnyOrder(1, 2, 3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A record deleted from the table but still in the index is dropped by
|
||||
** core's re-read.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(4)
|
||||
void testStaleIndexHitDropped() throws Exception
|
||||
{
|
||||
new DeleteAction().execute(new DeleteInput(PRODUCT_TABLE).withPrimaryKeys(List.of(1)));
|
||||
|
||||
assertThat(config.getRuntime().getClient().countDocumentsForTable(PRODUCT_TABLE)).as("the deleted record is still indexed").isEqualTo(4L);
|
||||
|
||||
assertThat(idsFor(coreSearch("blue widget", 5), PRODUCT_TABLE)).containsExactly(2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** With OpenSearch stopped, core search still answers, from core's own
|
||||
** contains-search over the searchFields Quick Search set, and the
|
||||
** provider backs off.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Order(5)
|
||||
void testFallbackWhenOpenSearchStopped() throws Exception
|
||||
{
|
||||
opensearch.stop();
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
RecordSearchOutput output = coreSearch("widget", 5);
|
||||
assertThat(System.currentTimeMillis() - start).isLessThan(15_000);
|
||||
|
||||
assertThat(idsFor(output, PRODUCT_TABLE)).containsExactly(2, 3);
|
||||
assertThat(config.getRuntime().isRecordSearchBackingOff()).isTrue();
|
||||
|
||||
assertThat(idsFor(coreSearch("blue", 5), PRODUCT_TABLE)).containsExactly(2);
|
||||
}
|
||||
|
||||
}
|
||||
-360
@@ -1,360 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.actions;
|
||||
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.ServerSocket;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.tuple;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit tests for QuickSearchRecordSearchProvider: claim logic, result
|
||||
** mapping, order and per-table id cap, failure and timeout handling, and
|
||||
** core RecordSearchAction end to end (re-read by primary key, fallback to
|
||||
** core's own search) over the memory backend with a mock OpenSearch client.
|
||||
*******************************************************************************/
|
||||
class QuickSearchRecordSearchProviderTest extends BaseQuickSearchTest
|
||||
{
|
||||
private static final String OTHER_TABLE = "otherEntity";
|
||||
private static final String UNINDEXED_TABLE = "unindexedEntity";
|
||||
|
||||
private QuickSearchQBitConfig config;
|
||||
private QuickSearchRuntime runtime;
|
||||
private QuickSearchOpenSearchClient mockClient;
|
||||
private QuickSearchRecordSearchProvider provider;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register a Quick Search QBit whose runtime returns a mock client, with
|
||||
** testEntity and otherEntity indexed; add the provider to the instance.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws QException
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
qInstance.addTable(simpleTable(OTHER_TABLE));
|
||||
qInstance.addTable(simpleTable(UNINDEXED_TABLE));
|
||||
qInstance.getTable(TEST_ENTITY_TABLE).withSearchFields("name", "description");
|
||||
qInstance.withRecordSearchProvider(new QCodeReference(QuickSearchRecordSearchProvider.class));
|
||||
|
||||
config = new QuickSearchQBitConfig()
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withOpensearchHost("localhost")
|
||||
.withOpensearchPort(9200)
|
||||
.withOpensearchIndexName("test-index")
|
||||
.withRecordSearchTimeoutMillis(1234);
|
||||
|
||||
mockClient = mock(QuickSearchOpenSearchClient.class);
|
||||
runtime = registerRuntime(config, mockClient);
|
||||
provider = new QuickSearchRecordSearchProvider();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register the QBit with a runtime (a spy whose client is the given one,
|
||||
** or the real lazily built client when null).
|
||||
*******************************************************************************/
|
||||
private QuickSearchRuntime registerRuntime(QuickSearchQBitConfig config, QuickSearchOpenSearchClient client) throws QException
|
||||
{
|
||||
QuickSearchRuntime runtime = new QuickSearchRuntime(config);
|
||||
if(client != null)
|
||||
{
|
||||
runtime = spy(runtime);
|
||||
doReturn(client).when(runtime).getClient();
|
||||
}
|
||||
runtime.setDiscoveredTables(List.of(
|
||||
new QuickSearchableTableConfig().withTableName(TEST_ENTITY_TABLE).withPrimaryKeyField("id").withSearchableFields(List.of("name", "description")),
|
||||
new QuickSearchableTableConfig().withTableName(OTHER_TABLE).withPrimaryKeyField("id").withSearchableFields(List.of("name"))));
|
||||
config.setRuntime(runtime);
|
||||
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
if(qInstance.getQBits() != null)
|
||||
{
|
||||
qInstance.getQBits().clear();
|
||||
}
|
||||
qInstance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("quick-search").withVersion("test").withConfig(config));
|
||||
return (runtime);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData simpleTable(String name)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(name)
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withSearchFields("name"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert testEntity rows with ids 1..n named "<prefix> <i>".
|
||||
*******************************************************************************/
|
||||
private void insertEntities(String prefix, int count) throws QException
|
||||
{
|
||||
List<QRecord> records = new ArrayList<>();
|
||||
for(int i = 1; i <= count; i++)
|
||||
{
|
||||
records.add(new QRecord().withValue("id", i).withValue("name", prefix + " " + i).withValue("description", "row " + i));
|
||||
}
|
||||
new InsertAction().execute(new InsertInput(TEST_ENTITY_TABLE).withRecords(records));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testClaimsTable_indexedEnabledTablesOnly()
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
assertThat(provider.claimsTable(qInstance.getTable(TEST_ENTITY_TABLE))).isTrue();
|
||||
assertThat(provider.claimsTable(qInstance.getTable(OTHER_TABLE))).isTrue();
|
||||
assertThat(provider.claimsTable(qInstance.getTable(UNINDEXED_TABLE))).isFalse();
|
||||
assertThat(provider.claimsTable(null)).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testClaimsTable_disabledTable_notClaimed() throws QException
|
||||
{
|
||||
new InsertAction().execute(new InsertInput(config.getQuickSearchIndexTableName()).withRecords(List.of(
|
||||
new QRecord().withValue("tableName", TEST_ENTITY_TABLE).withValue("enabled", false))));
|
||||
runtime.invalidateEnabledCache();
|
||||
|
||||
assertThat(provider.claimsTable(QContext.getQInstance().getTable(TEST_ENTITY_TABLE))).isFalse();
|
||||
assertThat(provider.claimsTable(QContext.getQInstance().getTable(OTHER_TABLE))).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testClaimsTable_serveCoreRecordSearchOff_orBackingOff_orNoRuntime()
|
||||
{
|
||||
QTableMetaData table = QContext.getQInstance().getTable(TEST_ENTITY_TABLE);
|
||||
|
||||
config.setServeCoreRecordSearch(false);
|
||||
assertThat(provider.claimsTable(table)).isFalse();
|
||||
config.setServeCoreRecordSearch(true);
|
||||
|
||||
runtime.markRecordSearchFailed();
|
||||
assertThat(runtime.isRecordSearchBackingOff()).isTrue();
|
||||
assertThat(provider.claimsTable(table)).isFalse();
|
||||
runtime.clearRecordSearchBackoff();
|
||||
assertThat(provider.claimsTable(table)).isTrue();
|
||||
|
||||
QContext.getQInstance().getQBits().clear();
|
||||
assertThat(provider.claimsTable(table)).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testSearch_mapsIdsPerTableInOrder_andCapsIdsPerTable() throws QException
|
||||
{
|
||||
Map<String, List<String>> ids = new LinkedHashMap<>();
|
||||
ids.put(OTHER_TABLE, List.of("7"));
|
||||
ids.put(TEST_ENTITY_TABLE, List.of("3", "1", "2"));
|
||||
when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenReturn(ids);
|
||||
|
||||
RecordSearchOutput output = provider.search(new RecordSearchInput()
|
||||
.withSearchTerm(" widget blue ")
|
||||
.withTableNames(List.of(TEST_ENTITY_TABLE, UNINDEXED_TABLE, OTHER_TABLE, TEST_ENTITY_TABLE))
|
||||
.withLimitPerTable(5));
|
||||
|
||||
assertThat(output.getResults()).extracting(RecordSearchResult::getTableName, RecordSearchResult::getRecordId).containsExactly(
|
||||
tuple(TEST_ENTITY_TABLE, "3"),
|
||||
tuple(TEST_ENTITY_TABLE, "1"),
|
||||
tuple(TEST_ENTITY_TABLE, "2"),
|
||||
tuple(OTHER_TABLE, "7"));
|
||||
|
||||
verify(mockClient).searchRecordIdsPerTable(eq("widget blue"), eq(List.of(TEST_ENTITY_TABLE, OTHER_TABLE)), eq(5 * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), anyList(), eq(1234));
|
||||
|
||||
provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE)).withLimitPerTable(25));
|
||||
verify(mockClient).searchRecordIdsPerTable(eq("widget"), eq(List.of(TEST_ENTITY_TABLE)), eq(RecordSearchAction.MAX_PROVIDER_IDS_PER_TABLE), anyList(), eq(1234));
|
||||
|
||||
provider.search(new RecordSearchInput().withSearchTerm("gadget").withTableNames(List.of(TEST_ENTITY_TABLE)));
|
||||
verify(mockClient).searchRecordIdsPerTable(eq("gadget"), eq(List.of(TEST_ENTITY_TABLE)), eq(RecordSearchAction.DEFAULT_LIMIT_PER_TABLE * QuickSearchRecordSearchProvider.IDS_PER_LIMIT_MULTIPLIER), anyList(), eq(1234));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testSearch_shortTermOrNoIndexedTables_returnsEmptyWithoutOpenSearch() throws QException
|
||||
{
|
||||
assertThat(provider.search(new RecordSearchInput().withSearchTerm("x").withTableNames(List.of(TEST_ENTITY_TABLE))).getResults()).isEmpty();
|
||||
assertThat(provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(UNINDEXED_TABLE))).getResults()).isEmpty();
|
||||
assertThat(provider.search(new RecordSearchInput().withSearchTerm("widget")).getResults()).isEmpty();
|
||||
verify(mockClient, never()).searchRecordIdsPerTable(any(), any(), anyInt(), any(), anyInt());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testSearch_clientFailure_throwsAndBacksOff() throws QException
|
||||
{
|
||||
when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenThrow(new QException("boom"));
|
||||
|
||||
assertThatThrownBy(() -> provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE))))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("boom");
|
||||
assertThat(runtime.isRecordSearchBackingOff()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testSearch_uncheckedClientFailure_wrappedAsQException() throws QException
|
||||
{
|
||||
when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenThrow(new IllegalStateException("closed"));
|
||||
|
||||
assertThatThrownBy(() -> provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE))))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("closed");
|
||||
assertThat(runtime.isRecordSearchBackingOff()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A server that accepts connections and never answers (a hung OpenSearch):
|
||||
** the real client gives up after recordSearchTimeoutMillis with a
|
||||
** QException, well before the transport's response timeout.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSearch_hungOpenSearch_timesOutWithQException() throws Exception
|
||||
{
|
||||
try(ServerSocket hung = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()))
|
||||
{
|
||||
QuickSearchQBitConfig hungConfig = new QuickSearchQBitConfig()
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withOpensearchHost("127.0.0.1")
|
||||
.withOpensearchPort(hung.getLocalPort())
|
||||
.withOpensearchIndexName("test-index")
|
||||
.withResponseTimeoutMillis(60_000)
|
||||
.withRecordSearchTimeoutMillis(300);
|
||||
QuickSearchRuntime hungRuntime = registerRuntime(hungConfig, null);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
assertThatThrownBy(() -> provider.search(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE))))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("timed out after 300 ms");
|
||||
assertThat(System.currentTimeMillis() - start).isLessThan(10_000);
|
||||
assertThat(hungRuntime.isRecordSearchBackingOff()).isTrue();
|
||||
assertThat(provider.claimsTable(QContext.getQInstance().getTable(TEST_ENTITY_TABLE))).isFalse();
|
||||
|
||||
hungRuntime.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core RecordSearchAction re-reads the provider's ids by primary key, keeps
|
||||
** its order, drops ids that no longer exist, applies limitPerTable, and
|
||||
** fills record labels; unclaimed tables keep core's own search.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCoreRecordSearchAction_usesProviderIds() throws QException
|
||||
{
|
||||
insertEntities("Widget", 6);
|
||||
new InsertAction().execute(new InsertInput(UNINDEXED_TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue("name", "Widget elsewhere"))));
|
||||
|
||||
Map<String, List<String>> ids = new LinkedHashMap<>();
|
||||
ids.put(TEST_ENTITY_TABLE, List.of("5", "999", "2", "4"));
|
||||
ids.put(OTHER_TABLE, List.of());
|
||||
when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenReturn(ids);
|
||||
|
||||
RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withLimitPerTable(2));
|
||||
|
||||
assertThat(output.getResults()).filteredOn(r -> r.getTableName().equals(TEST_ENTITY_TABLE)).extracting(RecordSearchResult::getRecordId).containsExactly(5, 2);
|
||||
assertThat(output.getResults()).filteredOn(r -> r.getTableName().equals(UNINDEXED_TABLE)).extracting(RecordSearchResult::getRecordId).containsExactly(1);
|
||||
assertThat(output.getResults()).filteredOn(r -> r.getTableName().equals(OTHER_TABLE)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** When the provider fails, core searches the claimed tables itself.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCoreRecordSearchAction_providerFailure_fallsBackToCoreSearch() throws QException
|
||||
{
|
||||
insertEntities("Widget", 3);
|
||||
when(mockClient.searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt())).thenThrow(new QException("OpenSearch is down"));
|
||||
|
||||
RecordSearchOutput output = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE)));
|
||||
|
||||
assertThat(output.getResults()).extracting(RecordSearchResult::getRecordId).containsExactly(1, 2, 3);
|
||||
assertThat(runtime.isRecordSearchBackingOff()).isTrue();
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// while backing off, the provider claims nothing: OpenSearch is not //
|
||||
// called again and core still answers //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
RecordSearchOutput again = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(TEST_ENTITY_TABLE)));
|
||||
assertThat(again.getResults()).hasSize(3);
|
||||
verify(mockClient, times(1)).searchRecordIdsPerTable(anyString(), anyList(), anyInt(), anyList(), anyInt());
|
||||
}
|
||||
|
||||
}
|
||||
-490
@@ -1,490 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.RecordSearchAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.search.RecordSearchInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableFieldConfig;
|
||||
import com.kingsrook.qbits.quicksearch.actions.QuickSearchRecordSearchProvider;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchDiagnosticEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchValidationRun;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.AbstractDiagnosticToolStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.diagnostics.SearchPlaygroundStep;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The diagnostic tools without a reachable OpenSearch (the client points at a
|
||||
** closed port), plus their pure parts: rate limiting, HTML escaping, target
|
||||
** parsing, redaction, the path diagnosis branches, event recording and its
|
||||
** cap, provider failure events and counters, the widget, the playground's
|
||||
** honest failure output and its form loop through RunProcessAction.
|
||||
*******************************************************************************/
|
||||
class DiagnosticsOfflineTest
|
||||
{
|
||||
private static final String BACKEND = "memory";
|
||||
private static final String PRODUCT = "product";
|
||||
private static final String NOTE = "note";
|
||||
private static final String NO_SEARCH = "plain";
|
||||
private static final String LOCKED = "secret";
|
||||
|
||||
private QuickSearchQBitConfig config;
|
||||
private QuickSearchRuntime runtime;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An instance with an indexed product table, an unindexed note table with
|
||||
** core searchFields, a table without searchFields, and a permissioned
|
||||
** table; Quick Search produced in DEGRADED mode against a closed port.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws QException
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(table(PRODUCT));
|
||||
qInstance.addTable(table(NOTE).withSearchFields("name"));
|
||||
qInstance.addTable(table(NO_SEARCH));
|
||||
qInstance.addTable(table(LOCKED).withSearchFields("name").withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)));
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND)
|
||||
.withOpensearchHost("127.0.0.1")
|
||||
.withOpensearchPort(1)
|
||||
.withConnectTimeoutMillis(500)
|
||||
.withResponseTimeoutMillis(1000)
|
||||
.withRecordSearchTimeoutMillis(1000)
|
||||
.withOpensearchIndexName("diag-offline")
|
||||
.withEnableRealTimeIndexing(false)
|
||||
.withDiagnosticRunsPerMinute(1000)
|
||||
.withSearchableTable(PRODUCT, List.of(new SearchableFieldConfig("name")));
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
runtime = config.getRuntime();
|
||||
|
||||
new InsertAction().execute(new InsertInput(PRODUCT).withRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("name", "Blue Widget").withValue("modifyDate", Instant.now()),
|
||||
new QRecord().withValue("id", 2).withValue("name", "Red Gadget").withValue("modifyDate", Instant.now()))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
if(runtime != null)
|
||||
{
|
||||
runtime.close();
|
||||
}
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData table(String name)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(name)
|
||||
.withLabel(name)
|
||||
.withBackendName(BACKEND)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> events() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(QuickSearchDiagnosticEvent.TABLE_NAME);
|
||||
return (new QueryAction().execute(queryInput).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testRateLimiter_slidingMinute()
|
||||
{
|
||||
AtomicLong now = new AtomicLong(1_000_000L);
|
||||
DiagnosticsRateLimiter limiter = new DiagnosticsRateLimiter(now::get);
|
||||
|
||||
assertThat(limiter.tryAcquire("a", 2)).isTrue();
|
||||
assertThat(limiter.tryAcquire("a", 2)).isTrue();
|
||||
assertThat(limiter.tryAcquire("a", 2)).isFalse();
|
||||
assertThat(limiter.tryAcquire("b", 2)).isTrue();
|
||||
|
||||
now.addAndGet(60_000L);
|
||||
assertThat(limiter.tryAcquire("a", 2)).isTrue();
|
||||
assertThat(limiter.tryAcquire("a", null)).isTrue();
|
||||
assertThat(limiter.tryAcquire("a", 0)).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testHtml_escapesAndKeepsHighlights()
|
||||
{
|
||||
assertThat(DiagnosticsHtml.escape("<script>alert('x')</script> & \"q\"")).isEqualTo("<script>alert('x')</script> & "q"");
|
||||
assertThat(DiagnosticsHtml.highlight("a <em>blue</em> <b>x</b>")).isEqualTo("a <mark>blue</mark> <b>x</b>");
|
||||
assertThat(DiagnosticsHtml.prettyJson("{\"a\":1}")).contains("\"a\" : 1");
|
||||
assertThat(DiagnosticsHtml.prettyJson("not json")).isEqualTo("not json");
|
||||
assertThat(DiagnosticsHtml.prettyJson("\"" + "x".repeat(DiagnosticsHtml.MAX_JSON_CHARS + 10) + "\"")).contains("truncated at");
|
||||
|
||||
String html = new DiagnosticsHtml().open().facts(java.util.Collections.singletonList(new String[] { "<k>", null })).pre("<v>").close().toString();
|
||||
assertThat(html).contains("<k>").contains("<em>none</em>").contains("<v>").contains("overflow-wrap:anywhere").doesNotContain("<k>");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testTargetParseAndTableLists()
|
||||
{
|
||||
assertThat(AbstractDiagnosticToolStep.Target.parse("blogPost:42")).isEqualTo(new AbstractDiagnosticToolStep.Target("blogPost", "42"));
|
||||
assertThat(AbstractDiagnosticToolStep.Target.parse("t:a:b")).isEqualTo(new AbstractDiagnosticToolStep.Target("t", "a:b"));
|
||||
assertThat(AbstractDiagnosticToolStep.Target.parse(":1")).isNull();
|
||||
assertThat(AbstractDiagnosticToolStep.Target.parse("t:")).isNull();
|
||||
assertThat(AbstractDiagnosticToolStep.Target.parse(null)).isNull();
|
||||
|
||||
assertThat(SearchPlaygroundStep.parseTables(" a, b c,a ")).containsExactly("a", "b", "c");
|
||||
assertThat(SearchPlaygroundStep.parseTables(null)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testRedactResponse_hidesUnreadableHits()
|
||||
{
|
||||
String json = "{\"hits\":{\"hits\":[{\"_source\":{\"recordId\":\"1\",\"recordLabel\":\"ok\"},\"highlight\":{\"x\":[\"y\"]}},"
|
||||
+ "{\"_source\":{\"recordId\":\"2\",\"recordLabel\":\"private\"},\"highlight\":{\"x\":[\"secret\"]}}]}}";
|
||||
String redacted = SearchPlayground.redactResponse(json, Set.of("1"));
|
||||
assertThat(redacted).contains("\"recordLabel\":\"ok\"").doesNotContain("private").doesNotContain("secret").doesNotContain("\"2\"").contains("redacted: 1 hit(s)");
|
||||
assertThat(SearchPlayground.redactResponse("nope", Set.of())).contains("could not be parsed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Each branch of the path diagnosis.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPathDiagnosis_branches() throws QException
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
List<SearchPathDiagnosis.TablePath> paths = SearchPathDiagnosis.diagnose(qInstance, runtime, List.of(PRODUCT, NOTE, NO_SEARCH, LOCKED, "missing"), new QueryInput());
|
||||
|
||||
assertThat(paths.get(0).path()).isEqualTo(SearchPathDiagnosis.PATH_OPENSEARCH);
|
||||
assertThat(paths.get(1).path()).isEqualTo(SearchPathDiagnosis.PATH_CORE);
|
||||
assertThat(paths.get(1).reason()).contains("does not index");
|
||||
assertThat(paths.get(2).path()).isEqualTo(SearchPathDiagnosis.PATH_NONE);
|
||||
assertThat(paths.get(2).reason()).contains("no core searchFields");
|
||||
assertThat(paths.get(3).path()).isEqualTo(SearchPathDiagnosis.PATH_NONE);
|
||||
assertThat(paths.get(3).reason()).contains("read permission");
|
||||
assertThat(paths.get(4).reason()).contains("no table");
|
||||
|
||||
runtime.markRecordSearchFailed("connection refused");
|
||||
assertThat(SearchPathDiagnosis.diagnose(qInstance, runtime, List.of(PRODUCT), new QueryInput()).get(0).reason()).contains("backing off").contains("connection refused");
|
||||
runtime.clearRecordSearchBackoff();
|
||||
|
||||
new InsertAction().execute(new InsertInput(QuickSearchIndex.TABLE_NAME).withRecord(new QRecord().withValue("tableName", PRODUCT).withValue("enabled", false)));
|
||||
runtime.invalidateEnabledCache();
|
||||
assertThat(SearchPathDiagnosis.diagnose(qInstance, runtime, List.of(PRODUCT), new QueryInput()).get(0).reason()).contains("disabled");
|
||||
runtime.invalidateEnabledCache();
|
||||
new UpdateAction().execute(new UpdateInput(QuickSearchIndex.TABLE_NAME).withRecord(new QRecord().withValue("id", 1).withValue("enabled", true)));
|
||||
runtime.invalidateEnabledCache();
|
||||
|
||||
config.setServeCoreRecordSearch(false);
|
||||
assertThat(SearchPathDiagnosis.diagnose(qInstance, runtime, List.of(PRODUCT), new QueryInput()).get(0).reason()).contains("serveCoreRecordSearch is off");
|
||||
config.setServeCoreRecordSearch(true);
|
||||
|
||||
qInstance.withRecordSearchProvider(new QCodeReference(DiagnosticsOfflineTest.class));
|
||||
assertThat(SearchPathDiagnosis.diagnose(qInstance, runtime, List.of(PRODUCT), new QueryInput()).get(0).reason()).contains("Another record search provider");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testRecorder_writesEventsAsSystemAndCapsVolume() throws QException
|
||||
{
|
||||
DiagnosticEventRecorder recorder = runtime.getDiagnosticEventRecorder();
|
||||
recorder.record(new QuickSearchDiagnosticEvent().withEventType(QuickSearchDiagnosticEvent.TYPE_TOOL_RUN).withSource("TEST").withSearchTerm("x".repeat(500)));
|
||||
recorder.flush(5000);
|
||||
|
||||
List<QRecord> rows = events();
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).getValueString("hostName")).isNotBlank();
|
||||
assertThat(rows.get(0).getValueString("searchTerm")).hasSize(100);
|
||||
assertThat(rows.get(0).getValueInstant("createDate")).isNotNull();
|
||||
|
||||
for(int i = 0; i < DiagnosticEventRecorder.MAX_EVENTS_PER_MINUTE + 5; i++)
|
||||
{
|
||||
recorder.record(new QuickSearchDiagnosticEvent().withEventType(QuickSearchDiagnosticEvent.TYPE_TOOL_RUN).withSource("FLOOD"));
|
||||
}
|
||||
recorder.flush(5000);
|
||||
assertThat(events()).hasSize(DiagnosticEventRecorder.MAX_EVENTS_PER_MINUTE);
|
||||
assertThat(recorder.getDroppedCount()).isEqualTo(6);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testRecorder_noTableIsANoOp()
|
||||
{
|
||||
QContext.getQInstance().getTables().remove(QuickSearchDiagnosticEvent.TABLE_NAME);
|
||||
DiagnosticEventRecorder recorder = runtime.getDiagnosticEventRecorder();
|
||||
recorder.record(new QuickSearchDiagnosticEvent().withEventType(QuickSearchDiagnosticEvent.TYPE_TOOL_RUN));
|
||||
recorder.flush(5000);
|
||||
assertThat(recorder.getFailedCount()).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testRateLimit_refusesAndRecords() throws QException
|
||||
{
|
||||
config.setDiagnosticRunsPerMinute(1);
|
||||
DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_PLAYGROUND);
|
||||
assertThatThrownBy(() -> DiagnosticsSupport.checkRateLimit(runtime, DiagnosticsSupport.TOOL_PLAYGROUND))
|
||||
.isInstanceOf(QUserFacingException.class).hasMessageContaining("Too many");
|
||||
runtime.getDiagnosticEventRecorder().flush(5000);
|
||||
assertThat(events()).anyMatch(r -> QuickSearchDiagnosticEvent.TYPE_RATE_LIMITED.equals(r.getValueString("eventType")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A global search with OpenSearch down: core falls back (results still
|
||||
** come back), the provider counts and records the failure without the
|
||||
** term, and claims during the backoff are counted.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testProviderFailure_recordedAndCounted() throws QException
|
||||
{
|
||||
List<?> results = new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(PRODUCT))).getResults();
|
||||
assertThat(results).hasSize(1);
|
||||
|
||||
assertThat(runtime.getRecordSearchFailureCount()).isEqualTo(1);
|
||||
assertThat(runtime.getLastRecordSearchFailureMessage()).isNotBlank();
|
||||
assertThat(runtime.getRecordSearchBackoffRemainingMillis()).isPositive();
|
||||
|
||||
new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(PRODUCT)));
|
||||
assertThat(runtime.getBackoffDeclinedClaimCount()).isEqualTo(1);
|
||||
|
||||
runtime.getDiagnosticEventRecorder().flush(5000);
|
||||
QRecord event = events().stream().filter(r -> QuickSearchDiagnosticEvent.TYPE_SEARCH_FAILED.equals(r.getValueString("eventType"))).findFirst().orElseThrow();
|
||||
assertThat(event.getValueString("source")).isEqualTo(QuickSearchDiagnosticEvent.SOURCE_CORE_RECORD_SEARCH);
|
||||
assertThat(event.getValueString("searchTerm")).isNull();
|
||||
assertThat(event.getValueString("detail")).contains("term length 6");
|
||||
assertThat(event.getValueString("tableNames")).isEqualTo(PRODUCT);
|
||||
assertThat(event.getValueString("errorClass")).isNotBlank();
|
||||
assertThat(event.getValueString("errorMessage")).doesNotContainIgnoringCase("widget");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QuickSearchAction's failure message names the term; the event never
|
||||
** keeps it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQuickSearchActionFailure_termNeverStored() throws QException
|
||||
{
|
||||
assertThatThrownBy(() -> new com.kingsrook.qbits.quicksearch.actions.QuickSearchAction().execute(
|
||||
new com.kingsrook.qbits.quicksearch.actions.QuickSearchInput().withSearchTerm("Jane Doe-Smith")))
|
||||
.hasMessageContaining("Jane Doe-Smith");
|
||||
|
||||
runtime.getDiagnosticEventRecorder().flush(5000);
|
||||
QRecord event = events().stream().filter(r -> QuickSearchDiagnosticEvent.SOURCE_QUICK_SEARCH_ACTION.equals(r.getValueString("source"))).findFirst().orElseThrow();
|
||||
assertThat(event.getValueString("searchTerm")).isNull();
|
||||
assertThat(event.getValueString("errorMessage")).contains("[search term]").doesNotContainIgnoringCase("jane").doesNotContainIgnoringCase("smith");
|
||||
assertThat(event.getValueString("detail")).contains("term length 14");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testRedactTerm()
|
||||
{
|
||||
assertThat(DiagnosticsSupport.redactTerm("Search failed for term [Blue Widget]: parse error near 'widget'", "blue widget"))
|
||||
.isEqualTo("Search failed for term [[search term]]: parse error near '[search term]'");
|
||||
assertThat(DiagnosticsSupport.redactTerm("no term here", "a b")).isEqualTo("no term here");
|
||||
assertThat(DiagnosticsSupport.redactTerm(null, "x")).isNull();
|
||||
assertThat(DiagnosticsSupport.redactTerm("x (y)", "(y)")).isEqualTo("x [search term]");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The playground with OpenSearch down still reports honestly: the core
|
||||
** fallback's results, the provider failure, and failed probes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPlayground_openSearchDown_reportsFailureAndFallback()
|
||||
{
|
||||
SearchPlayground.Report report = new SearchPlayground().run(runtime, new SearchPlayground.Input("widget", List.of(PRODUCT, NOTE), null, "product:1"), new QueryInput());
|
||||
|
||||
assertThat(report.providerError()).isNotBlank();
|
||||
assertThat(report.coreError()).isNull();
|
||||
assertThat(report.coreResults().get(PRODUCT)).hasSize(1);
|
||||
assertThat(report.probes()).singleElement().satisfies(p -> assertThat(p.error()).isNotBlank());
|
||||
assertThat(report.termTokensError()).isNotBlank();
|
||||
assertThat(report.explainError()).isNotBlank();
|
||||
|
||||
String html = SearchPlayground.render(report);
|
||||
assertThat(html).contains("FAIL").contains("falls back").contains("(#1)");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The playground process through RunProcessAction: it stops at the form
|
||||
** with intro HTML, and after a run the next step is the form again (with
|
||||
** results), so the user can search again.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPlaygroundProcess_loopsBackToTheForm() throws QException
|
||||
{
|
||||
String processName = QuickSearchDiagnosticsProcessMetaDataHelper.PLAYGROUND_PROCESS_NAME;
|
||||
|
||||
RunProcessInput first = new RunProcessInput();
|
||||
first.setProcessName(processName);
|
||||
first.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
RunProcessOutput firstOutput = new RunProcessAction().execute(first);
|
||||
assertThat(firstOutput.getProcessState().getNextStepName()).hasValue(AbstractDiagnosticToolStep.STEP_FORM);
|
||||
assertThat(firstOutput.getValues().get(AbstractDiagnosticToolStep.STEP_FORM + ".html").toString()).contains("Indexed tables").contains(PRODUCT);
|
||||
|
||||
for(int run = 0; run < 2; run++)
|
||||
{
|
||||
RunProcessInput again = new RunProcessInput();
|
||||
again.setProcessName(processName);
|
||||
again.setProcessUUID(firstOutput.getProcessUUID());
|
||||
again.setStartAfterStep(AbstractDiagnosticToolStep.STEP_FORM);
|
||||
again.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
again.addValue(SearchPlaygroundStep.FIELD_TERM, "widget");
|
||||
again.addValue(SearchPlaygroundStep.FIELD_TABLES, PRODUCT);
|
||||
RunProcessOutput output = new RunProcessAction().execute(again);
|
||||
assertThat(output.getProcessState().getNextStepName()).hasValue(AbstractDiagnosticToolStep.STEP_FORM);
|
||||
assertThat(output.getValues().get(AbstractDiagnosticToolStep.STEP_FORM + ".html").toString()).contains("Search playground").contains("widget");
|
||||
}
|
||||
|
||||
runtime.getDiagnosticEventRecorder().flush(5000);
|
||||
assertThat(events()).filteredOn(r -> DiagnosticsSupport.TOOL_PLAYGROUND.equals(r.getValueString("source"))).hasSize(2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validation with OpenSearch down: a FAIL report that says why, saved as
|
||||
** a validation run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testValidate_openSearchDown_savesFailReport() throws QException
|
||||
{
|
||||
IndexValidator validator = new IndexValidator();
|
||||
IndexValidator.Report report = validator.validate(runtime, List.of(), 10);
|
||||
assertThat(report.status()).isEqualTo(QuickSearchValidationRun.STATUS_FAIL);
|
||||
assertThat(report.findings()).anyMatch(f -> f.check().equals("Reachable") && f.message().contains("did not answer"));
|
||||
|
||||
Integer runId = validator.persist(runtime, report, QuickSearchValidationRun.TRIGGER_MANUAL, "tester");
|
||||
assertThat(runId).isNotNull();
|
||||
QRecord row = new QueryAction().execute(new QueryInput(QuickSearchValidationRun.TABLE_NAME)).getRecords().get(0);
|
||||
assertThat(row.getValueString("reportHtml")).contains("did not answer");
|
||||
assertThat(row.getValueString("userName")).isEqualTo("tester");
|
||||
assertThat(IndexValidator.toJson(report)).contains("\"status\":\"FAIL\"");
|
||||
assertThat(row.getValueString("reportJson")).contains("\"status\":\"FAIL\"");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The widget says plainly what it knows: the backoff, the recorded
|
||||
** failure, and that no validation ran yet; then the latest validation.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWidget_honestStates() throws QException
|
||||
{
|
||||
String empty = QuickSearchDiagnosticsWidgetRenderer.renderHtml(runtime);
|
||||
assertThat(empty).contains("Served by OpenSearch").contains("None recorded").contains("None yet");
|
||||
|
||||
new RecordSearchAction().execute(new RecordSearchInput().withSearchTerm("widget").withTableNames(List.of(PRODUCT)));
|
||||
runtime.getDiagnosticEventRecorder().flush(5000);
|
||||
new InsertAction().execute(new InsertInput(QuickSearchValidationRun.TABLE_NAME).withRecordEntity(new QuickSearchValidationRun()
|
||||
.withStatus(QuickSearchValidationRun.STATUS_WARN).withTriggerType(QuickSearchValidationRun.TRIGGER_MANUAL).withSummary("WARN: 1 table(s)").withStartTime(Instant.now())));
|
||||
|
||||
String html = QuickSearchDiagnosticsWidgetRenderer.renderHtml(runtime);
|
||||
assertThat(html).contains("Backing off").contains(QuickSearchDiagnosticEvent.SOURCE_CORE_RECORD_SEARCH).contains("WARN: 1 table(s)")
|
||||
.contains("/app/" + QuickSearchValidationRun.TABLE_NAME + "/1");
|
||||
|
||||
assertThat(new QuickSearchRecordSearchProvider().claimsTable(QContext.getQInstance().getTable(PRODUCT))).isFalse();
|
||||
}
|
||||
|
||||
}
|
||||
-428
@@ -1,428 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.quicksearch.diagnostics;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.developer.DeveloperPanelsAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanel;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelBlock;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelBlockType;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelContext;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelKeyValue;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelTone;
|
||||
import com.kingsrook.qqq.backend.core.model.developer.DeveloperPanelsMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableFieldConfig;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDeveloperPanelsMetaDataProducer;
|
||||
import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for {@link QuickSearchDeveloperPanelProvider}: registration,
|
||||
** availability (indexed table, index read permission), the panel's content
|
||||
** with OpenSearch unreachable (a warning, not a failure) and with a faked
|
||||
** inspection, and actions and blocks hidden without permission.
|
||||
*******************************************************************************/
|
||||
class QuickSearchDeveloperPanelProviderTest
|
||||
{
|
||||
private static final String BACKEND = "memory";
|
||||
private static final String PRODUCT = "product";
|
||||
private static final String NOTE = "note";
|
||||
|
||||
private QuickSearchQBitConfig config;
|
||||
private QuickSearchRuntime runtime;
|
||||
private Integer indexRowId;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An indexed product table and an unindexed note table; Quick Search
|
||||
** produced in DEGRADED mode against a closed port (OpenSearch "down").
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws QException
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
QInstance qInstance = new QInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
|
||||
qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
qInstance.addTable(table(PRODUCT));
|
||||
qInstance.addTable(table(NOTE));
|
||||
QContext.init(qInstance, new QSession());
|
||||
|
||||
config = new QuickSearchQBitConfig()
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withBackendName(BACKEND)
|
||||
.withOpensearchHost("127.0.0.1")
|
||||
.withOpensearchPort(1)
|
||||
.withConnectTimeoutMillis(500)
|
||||
.withResponseTimeoutMillis(1000)
|
||||
.withOpensearchIndexName("devpanel-test")
|
||||
.withEnableRealTimeIndexing(false)
|
||||
.withSearchableTable(PRODUCT, List.of(new SearchableFieldConfig("name")));
|
||||
new QuickSearchQBitProducer().withConfig(config).produce(qInstance);
|
||||
runtime = config.getRuntime();
|
||||
|
||||
///////////////////////////////////////////////////////////////
|
||||
// data, inserted by a session that may write every table //
|
||||
///////////////////////////////////////////////////////////////
|
||||
QContext.setQSession(new QSession().withPermissions(allTablePermissions()));
|
||||
new InsertAction().execute(new InsertInput(PRODUCT).withRecords(List.of(
|
||||
new QRecord().withValue("id", 1).withValue("name", "Blue Widget").withValue("modifyDate", Instant.now()))));
|
||||
indexRowId = new InsertAction().execute(new InsertInput(config.getQuickSearchIndexTableName()).withRecords(List.of(
|
||||
new QRecord().withValue("tableName", PRODUCT).withValue("enabled", true).withValue("status", "ACTIVE").withValue("recordCount", 1)
|
||||
.withValue("lastBasepullTime", Instant.parse("2026-10-01T10:00:00Z"))))).getRecords().get(0).getValueInteger("id");
|
||||
new InsertAction().execute(new InsertInput(config.getQuickSearchIndexRunTableName()).withRecords(List.of(
|
||||
new QRecord().withValue("quickSearchIndexId", indexRowId).withValue("runType", "BASEPULL").withValue("status", "COMPLETED")
|
||||
.withValue("startTime", Instant.parse("2026-10-01T10:00:00Z")).withValue("recordsProcessed", 1).withValue("recordsIndexed", 1).withValue("errorCount", 0))));
|
||||
new InsertAction().execute(new InsertInput(config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)).withRecords(List.of(
|
||||
new QRecord().withValue("tableName", PRODUCT).withValue("recordId", "1").withValue("action", "UPSERT").withValue("errorMessage", "connection refused")
|
||||
.withValue("attempts", 3).withValue("status", QuickSearchFailedEvent.STATUS_PENDING))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
if(runtime != null)
|
||||
{
|
||||
runtime.close();
|
||||
}
|
||||
QContext.clear();
|
||||
QuickSearchQBitContext.clear();
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData table(String name)
|
||||
{
|
||||
return (new QTableMetaData()
|
||||
.withName(name)
|
||||
.withLabel(name)
|
||||
.withBackendName(BACKEND)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read and write permissions on the three operational tables.
|
||||
*******************************************************************************/
|
||||
private List<String> allTablePermissions()
|
||||
{
|
||||
List<String> permissions = new ArrayList<>();
|
||||
for(String tableName : List.of(config.getQuickSearchIndexTableName(), config.getQuickSearchIndexRunTableName(), config.applyPrefix(QuickSearchFailedEvent.TABLE_NAME)))
|
||||
{
|
||||
for(TablePermissionSubType subType : TablePermissionSubType.values())
|
||||
{
|
||||
permissions.add(PermissionsHelper.getTablePermissionName(tableName, subType));
|
||||
}
|
||||
}
|
||||
return (permissions);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A session with read on the operational tables, plus the debug and admin
|
||||
** permissions.
|
||||
*******************************************************************************/
|
||||
private void loginAsAdmin()
|
||||
{
|
||||
List<String> permissions = new ArrayList<>(allTablePermissions());
|
||||
permissions.add(QuickSearchQBitConfig.DEFAULT_DEBUG_PERMISSION_BASE_NAME + ".hasAccess");
|
||||
permissions.add(QuickSearchQBitConfig.DEFAULT_ADMIN_PERMISSION_BASE_NAME + ".hasAccess");
|
||||
QContext.setQSession(new QSession().withPermissions(permissions));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static DeveloperPanelContext context(String tableName, Integer id)
|
||||
{
|
||||
return (new DeveloperPanelContext()
|
||||
.withTable(QContext.getQInstance().getTable(tableName))
|
||||
.withRecord(new QRecord().withValue("id", id).withValue("name", "Blue Widget")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static DeveloperPanel quickSearchPanel(DeveloperPanelContext context)
|
||||
{
|
||||
return (new DeveloperPanelsAction().execute(context).stream()
|
||||
.filter(panel -> QuickSearchDeveloperPanelProvider.ID.equals(panel.getId()))
|
||||
.findFirst().orElse(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static DeveloperPanelBlock block(DeveloperPanel panel, String label)
|
||||
{
|
||||
return (panel.getBlocks().stream().filter(block -> label.equals(block.getLabel())).findFirst().orElse(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
private static DeveloperPanelKeyValue item(DeveloperPanelBlock block, String label)
|
||||
{
|
||||
return (block.getItems().stream().filter(item -> label.equals(item.getLabel())).findFirst().orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testProducerRegistersProviderOnce() throws QException
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
new QuickSearchDeveloperPanelsMetaDataProducer().produce(qInstance).addSelfToInstance(qInstance);
|
||||
|
||||
assertThat(DeveloperPanelsMetaData.of(qInstance).getProviders())
|
||||
.filteredOn(reference -> QuickSearchDeveloperPanelProvider.class.getName().equals(reference.getName()))
|
||||
.hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testAvailability()
|
||||
{
|
||||
QuickSearchDeveloperPanelProvider provider = new QuickSearchDeveloperPanelProvider();
|
||||
|
||||
loginAsAdmin();
|
||||
assertThat(provider.isAvailable(context(PRODUCT, 1))).isTrue();
|
||||
assertThat(provider.isAvailable(context(NOTE, 1))).as("table Quick Search does not index").isFalse();
|
||||
|
||||
QContext.setQSession(new QSession());
|
||||
assertThat(provider.isAvailable(context(PRODUCT, 1))).as("no read permission on Quick Search Index").isFalse();
|
||||
assertThat(quickSearchPanel(context(PRODUCT, 1))).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testPanelWithOpenSearchDown_warnsInsteadOfFailing()
|
||||
{
|
||||
loginAsAdmin();
|
||||
DeveloperPanel panel = quickSearchPanel(context(PRODUCT, 1));
|
||||
|
||||
assertThat(panel).isNotNull();
|
||||
assertThat(panel.getError()).isNull();
|
||||
assertThat(panel.getTitle()).isEqualTo("Quick Search");
|
||||
assertThat(panel.getOrder()).isEqualTo(400);
|
||||
assertThat(panel.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_UNAVAILABLE);
|
||||
assertThat(panel.getStatusTone()).isEqualTo(DeveloperPanelTone.WARNING);
|
||||
|
||||
DeveloperPanelBlock index = block(panel, "Index");
|
||||
assertThat(index.getType()).isEqualTo(DeveloperPanelBlockType.KEY_VALUES);
|
||||
assertThat(item(index, "Index row").getHref()).isEqualTo("/app/" + config.getQuickSearchIndexTableName() + "/" + indexRowId);
|
||||
assertThat(item(index, "Enabled").getValue()).isEqualTo("Yes");
|
||||
assertThat(item(index, "Status").getValue()).isEqualTo("ACTIVE");
|
||||
assertThat(item(index, "Indexed records").getValue()).isEqualTo("1");
|
||||
assertThat(item(index, "Last basepull").getValue()).contains("2026-10-01");
|
||||
assertThat(item(index, "Searchable fields").getValue()).isEqualTo("name");
|
||||
|
||||
DeveloperPanelBlock run = block(panel, "Latest index run");
|
||||
assertThat(item(run, "Type").getValue()).isEqualTo("BASEPULL");
|
||||
assertThat(item(run, "Status").getValue()).isEqualTo("COMPLETED");
|
||||
assertThat(item(run, "Run").getHref()).startsWith("/app/" + config.getQuickSearchIndexRunTableName() + "/");
|
||||
|
||||
DeveloperPanelBlock document = block(panel, "Search document");
|
||||
assertThat(document.getType()).isEqualTo(DeveloperPanelBlockType.TEXT);
|
||||
assertThat(document.getTone()).isEqualTo(DeveloperPanelTone.WARNING);
|
||||
assertThat(document.getText()).contains("OpenSearch could not be reached");
|
||||
|
||||
DeveloperPanelBlock failed = block(panel, "Failed index events");
|
||||
assertThat(failed.getType()).isEqualTo(DeveloperPanelBlockType.TABLE);
|
||||
assertThat(failed.getRows()).hasSize(1);
|
||||
assertThat(failed.getRows().get(0)).contains("UPSERT", QuickSearchFailedEvent.STATUS_PENDING, "3", "connection refused");
|
||||
|
||||
DeveloperPanelBlock actions = block(panel, "Actions");
|
||||
assertThat(actions.getActions()).extracting(action -> action.getProcessName()).containsExactly(
|
||||
QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, PRODUCT),
|
||||
QuickSearchDiagnosticsProcessMetaDataHelper.reindexRecordsProcessName(config, PRODUCT));
|
||||
assertThat(actions.getActions()).allMatch(action -> PRODUCT.equals(action.getTableName()));
|
||||
|
||||
DeveloperPanelBlock links = block(panel, "Links");
|
||||
assertThat(links.getLinks()).extracting(link -> link.getHref()).contains(
|
||||
"/app/" + config.applyPrefix(QuickSearchAdminAppMetaDataProducer.APP_NAME),
|
||||
"/app/" + config.getQuickSearchIndexTableName() + "/" + indexRowId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testWithoutDebugOrAdminPermission_noDocumentNoActions()
|
||||
{
|
||||
QContext.setQSession(new QSession().withPermission(PermissionsHelper.getTablePermissionName(config.getQuickSearchIndexTableName(), TablePermissionSubType.READ)));
|
||||
DeveloperPanel panel = quickSearchPanel(context(PRODUCT, 1));
|
||||
|
||||
assertThat(panel).isNotNull();
|
||||
assertThat(panel.getError()).isNull();
|
||||
assertThat(block(panel, "Search document").getText()).contains("debug permission");
|
||||
assertThat(block(panel, "Actions")).isNull();
|
||||
assertThat(block(panel, "Latest index run")).as("run table not readable").isNull();
|
||||
assertThat(block(panel, "Failed index events")).as("failed event table not readable").isNull();
|
||||
assertThat(block(panel, "Links").getLinks()).extracting(link -> link.getHref())
|
||||
.doesNotContain("/app/" + config.applyPrefix(QuickSearchAdminAppMetaDataProducer.APP_NAME));
|
||||
assertThat(panel.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_ENABLED);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDebugWithoutAdmin_inspectActionOnly()
|
||||
{
|
||||
QContext.setQSession(new QSession()
|
||||
.withPermission(PermissionsHelper.getTablePermissionName(config.getQuickSearchIndexTableName(), TablePermissionSubType.READ))
|
||||
.withPermission(QuickSearchQBitConfig.DEFAULT_DEBUG_PERMISSION_BASE_NAME + ".hasAccess"));
|
||||
DeveloperPanel panel = quickSearchPanel(context(PRODUCT, 1));
|
||||
|
||||
assertThat(block(panel, "Actions").getActions()).extracting(action -> action.getProcessName())
|
||||
.containsExactly(QuickSearchDiagnosticsProcessMetaDataHelper.inspectProcessName(config, PRODUCT));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testFakedInspection_currentAndStale() throws QException
|
||||
{
|
||||
loginAsAdmin();
|
||||
|
||||
DeveloperPanel current = new FakeInspectionProvider(RecordInspector.VERDICT_CURRENT,
|
||||
List.of(new RecordInspector.FieldDiff("fieldValues.name", "Blue Widget", "Blue Widget"))).build(context(PRODUCT, 1));
|
||||
assertThat(current.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_INDEXED);
|
||||
assertThat(current.getStatusTone()).isEqualTo(DeveloperPanelTone.SUCCESS);
|
||||
DeveloperPanelBlock comparison = block(current, "Database vs index");
|
||||
assertThat(comparison.getColumns()).containsExactly("Field", "Database", "Index", "Match");
|
||||
assertThat(comparison.getRows()).containsExactly(List.of("fieldValues.name", "Blue Widget", "Blue Widget", "yes"));
|
||||
|
||||
DeveloperPanel stale = new FakeInspectionProvider(RecordInspector.VERDICT_STALE,
|
||||
List.of(new RecordInspector.FieldDiff("fieldValues.name", "Blue Widget", "Old Name"))).build(context(PRODUCT, 1));
|
||||
assertThat(stale.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_STALE);
|
||||
assertThat(block(stale, "Database vs index").getRows().get(0)).containsExactly("fieldValues.name", "Blue Widget", "Old Name", "NO");
|
||||
|
||||
DeveloperPanel missing = new FakeInspectionProvider(RecordInspector.VERDICT_MISSING, List.of()).build(context(PRODUCT, 1));
|
||||
assertThat(missing.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_NOT_INDEXED);
|
||||
assertThat(missing.getStatusTone()).isEqualTo(DeveloperPanelTone.ERROR);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testDisabledTable() throws QException
|
||||
{
|
||||
QContext.setQSession(new QSession().withPermissions(allTablePermissions()));
|
||||
new UpdateAction().execute(new UpdateInput(config.getQuickSearchIndexTableName()).withRecords(List.of(new QRecord().withValue("id", indexRowId).withValue("enabled", false))));
|
||||
|
||||
loginAsAdmin();
|
||||
DeveloperPanel panel = new FakeInspectionProvider(RecordInspector.VERDICT_CURRENT, List.of()).build(context(PRODUCT, 1));
|
||||
assertThat(panel.getStatusLabel()).isEqualTo(QuickSearchDeveloperPanelProvider.STATUS_DISABLED);
|
||||
assertThat(item(block(panel, "Index"), "Enabled").getValue()).isEqualTo("No");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testTruncate()
|
||||
{
|
||||
assertThat(QuickSearchDeveloperPanelProvider.truncate(null)).isNull();
|
||||
assertThat(QuickSearchDeveloperPanelProvider.truncate("short")).isEqualTo("short");
|
||||
assertThat(QuickSearchDeveloperPanelProvider.truncate("x".repeat(QuickSearchDeveloperPanelProvider.MAX_VALUE_LENGTH + 5))).endsWith("(" + (QuickSearchDeveloperPanelProvider.MAX_VALUE_LENGTH + 5) + " characters)");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A provider whose inspection is a fixed result (no OpenSearch).
|
||||
*******************************************************************************/
|
||||
private static class FakeInspectionProvider extends QuickSearchDeveloperPanelProvider
|
||||
{
|
||||
private final String verdict;
|
||||
private final List<RecordInspector.FieldDiff> diffs;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
FakeInspectionProvider(String verdict, List<RecordInspector.FieldDiff> diffs)
|
||||
{
|
||||
this.verdict = verdict;
|
||||
this.diffs = diffs;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected RecordInspector.Inspection inspect(QuickSearchRuntime runtime, String tableName, String recordId, AbstractActionInput permissionInput)
|
||||
{
|
||||
return (new RecordInspector.Inspection(tableName, recordId, verdict, diffs, null, null, List.of(), null, null));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+66
-84
@@ -39,9 +39,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.MissingLibraryTransportCustomizer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
@@ -50,7 +52,9 @@ import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
|
||||
import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
|
||||
import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
|
||||
import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
|
||||
import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
|
||||
@@ -78,8 +82,6 @@ import static org.mockito.Mockito.when;
|
||||
*******************************************************************************/
|
||||
class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
{
|
||||
private static final String COMPANY_TABLE = "company";
|
||||
|
||||
private IndexEventPublisher publisher;
|
||||
private QuickSearchRuntime runtime;
|
||||
|
||||
@@ -106,25 +108,6 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
config.setRuntime(runtime);
|
||||
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// a foreign key to a company table, through a table PVS, and a //
|
||||
// record label over it, so display values can be asserted //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
qInstance.addTable(new QTableMetaData()
|
||||
.withName(COMPANY_TABLE)
|
||||
.withBackendName(TEST_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s")
|
||||
.withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING)));
|
||||
qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(COMPANY_TABLE));
|
||||
qInstance.getTable(TEST_ENTITY_TABLE)
|
||||
.withField(new QFieldMetaData("companyId", QFieldType.INTEGER).withPossibleValueSourceName(COMPANY_TABLE))
|
||||
.withRecordLabelFormat("%s - %s")
|
||||
.withRecordLabelFields("name", "companyId");
|
||||
|
||||
qInstance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("quick-search").withVersion("test").withConfig(config));
|
||||
qInstance.withRecordChangeListener(new QCodeReference(QuickSearchRecordChangeListener.class));
|
||||
}
|
||||
@@ -170,25 +153,56 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withTransaction_publishesOnlyAfterCommit() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
insert("widget", transaction);
|
||||
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
|
||||
transaction.commit();
|
||||
verify(publisher, times(1)).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withTransaction_rollbackNeverPublishes() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
insert("widget", transaction);
|
||||
transaction.rollback();
|
||||
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_indexesDisplayValuesAndPossibleValueLabels() throws QException
|
||||
{
|
||||
InsertInput companyInsert = new InsertInput();
|
||||
companyInsert.setTableName(COMPANY_TABLE);
|
||||
companyInsert.setRecords(List.of(new QRecord().withValue("name", "Acme Corp")));
|
||||
Integer companyId = new InsertAction().execute(companyInsert).getRecords().get(0).getValueInteger("id");
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// the inserted record holds the raw categoryId (1); its document //
|
||||
// must carry the possible-value label, as an updated record's does //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
addCategoryPossibleValueField();
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TEST_ENTITY_TABLE);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("name", "widget").withValue("description", "d").withValue("companyId", companyId)));
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("name", "widget").withValue("description", "d").withValue("categoryId", 1)));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(publisher, times(1)).publishIndexEvents(captor.capture());
|
||||
QRecord indexed = captor.getValue().get(0).getRecord();
|
||||
assertThat(indexed.getValueInteger("companyId")).isEqualTo(companyId);
|
||||
assertThat(indexed.getDisplayValue("companyId")).as("the PVS label, not the raw key").isEqualTo("Acme Corp");
|
||||
assertThat(indexed.getRecordLabel()).isEqualTo("widget - Acme Corp");
|
||||
OpenSearchDocument document = IndexingUtils.buildDocument(captor.getValue().get(0).getRecord(), new QuickSearchableTableConfig()
|
||||
.withTableName(TEST_ENTITY_TABLE)
|
||||
.withPrimaryKeyField("id")
|
||||
.withSearchableFields(List.of("name", "categoryId")));
|
||||
|
||||
assertThat(document.getFieldValues()).containsEntry("categoryId", "Gadgets");
|
||||
assertThat(document.getSearchableText()).isEqualTo("widget Gadgets");
|
||||
assertThat(document.getRecordLabel()).isEqualTo("widget - Gadgets");
|
||||
}
|
||||
|
||||
|
||||
@@ -218,57 +232,6 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_reReadFailsThenRollback_logsCauseAndRecordsNothing() throws QException
|
||||
{
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
try(MockedConstruction<QueryAction> ignored = failingQueries(new NoClassDefFoundError("com/example/pvs/Provider")))
|
||||
{
|
||||
new QuickSearchRecordChangeListener().onRecordsChanged(changeEvent(RecordChangeType.INSERT, transaction, 7));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QLogger.deactivateCollectingLoggerForClass(QuickSearchRecordChangeListener.class);
|
||||
}
|
||||
transaction.rollback();
|
||||
|
||||
assertThat(queryFailedEvents()).isEmpty();
|
||||
assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains("Could not re-read inserted records").contains(TEST_ENTITY_TABLE).contains("com/example/pvs/Provider");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withTransaction_publishesOnlyAfterCommit() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
insert("widget", transaction);
|
||||
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
|
||||
transaction.commit();
|
||||
verify(publisher, times(1)).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testInsert_withTransaction_rollbackNeverPublishes() throws QException
|
||||
{
|
||||
QBackendTransaction transaction = new QBackendTransaction();
|
||||
insert("widget", transaction);
|
||||
transaction.rollback();
|
||||
|
||||
verify(publisher, never()).publishIndexEvents(anyList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void testUpdate_reReadsFullRecord() throws QException
|
||||
{
|
||||
@@ -452,7 +415,7 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains("Could not re-read updated records").contains(TEST_ENTITY_TABLE).contains("\"recordCount\":2").doesNotContain("changed");
|
||||
assertThat(m.getMessage()).contains("Could not re-read inserted or updated records").contains(TEST_ENTITY_TABLE).contains("\"recordCount\":2").doesNotContain("changed");
|
||||
assertThat(m.getMessage()).contains("possible-value backend unavailable");
|
||||
});
|
||||
}
|
||||
@@ -506,7 +469,7 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
.anySatisfy(m ->
|
||||
{
|
||||
assertThat(m.getLevel()).isEqualTo(Level.WARN);
|
||||
assertThat(m.getMessage()).contains("Could not re-read updated records").contains(TEST_ENTITY_TABLE).doesNotContain("changed");
|
||||
assertThat(m.getMessage()).contains("Could not re-read inserted or updated records").contains(TEST_ENTITY_TABLE).doesNotContain("changed");
|
||||
})
|
||||
.anySatisfy(m ->
|
||||
{
|
||||
@@ -586,6 +549,25 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Give the test table a categoryId field backed by an enum possible-value
|
||||
** source (1 is "Gadgets"), and a record label that includes it.
|
||||
*******************************************************************************/
|
||||
private void addCategoryPossibleValueField()
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
qInstance.addPossibleValueSource(new QPossibleValueSource()
|
||||
.withName("testCategory")
|
||||
.withType(QPossibleValueSourceType.ENUM)
|
||||
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
|
||||
.withEnumValues(List.of(new QPossibleValue<>(1, "Gadgets"))));
|
||||
qInstance.getTable(TEST_ENTITY_TABLE)
|
||||
.withField(new QFieldMetaData("categoryId", QFieldType.INTEGER).withPossibleValueSourceName("testCategory"))
|
||||
.withRecordLabelFormatAndFields("%s - %s", "name", "categoryId");
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void delete(Integer id) throws QException
|
||||
{
|
||||
DeleteInput deleteInput = new DeleteInput();
|
||||
|
||||
@@ -224,34 +224,6 @@ class BasepullIndexStepTest extends BaseQuickSearchTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a RUNNING FULL_REINDEX run record for the index row whose start
|
||||
** and last heartbeat were the given number of minutes ago.
|
||||
*******************************************************************************/
|
||||
private QRecord insertRunningFullReindex(QRecord indexRow, long minutesAgo) throws QException
|
||||
{
|
||||
InsertInput input = new InsertInput();
|
||||
input.setTableName(QuickSearchIndexRun.TABLE_NAME);
|
||||
input.setRecords(List.of(new QRecord()
|
||||
.withValue("quickSearchIndexId", indexRow.getValue("id"))
|
||||
.withValue("runType", FullReindexStep.RUN_TYPE)
|
||||
.withValue("status", AbstractIndexingStep.RUN_RUNNING)));
|
||||
QRecord run = new InsertAction().execute(input).getRecords().get(0);
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// set both times without QQQ stamping modifyDate with the present //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
Instant lastSeen = Instant.now().minus(minutesAgo, ChronoUnit.MINUTES);
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndexRun.TABLE_NAME);
|
||||
updateInput.setOmitModifyDateUpdate(true);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", run.getValue("id")).withValue("startTime", lastSeen).withValue("modifyDate", lastSeen)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
return (run);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Query all rows of a table.
|
||||
*******************************************************************************/
|
||||
@@ -951,8 +923,7 @@ class BasepullIndexStepTest extends BaseQuickSearchTest
|
||||
@Test
|
||||
void testReplayFailedEvents_awaitingReindexRowsAreNotReplayed() throws QException
|
||||
{
|
||||
QRecord row = insertIndexRow(TEST_ENTITY_TABLE, true, Instant.now(), 60);
|
||||
insertRunningFullReindex(row, 1);
|
||||
insertIndexRow(TEST_ENTITY_TABLE, true, Instant.now(), 60);
|
||||
QRecord captured = insertFailedEvent(TEST_ENTITY_TABLE, "7", "DELETE", 0);
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchFailedEvent.TABLE_NAME);
|
||||
@@ -969,65 +940,6 @@ class BasepullIndexStepTest extends BaseQuickSearchTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a DELETE event whose record exists again (re-created with the same
|
||||
** key since) keeps the record's document; the event is resolved.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReplayFailedEvents_deleteOfRecreatedRecord_keepsItsDocument() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(TEST_ENTITY_TABLE, true, Instant.now(), 60);
|
||||
insertFailedEvent(TEST_ENTITY_TABLE, "1", "DELETE", 0);
|
||||
insertFailedEvent(TEST_ENTITY_TABLE, "9", "DELETE", 0);
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new BasepullIndexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
verify(mockClient, times(1)).deleteDocuments(anyList(), anyInt());
|
||||
verify(mockClient).deleteDocuments(List.of(TEST_ENTITY_TABLE + ":9"), QuickSearchQBitContext.getConfig().getBulkBatchSize());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
assertThat(output.getValueInteger("failedEventsReplayed")).isEqualTo(2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a full reindex that stopped without finishing (table left
|
||||
** REBUILDING, run record without a heartbeat for two hours) is recovered
|
||||
** by the scheduled basepull, which replays its captured deletes in the same
|
||||
** run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBasepull_recoversAStoppedFullReindex_andReplaysItsCapturedDeletes() throws QException
|
||||
{
|
||||
QRecord row = insertIndexRow(TEST_ENTITY_TABLE, true, Instant.now(), 60);
|
||||
QRecord deadRun = insertRunningFullReindex(row, 120);
|
||||
QRecord captured = insertFailedEvent(TEST_ENTITY_TABLE, "7", "DELETE", 0);
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchFailedEvent.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", captured.getValue("id")).withValue("status", QuickSearchFailedEvent.STATUS_AWAITING_REINDEX)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id")).withValue("status", AbstractIndexingStep.STATUS_REBUILDING)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new BasepullIndexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
verify(mockClient).deleteDocuments(List.of(TEST_ENTITY_TABLE + ":7"), QuickSearchQBitContext.getConfig().getBulkBatchSize());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
assertThat(output.getValueInteger("failedEventsReplayed")).isEqualTo(1);
|
||||
assertThat(queryAll(QuickSearchIndex.TABLE_NAME).get(0).getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME))
|
||||
.filteredOn(run -> deadRun.getValue("id").equals(run.getValue("id")))
|
||||
.singleElement()
|
||||
.satisfies(run -> assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: drift found while a full reindex is running does not replace the
|
||||
** REBUILDING status the listener relies on.
|
||||
@@ -1036,7 +948,6 @@ class BasepullIndexStepTest extends BaseQuickSearchTest
|
||||
void testDriftDetection_duringFullReindex_keepsRebuilding() throws QException
|
||||
{
|
||||
QRecord row = insertIndexRow(TEST_ENTITY_TABLE, true, Instant.now(), 60);
|
||||
insertRunningFullReindex(row, 1);
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id"))
|
||||
|
||||
@@ -17,14 +17,9 @@
|
||||
package com.kingsrook.qbits.quicksearch.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
@@ -42,11 +37,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
|
||||
import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
|
||||
import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
|
||||
@@ -68,7 +61,6 @@ import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -564,9 +556,9 @@ class FullReindexStepTest extends BaseQuickSearchTest
|
||||
new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex, run);
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex);
|
||||
statusesDuringRebuild.add(FullReindexStepTest.this.queryIndexRow().getValueString("status"));
|
||||
insertCapturedDelete("7");
|
||||
return (counts);
|
||||
@@ -759,777 +751,4 @@ class FullReindexStepTest extends BaseQuickSearchTest
|
||||
assertThat(rows.get(0).getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a RUNNING FULL_REINDEX run record for the given index row, started
|
||||
** startedMinutesAgo and last refreshed (heartbeat) heartbeatMinutesAgo.
|
||||
*******************************************************************************/
|
||||
private QRecord insertRunningRun(Integer quickSearchIndexId, long startedMinutesAgo, long heartbeatMinutesAgo) throws QException
|
||||
{
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(QuickSearchIndexRun.TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord()
|
||||
.withValue("quickSearchIndexId", quickSearchIndexId)
|
||||
.withValue("runType", FullReindexStep.RUN_TYPE)
|
||||
.withValue("status", AbstractIndexingStep.RUN_RUNNING)));
|
||||
QRecord run = new InsertAction().execute(insertInput).getRecords().get(0);
|
||||
setRunTimes(run.getValueInteger("id"), Instant.now().minus(startedMinutesAgo, ChronoUnit.MINUTES), Instant.now().minus(heartbeatMinutesAgo, ChronoUnit.MINUTES));
|
||||
return (queryRun(run.getValueInteger("id")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Move a run record's start and heartbeat two hours into the past, as if
|
||||
** its run had stopped refreshing it.
|
||||
*******************************************************************************/
|
||||
private void backdateRun(Integer runId) throws QException
|
||||
{
|
||||
Instant twoHoursAgo = Instant.now().minus(2, ChronoUnit.HOURS);
|
||||
setRunTimes(runId, twoHoursAgo, twoHoursAgo);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set a run record's startTime and modifyDate, without QQQ stamping the
|
||||
** modifyDate with the current time.
|
||||
*******************************************************************************/
|
||||
private void setRunTimes(Integer runId, Instant startTime, Instant modifyDate) throws QException
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndexRun.TABLE_NAME);
|
||||
updateInput.setOmitModifyDateUpdate(true);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", runId).withValue("startTime", startTime).withValue("modifyDate", modifyDate)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The run record with the given id, for lambdas and overrides that cannot
|
||||
** throw a checked exception.
|
||||
*******************************************************************************/
|
||||
private QRecord queryRunQuietly(Integer runId)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (queryRun(runId));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
throw (new IllegalStateException(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The run record with the given id.
|
||||
*******************************************************************************/
|
||||
private QRecord queryRun(Integer runId) throws QException
|
||||
{
|
||||
return (queryAll(QuickSearchIndexRun.TABLE_NAME).stream()
|
||||
.filter(run -> runId.equals(run.getValueInteger("id")))
|
||||
.findFirst().orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the status and lastBasepullTime on the testEntity quickSearchIndex row.
|
||||
*******************************************************************************/
|
||||
private void updateIndexRow(String status, Instant lastBasepullTime) throws QException
|
||||
{
|
||||
UpdateInput updateInput = new UpdateInput();
|
||||
updateInput.setTableName(QuickSearchIndex.TABLE_NAME);
|
||||
updateInput.setRecords(List.of(new QRecord().withValue("id", queryIndexRow().getValue("id"))
|
||||
.withValue("status", status)
|
||||
.withValue("lastBasepullTime", lastBasepullTime)));
|
||||
new UpdateAction().execute(updateInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a full reindex whose run record is fresh blocks a second one, which
|
||||
** fails before it creates anything.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLiveRun_blocksASecondRun() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(true);
|
||||
insertRunningRun(queryIndexRow().getValueInteger("id"), 1, 1);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("already running");
|
||||
|
||||
verify(mockClient, never()).createPhysicalIndex(anyString());
|
||||
verify(mockClient, never()).indexDocuments(anyString(), anyList(), anyInt());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).singleElement()
|
||||
.satisfies(run -> assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_RUNNING));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a live full reindex also blocks a single-table full reindex.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLiveRun_blocksASingleTableRun() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(true);
|
||||
insertRunningRun(queryIndexRow().getValueInteger("id"), 1, 1);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(inputForTable(TEST_ENTITY_TABLE), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("already running");
|
||||
|
||||
verify(mockClient, never()).indexDocuments(anyList(), anyInt());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a dead run (no heartbeat past the threshold) does not block a new
|
||||
** one; it is recovered first, and the deletes it captured wait for the new
|
||||
** run, which applies them after its swap.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testStaleRun_doesNotBlock_andIsRecovered() throws QException
|
||||
{
|
||||
insertTestEntities(2);
|
||||
insertIndexRow(true);
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
Integer staleRunId = insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 120).getValueInteger("id");
|
||||
insertCapturedDelete("9");
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(1));
|
||||
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
new FullReindexStep().run(new RunBackendStepInput(), output);
|
||||
|
||||
assertThat(output.getValueBoolean("aliasSwapped")).isTrue();
|
||||
assertThat(queryRun(staleRunId).getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED);
|
||||
assertThat(queryRun(staleRunId).getValueString("errorMessage")).contains("stopped without finishing");
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
|
||||
InOrder inOrder = inOrder(mockClient);
|
||||
inOrder.verify(mockClient).swapAliasTo(PHYSICAL_INDEX);
|
||||
inOrder.verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":9")), anyInt());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the run records stay RUNNING while the captured deletes are applied
|
||||
** after the swap, so a basepull meanwhile leaves those rows to this run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_runStaysRunningUntilCapturedDeletesAreApplied() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("9");
|
||||
|
||||
List<String> runStatusesWhileApplying = new ArrayList<>();
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
queryAll(QuickSearchIndexRun.TABLE_NAME).forEach(run -> runStatusesWhileApplying.add(run.getValueString("status")));
|
||||
return (new BulkIndexResult().withSuccessCount(1));
|
||||
});
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
assertThat(runStatusesWhileApplying).containsExactly(AbstractIndexingStep.RUN_RUNNING);
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).singleElement()
|
||||
.satisfies(run -> assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_COMPLETED));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the run record's modifyDate is refreshed after every page, so a
|
||||
** long rebuild keeps showing it is alive.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testHeartbeat_refreshesTheRunRecordAfterEachPage() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(2);
|
||||
insertTestEntities(5);
|
||||
insertIndexRow(true);
|
||||
Integer runId = insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 120).getValueInteger("id");
|
||||
|
||||
AtomicInteger pagesIndexed = new AtomicInteger(0);
|
||||
when(mockClient.indexDocuments(anyString(), anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
pagesIndexed.incrementAndGet();
|
||||
return (new BulkIndexResult().withSuccessCount(((List<?>) invocation.getArgument(1)).size()));
|
||||
});
|
||||
|
||||
List<Integer> pagesIndexedAtEachHeartbeat = new ArrayList<>();
|
||||
List<Instant> heartbeatsWritten = new ArrayList<>();
|
||||
FullReindexStep step = new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected void heartbeat(QuickSearchIndexRun run)
|
||||
{
|
||||
super.heartbeat(run);
|
||||
pagesIndexedAtEachHeartbeat.add(pagesIndexed.get());
|
||||
heartbeatsWritten.add(queryRunQuietly(runId).getValueInstant("modifyDate"));
|
||||
}
|
||||
};
|
||||
|
||||
Instant before = Instant.now();
|
||||
step.indexAllRecords(mockClient, QuickSearchQBitContext.getTableConfig(TEST_ENTITY_TABLE), List.of(), PHYSICAL_INDEX,
|
||||
new QuickSearchIndexRun().withId(runId).withRunType(FullReindexStep.RUN_TYPE));
|
||||
|
||||
assertThat(pagesIndexedAtEachHeartbeat).containsExactly(1, 2, 3);
|
||||
assertThat(heartbeatsWritten).hasSize(3).allSatisfy(heartbeat -> assertThat(heartbeat).isAfterOrEqualTo(before));
|
||||
assertThat(heartbeatsWritten).isSorted();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: only FULL_REINDEX runs have a heartbeat (nothing reads one for the
|
||||
** other run types).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testHeartbeat_onlyForFullReindexRuns() throws QException
|
||||
{
|
||||
insertIndexRow(true);
|
||||
QRecord run = insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 120);
|
||||
|
||||
new FullReindexStep().heartbeat(new QuickSearchIndexRun().withId(run.getValueInteger("id")).withRunType(BasepullIndexStep.RUN_TYPE));
|
||||
assertThat(queryRun(run.getValueInteger("id")).getValueInstant("modifyDate")).isEqualTo(run.getValueInstant("modifyDate"));
|
||||
|
||||
new FullReindexStep().heartbeat(new QuickSearchIndexRun().withId(run.getValueInteger("id")).withRunType(FullReindexStep.RUN_TYPE));
|
||||
assertThat(queryRun(run.getValueInteger("id")).getValueInstant("modifyDate")).isAfter(run.getValueInstant("modifyDate"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a heartbeat that cannot be written is logged and the pages are
|
||||
** still indexed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testHeartbeat_failureDoesNotStopIndexing() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(2);
|
||||
insertTestEntities(3);
|
||||
|
||||
AbstractIndexingStep.IndexCounts counts;
|
||||
try(MockedConstruction<UpdateAction> updateActions = mockConstruction(UpdateAction.class, (updateAction, context) ->
|
||||
when(updateAction.execute(any())).thenThrow(new QException("database unavailable"))))
|
||||
{
|
||||
counts = new FullReindexStep().indexAllRecords(mockClient, QuickSearchQBitContext.getTableConfig(TEST_ENTITY_TABLE), List.of(), PHYSICAL_INDEX,
|
||||
new QuickSearchIndexRun().withId(1).withRunType(FullReindexStep.RUN_TYPE));
|
||||
assertThat(updateActions.constructed()).hasSize(2);
|
||||
}
|
||||
|
||||
assertThat(counts.processed()).isEqualTo(3);
|
||||
assertThat(counts.indexed()).isEqualTo(3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a run that stalls past the threshold is recovered by another node
|
||||
** while still going. It notices before its swap and stops without swapping
|
||||
** and without touching the tables or the captured deletes, which the
|
||||
** recovery already handed to basepull.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRunJudgedDead_recoveredWhileGoing_doesNotSwap() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex, run);
|
||||
insertCapturedDelete("7");
|
||||
backdateRun(run.getId());
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).isEmpty();
|
||||
return (counts);
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("recovered");
|
||||
|
||||
verify(mockClient, never()).swapAliasTo(anyString());
|
||||
verify(mockClient).deletePhysicalIndex(PHYSICAL_INDEX);
|
||||
verify(mockClient, never()).deleteDocuments(anyList(), anyInt());
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).singleElement()
|
||||
.satisfies(row -> assertThat(row.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a table that is no longer REBUILDING before the swap (a recovery
|
||||
** that read the run records just before this run created its own put it
|
||||
** back to ACTIVE) means a delete since may have reached only the old index,
|
||||
** so the run fails without swapping and applies its captured deletes to the
|
||||
** old index.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTableNoLongerRebuildingBeforeTheSwap_doesNotSwap() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(1));
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected IndexCounts indexAllRecords(QuickSearchOpenSearchClient client, QuickSearchableTableConfig tableConfig, List<QFilterCriteria> extraCriteria, String targetIndex, QuickSearchIndexRun run) throws QException
|
||||
{
|
||||
IndexCounts counts = super.indexAllRecords(client, tableConfig, extraCriteria, targetIndex, run);
|
||||
insertCapturedDelete("7");
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
return (counts);
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("no longer REBUILDING");
|
||||
|
||||
verify(mockClient, never()).swapAliasTo(anyString());
|
||||
verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":7")), anyInt());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).isEmpty();
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).singleElement()
|
||||
.satisfies(run -> assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when another node starts a full reindex at the same moment (after
|
||||
** this one checked, before it created its run records), this one sees the
|
||||
** other's fresh run record and stops before it touches anything.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSimultaneousStart_secondRunStopsBeforeMarking() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
AtomicBoolean otherStarted = new AtomicBoolean(false);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected QuickSearchIndexRun createRunRecord(Integer indexId, String runType) throws QException
|
||||
{
|
||||
if(otherStarted.compareAndSet(false, true))
|
||||
{
|
||||
insertRunningRun(indexId, 0, 0);
|
||||
}
|
||||
return (super.createRunRecord(indexId, runType));
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("already running");
|
||||
|
||||
verify(mockClient, never()).createPhysicalIndex(anyString());
|
||||
verify(mockClient, never()).swapAliasTo(anyString());
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).extracting(run -> run.getValueString("status"))
|
||||
.containsExactlyInAnyOrder(AbstractIndexingStep.RUN_RUNNING, AbstractIndexingStep.RUN_FAILED);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: recovering a dead run marks its record FAILED, puts the table back
|
||||
** to ACTIVE with the basepull watermark rewound to the run's start, and
|
||||
** hands the captured deletes to basepull.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_deadRun_restoresTableAndHandsCapturedDeletesToBasepull() throws QException
|
||||
{
|
||||
insertIndexRow(true);
|
||||
updateIndexRow(AbstractIndexingStep.STATUS_REBUILDING, Instant.now().minus(5, ChronoUnit.MINUTES));
|
||||
QRecord deadRun = insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 90);
|
||||
insertCapturedDelete("4");
|
||||
insertCapturedDelete("5");
|
||||
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).isEmpty();
|
||||
|
||||
QRecord row = queryIndexRow();
|
||||
assertThat(row.getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(row.getValueInstant("lastBasepullTime")).isEqualTo(deadRun.getValueInstant("startTime"));
|
||||
assertThat(row.getValueString("lastRunStatus")).isEqualTo("FULL_REINDEX FAILED");
|
||||
assertThat(row.getValueString("lastErrorMessage")).contains("stopped without finishing");
|
||||
|
||||
QRecord run = queryRun(deadRun.getValueInteger("id"));
|
||||
assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED);
|
||||
assertThat(run.getValue("endTime")).isNotNull();
|
||||
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).extracting(event -> event.getValueString("status"))
|
||||
.containsExactly(QuickSearchFailedEvent.STATUS_PENDING, QuickSearchFailedEvent.STATUS_PENDING);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a run started long ago but with a recent heartbeat is alive;
|
||||
** recovery returns it and leaves its table, run record and captured
|
||||
** deletes alone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_liveRun_leavesItsTableAndRowsAlone() throws QException
|
||||
{
|
||||
insertIndexRow(true);
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
QRecord liveRun = insertRunningRun(queryIndexRow().getValueInteger("id"), 180, 1);
|
||||
insertCapturedDelete("4");
|
||||
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).extracting(run -> run.getValueInteger("id"))
|
||||
.containsExactly(liveRun.getValueInteger("id"));
|
||||
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
assertThat(queryRun(liveRun.getValueInteger("id")).getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_RUNNING);
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).singleElement()
|
||||
.satisfies(event -> assertThat(event.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_AWAITING_REINDEX));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a REBUILDING table whose run record is gone (purged) is recovered
|
||||
** too, without moving its basepull watermark.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_withoutRunRecord_restoresTheTable() throws QException
|
||||
{
|
||||
Instant lastBasepullTime = Instant.now().minus(5, ChronoUnit.MINUTES);
|
||||
insertIndexRow(true);
|
||||
updateIndexRow(AbstractIndexingStep.STATUS_REBUILDING, lastBasepullTime);
|
||||
insertCapturedDelete("4");
|
||||
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).isEmpty();
|
||||
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryIndexRow().getValueInstant("lastBasepullTime")).isEqualTo(lastBasepullTime);
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).singleElement()
|
||||
.satisfies(event -> assertThat(event.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: recovery hands over only the captured rows that existed when it
|
||||
** read them; a row captured after (by a run that started meanwhile) stays
|
||||
** AWAITING_REINDEX for that run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_handsOverOnlyRowsCapturedBeforeItRead() throws QException
|
||||
{
|
||||
insertIndexRow(true);
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 120);
|
||||
insertCapturedDelete("4");
|
||||
|
||||
AtomicBoolean capturedMeanwhile = new AtomicBoolean(false);
|
||||
new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
|
||||
{
|
||||
super.updateIndexRow(indexId, values);
|
||||
if(capturedMeanwhile.compareAndSet(false, true))
|
||||
{
|
||||
insertCapturedDelete("5");
|
||||
}
|
||||
}
|
||||
}.recoverDeadRun(true);
|
||||
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).extracting(event -> event.getValueString("recordId") + " " + event.getValueString("status"))
|
||||
.containsExactlyInAnyOrder("4 " + QuickSearchFailedEvent.STATUS_PENDING, "5 " + QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a recovery that fails while putting a table back leaves the dead
|
||||
** run's record RUNNING, so the next recovery still finds the run's start
|
||||
** and rewinds the basepull watermark.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_failsPartway_nextRecoveryStillRewinds() throws QException
|
||||
{
|
||||
insertIndexRow(true);
|
||||
updateIndexRow(AbstractIndexingStep.STATUS_REBUILDING, Instant.now().minus(5, ChronoUnit.MINUTES));
|
||||
QRecord deadRun = insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 120);
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
|
||||
{
|
||||
throw (new QException("database unavailable"));
|
||||
}
|
||||
}.recoverDeadRun(true))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("database unavailable");
|
||||
|
||||
assertThat(queryRun(deadRun.getValueInteger("id")).getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_RUNNING);
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).isEmpty();
|
||||
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryIndexRow().getValueInstant("lastBasepullTime")).isEqualTo(deadRun.getValueInstant("startTime"));
|
||||
assertThat(queryRun(deadRun.getValueInteger("id")).getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: a run whose drain of captured deletes is so slow that it is judged
|
||||
** stopped, while a new full reindex starts, marks the table and captures a
|
||||
** delete of its own: the drain stops at its next batch and leaves both its
|
||||
** remaining row and the new run's row alone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDrainTakenOverMidway_leavesTheNewRunsRowsAlone() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(1);
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("8");
|
||||
insertCapturedDelete("9");
|
||||
|
||||
AtomicBoolean newRunStarted = new AtomicBoolean(false);
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
if(newRunStarted.compareAndSet(false, true))
|
||||
{
|
||||
for(QRecord run : queryAll(QuickSearchIndexRun.TABLE_NAME))
|
||||
{
|
||||
backdateRun(run.getValueInteger("id"));
|
||||
}
|
||||
assertThat(new FullReindexStep().recoverDeadRun(false)).isEmpty();
|
||||
insertRunningRun(queryIndexRow().getValueInteger("id"), 0, 0);
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
insertCapturedDelete("7");
|
||||
}
|
||||
return (new BulkIndexResult().withSuccessCount(1));
|
||||
});
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, times(1)).deleteDocuments(anyList(), anyInt());
|
||||
verify(mockClient).deleteDocuments(eq(List.of(TEST_ENTITY_TABLE + ":8")), anyInt());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).extracting(event -> event.getValueString("recordId") + " " + event.getValueString("status"))
|
||||
.containsExactlyInAnyOrder("9 " + QuickSearchFailedEvent.STATUS_AWAITING_REINDEX, "7 " + QuickSearchFailedEvent.STATUS_AWAITING_REINDEX);
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the drain applies only rows up to the newest one at its start; a
|
||||
** row captured while it runs is left alone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDrain_stopsAtTheNewestRowReadAtItsStart() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(1);
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("8");
|
||||
insertCapturedDelete("9");
|
||||
|
||||
AtomicBoolean capturedMeanwhile = new AtomicBoolean(false);
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
if(capturedMeanwhile.compareAndSet(false, true))
|
||||
{
|
||||
insertCapturedDelete("7");
|
||||
}
|
||||
return (new BulkIndexResult().withSuccessCount(1));
|
||||
});
|
||||
|
||||
new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
verify(mockClient, times(2)).deleteDocuments(anyList(), anyInt());
|
||||
assertThat(queryAll(QuickSearchFailedEvent.TABLE_NAME)).singleElement()
|
||||
.satisfies(event -> assertThat(event.getValueString("recordId")).isEqualTo("7"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when the stale run records cannot be marked FAILED (every update
|
||||
** comes back with a per-row error), recovery fails and the new full
|
||||
** reindex does not start, since a live run misjudged as stopped would not
|
||||
** learn it was taken over.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_failedRunRecordsNotWritten_newRunDoesNotStart() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
insertIndexRow(true);
|
||||
updateIndexRowStatus(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
Integer staleRunId = insertRunningRun(queryIndexRow().getValueInteger("id"), 120, 120).getValueInteger("id");
|
||||
|
||||
try(MockedConstruction<UpdateAction> updateActions = mockConstruction(UpdateAction.class, (updateAction, context) ->
|
||||
when(updateAction.execute(any())).thenAnswer(invocation ->
|
||||
{
|
||||
UpdateInput updateInput = invocation.getArgument(0);
|
||||
UpdateOutput updateOutput = new UpdateOutput();
|
||||
updateOutput.setRecords(updateInput.getRecords().stream().map(record -> new QRecord(record).withError(new SystemErrorStatusMessage("row is locked"))).toList());
|
||||
return (updateOutput);
|
||||
})))
|
||||
{
|
||||
assertThatThrownBy(() -> new FullReindexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("FAILED")
|
||||
.hasMessageContaining("row is locked");
|
||||
}
|
||||
|
||||
verify(mockClient, never()).createPhysicalIndex(anyString());
|
||||
verify(mockClient, never()).indexDocuments(anyString(), anyList(), anyInt());
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).singleElement()
|
||||
.satisfies(run ->
|
||||
{
|
||||
assertThat(run.getValueInteger("id")).isEqualTo(staleRunId);
|
||||
assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_RUNNING);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: the run keeps itself alive at every point where it could otherwise
|
||||
** go quiet: after each page, right before its pre-swap ownership check,
|
||||
** before each table's update after the swap, and before each batch of the
|
||||
** captured-delete drain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAllTables_heartbeatsBeforeTheOwnershipCheckEachTableAndEachDrainBatch() throws QException
|
||||
{
|
||||
QuickSearchQBitContext.getConfig().withSourceBatchSize(1);
|
||||
insertTestEntities(1);
|
||||
insertCapturedDelete("8");
|
||||
insertCapturedDelete("9");
|
||||
|
||||
List<String> events = new ArrayList<>();
|
||||
doAnswer(invocation ->
|
||||
{
|
||||
events.add("refresh");
|
||||
return (null);
|
||||
}).when(mockClient).refreshIndex(PHYSICAL_INDEX);
|
||||
doAnswer(invocation ->
|
||||
{
|
||||
events.add("swap");
|
||||
return (null);
|
||||
}).when(mockClient).swapAliasTo(PHYSICAL_INDEX);
|
||||
when(mockClient.deleteDocuments(anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
events.add("drainBatch");
|
||||
return (new BulkIndexResult().withSuccessCount(1));
|
||||
});
|
||||
|
||||
new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected void heartbeat(QuickSearchIndexRun run)
|
||||
{
|
||||
events.add("heartbeat");
|
||||
super.heartbeat(run);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
|
||||
{
|
||||
if(values.containsKey("lastFullReindexTime"))
|
||||
{
|
||||
events.add("tableDone");
|
||||
}
|
||||
super.updateIndexRow(indexId, values);
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
|
||||
assertThat(events).containsExactly(
|
||||
"heartbeat",
|
||||
"refresh", "heartbeat", "swap",
|
||||
"heartbeat", "tableDone",
|
||||
"heartbeat", "drainBatch", "heartbeat", "drainBatch", "heartbeat");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: when a run fails and cannot read whether a recovery took it over,
|
||||
** it leaves its run records RUNNING and its tables REBUILDING, so the next
|
||||
** recovery (once the records are stale) puts everything back, watermark
|
||||
** rewind included.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testFailure_takeoverUnreadable_leavesRunRecordsRunningForRecovery() throws QException
|
||||
{
|
||||
insertTestEntities(1);
|
||||
AtomicBoolean databaseDown = new AtomicBoolean(false);
|
||||
when(mockClient.indexDocuments(anyString(), anyList(), anyInt())).thenAnswer(invocation ->
|
||||
{
|
||||
databaseDown.set(true);
|
||||
throw (new QException("bulk failed"));
|
||||
});
|
||||
|
||||
assertThatThrownBy(() -> new FullReindexStep()
|
||||
{
|
||||
@Override
|
||||
protected QuickSearchQBitConfig getConfig() throws QException
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// only the first read after the failure (the takeover check) is //
|
||||
// lost; everything after it would work //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
if(databaseDown.getAndSet(false))
|
||||
{
|
||||
throw (new QException("database unavailable"));
|
||||
}
|
||||
return (super.getConfig());
|
||||
}
|
||||
}.run(new RunBackendStepInput(), new RunBackendStepOutput()))
|
||||
.isInstanceOf(QException.class)
|
||||
.hasMessageContaining("bulk failed");
|
||||
|
||||
verify(mockClient).deletePhysicalIndex(PHYSICAL_INDEX);
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_REBUILDING);
|
||||
QRecord run = queryAll(QuickSearchIndexRun.TABLE_NAME).get(0);
|
||||
assertThat(run.getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_RUNNING);
|
||||
|
||||
backdateRun(run.getValueInteger("id"));
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).isEmpty();
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryRun(run.getValueInteger("id")).getValueString("status")).isEqualTo(AbstractIndexingStep.RUN_FAILED);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test: with nothing left behind, recovery changes nothing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRecovery_nothingLeftBehind_changesNothing() throws QException
|
||||
{
|
||||
insertIndexRow(true);
|
||||
|
||||
assertThat(new FullReindexStep().recoverDeadRun(true)).isEmpty();
|
||||
|
||||
assertThat(queryIndexRow().getValueString("status")).isEqualTo(AbstractIndexingStep.STATUS_ACTIVE);
|
||||
assertThat(queryIndexRow().getValueString("lastRunStatus")).isNull();
|
||||
assertThat(queryAll(QuickSearchIndexRun.TABLE_NAME)).isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user