diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 76822fb..c59db60 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -21,7 +21,38 @@ "mcp__github__get_file_contents", "Bash(mvn dependency:tree:*)", "Bash(mvn help:effective-pom:*)", - "Bash(git checkout:*)" + "Bash(git checkout:*)", + "Bash(ls /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/docs/*.md)", + "Bash(git:*)", + "Bash(ls:*)", + "Bash(open:*)", + "Bash(WT=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/.claude/worktrees/agent-a9dc6e8e)", + "Bash(DEST=/Users/james.maes/Git.Local/QRun-IO/qbit-quick-search)", + "Bash(cp \"$WT/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java\" \"$DEST/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/\")", + "Bash(mvn clean:*)", + "Bash(jar tf:*)", + "Bash(unzip -p /Users/james.maes/.m2/repository/com/kingsrook/qqq/qqq-backend-core/0.40.0-SNAPSHOT/qqq-backend-core-0.40.0-SNAPSHOT.jar com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java)", + "Bash(sed -i '' 's/postInsertRecord/postInsert/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java)", + "Bash(sed -i '' 's/postUpdateRecord/postUpdate/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java)", + "Bash(sed -i '' 's/postDeleteRecord/postDelete/g' src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java)", + "Bash(docker info:*)", + "Bash(docker run:*)", + "Read(//var/run/**)", + "Read(//Users/james.maes/**)", + "Bash(echo \"DOCKER_HOST=$DOCKER_HOST\")", + "Bash(echo \"TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=$TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE\")", + "Bash(DOCKER_HOST=unix:///var/run/docker.sock TESTCONTAINERS_RYUK_DISABLED=true mvn verify -Dit.test=OpenSearchIntegrationTest)", + "Bash(docker version:*)", + "Bash(DOCKER_HOST=unix:///var/run/docker.sock mvn verify -Dit.test=OpenSearchIntegrationTest)", + "Bash(curl -s --unix-socket /var/run/docker.sock http://localhost/v1.24/info)", + "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)} OS: {d.get\\(OperatingSystem,?\\)}''''\\)\")", + "Bash(curl -s --unix-socket /Users/james.maes/Library/Containers/com.docker.docker/Data/docker-cli.sock http://localhost/v1.47/info)", + "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)} OS: {d.get\\(OperatingSystem,?\\)} Containers: {d.get\\(Containers,?\\)}''''\\)\")", + "Bash(curl -s --unix-socket /Users/james.maes/.docker/run/docker.sock http://localhost/v1.47/info)", + "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(f''''ServerVersion: {d.get\\(ServerVersion,?\\)} NCPU: {d.get\\(NCPU,?\\)} OSType: {d.get\\(OSType,?\\)}''''\\)\")", + "Bash(DOCKER_HOST=unix:///Users/james.maes/.docker/run/docker.sock DOCKER_API_VERSION=1.47 mvn verify -Dit.test=OpenSearchIntegrationTest -pl .)", + "Bash(cp /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/LICENSE /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/LICENSE)", + "Bash(cp -r /Users/james.maes/Git.Local/QRun-IO/qbit-webhooks/checkstyle/* /Users/james.maes/Git.Local/QRun-IO/qbit-quick-search/checkstyle/)" ] } } diff --git a/.gitignore b/.gitignore index 9a52950..c2669c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,20 @@ -# Maven target/ +.env +.idea +*.iml +*.class +*.log *.jar *.war +*.nar *.ear - -# IDE -.idea/ -*.iml -.vscode/ -.project -.classpath -.settings/ - -# OS +*.zip +*.tar.gz +*.rar +hs_err_pid* .DS_Store -Thumbs.db - -# Logs -*.log +*.sw* +.flattened-pom.xml +dependency-reduced-pom.xml +/.env.local +/.cache/ diff --git a/CLAUDE.md b/CLAUDE.md index 4fb838f..1f835e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,42 +1,76 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file provides guidance to Claude Code when working with code in this repository. ## Project Overview -This is a QQQ Extension QBit template - a Maven-based Java project for creating infrastructure extensions to the QQQ low-code framework. Extension QBits provide customizers, action handlers, and other framework-level enhancements. +**QBit Quick Search** - A QQQ QBit providing OpenSearch-backed full-text search across QQQ applications. Current version: 0.1.0. ## Build Commands ```bash mvn compile # Compile -mvn test # Run all tests +mvn test # Run all tests (174 tests) mvn test -Dtest=ClassName # Run single test class mvn package # Build JAR +mvn verify # Full verify with coverage check ``` +Integration tests (require Docker) run via `mvn verify` using maven-failsafe-plugin. +Unit tests exclude `*IntegrationTest.java` and `*IT.java`. + ## Architecture -**QBit Producer Pattern**: The entry point is `*QBitProducer` which implements `QBitProducer` interface: -- `produce(QInstance, namespace)` - Registers the QBit and applies customizations -- Uses fluent `withConfig()` to accept configuration +**Entry point:** `QuickSearchQBitProducer.produce(QInstance)` validates config, discovers annotated entity classes, initializes the singleton `QuickSearchOpenSearchClient`, creates an `IndexEventPublisher`, registers all metadata into the `QInstance`, and attaches customizers to source tables. -**QBit Config Pattern**: Configuration via `*QBitConfig` implementing `QBitConfig`: -- `validate(QInstance, errors)` - Validates dependencies exist before production -- Fluent setters (`withX()`) for all properties +**Key Classes:** +- `QuickSearchQBitProducer` - Orchestrator: produces tables, processes, app, and registers customizers +- `QuickSearchQBitConfig` - Configuration (backend name, OpenSearch connection, feature flags) +- `QuickSearchQBitContext` - Static singleton holding the live config, client, publisher, and discovered table configs +- `QuickSearchOpenSearchClient` - All OpenSearch operations (ensure index, bulk index, delete, search) +- `QuickSearchableTableConfig` - Runtime config for a single indexed table (fields, weights, basepull settings) -**Customizers**: Classes in `customizers/` package modify existing QQQ tables/processes. +**Customizers** (`customizers/` package, all three registered when `enableRealTimeIndexing = true`): +- `QuickSearchPostInsertCustomizer` - Fires after insert, publishes index events +- `QuickSearchPostUpdateCustomizer` - Fires after update, publishes index events +- `QuickSearchPostDeleteCustomizer` - Fires after delete, publishes delete events + +**Processes** (`processes/` package): +- `AbstractIndexingStep` - Base class with shared indexing logic (query source table, build documents, publish events, update run record) +- `BasepullIndexStep` - Queries records modified since last run timestamp +- `FullReindexStep` - Wipes and rebuilds the index for one or all tables +- `IndexingUtils` - Converts `QRecord` values to indexed text using field configs + +**Publisher** (`publisher/` package): +- `IndexEventPublisher` - Strategy interface with `publishIndexEvents()`, `publishDeleteEvents()`, `close()` +- `SynchronousIndexEventPublisher` - Default implementation; processes events immediately via the OpenSearch client +- `IndexEvent` / `IndexEventAction` - Event model + +**Actions** (`actions/` package): +- `QuickSearchAction` - Executes a paginated multi-match search across all indexed fields with per-field boosting +- `QuickSearchInput` / `QuickSearchOutput` / `QuickSearchResult` - Search API model + +**Tables Produced:** +- `quickSearchIndex` - One row per indexed table; tracks enabled state, basepull interval, last run times +- `quickSearchIndexRun` - Run history with status, record counts, and error messages + +**Annotation Discovery:** +- `@QuickSearchable(tableName = "...")` - Mark entity classes; defines basepull settings +- `@QuickSearchField(weight, includeLabel)` - Mark fields to include in indexed text; takes precedence over `fields[]` array ## Code Style -- 3-space indentation -- Opening braces on next line +- 3-space indentation, opening braces on next line - Javadoc flower box comments (80-char width with `**` borders) -- Fluent-style setters (`withX()`) over traditional setters +- Fluent-style setters (`withX()`) - Use wrapper types (`Integer`, `Boolean`) not primitives ## Dependencies - Java 21 -- QQQ Backend Core (see `qqq.version` in pom.xml) -- JUnit 5 + AssertJ for testing +- QQQ Backend Core 0.40.0-SNAPSHOT (via `qqq-bom-pom`) +- OpenSearch Java Client 2.10.0 +- Apache HttpClient5 5.3 +- jackson-datatype-jsr310 2.21.0 (Instant serialization for OpenSearch client) +- JUnit 5 + AssertJ + Mockito for testing +- Testcontainers for integration tests (requires Docker) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..8e5d650 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +See Project level [Code of Conduct](https://github.com/Kingsrook/qqq/blob/develop/CODE_OF_CONDUCT.md) diff --git a/CODE_STYLE.md b/CODE_STYLE.md new file mode 100644 index 0000000..f96ac53 --- /dev/null +++ b/CODE_STYLE.md @@ -0,0 +1,3 @@ +# Code Style Guide + +* See Project [Code Style Guide](https://github.com/Kingsrook/qqq/blob/develop/CODE_STYLE.md) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0a5be02 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Contributing to QBit Quick Search + +* See Project [Contributing](https://github.com/Kingsrook/qqq/blob/develop/CONTRIBUTING.md) diff --git a/LICENSE b/LICENSE index b3f61de..bf20455 100644 --- a/LICENSE +++ b/LICENSE @@ -1,191 +1,680 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +QQQ - Low-code Application Framework for Engineers +Copyright (C) 2021-2025 QRun-IO, LLC +https://www.qrun.io | https://github.com/QRun-IO - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - 1. Definitions. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +-------------------------------------------------------------------------------- - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. + Preamble - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to the Licensor for inclusion in the Work by the copyright - owner or by an individual or Legal Entity authorized to submit on - behalf of the copyright owner. For the purposes of this definition, - "submitted" means any form of electronic, verbal, or written - communication sent to the Licensor or its representatives, including - but not limited to communication on electronic mailing lists, source - code control systems, and issue tracking systems that are managed by, - or on behalf of, the Licensor for the purpose of discussing and - improving the Work, but excluding communication that is conspicuously - marked or otherwise designated in writing by the copyright owner as - "Not a Contribution." + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: + The precise terms and conditions for copying, distribution and +modification follow. - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and + TERMS AND CONDITIONS - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and + 0. Definitions. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and + "This License" refers to version 3 of the GNU Affero General Public License. - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. + A "covered work" means either the unmodified Program or a work based +on the Program. - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. - END OF TERMS AND CONDITIONS + 1. Source Code. - Copyright 2022-2025 QRunIO, LLC + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. - http://www.apache.org/licenses/LICENSE-2.0 + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/NOTICE b/NOTICE index 29ff097..8fbf195 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,6 @@ -qBit Template - Extension -Copyright 2022-2025 QRunIO, LLC +QBit Quick Search +Copyright 2024-2025 QRunIO, LLC This product includes software developed at QRunIO, LLC (https://qrun.io/). -Licensed under the Apache License, Version 2.0. +Licensed under the GNU Affero General Public License, Version 3. diff --git a/README.md b/README.md index 15ea3b1..3879721 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,240 @@ -# QBit Template: Extension +# QBit: Quick Search -Template repository for creating QQQ Extension QBits - infrastructure extensions, customizers, and action handlers. +[![Version](https://img.shields.io/badge/version-0.1.0-blue.svg)](https://github.com/QRun-IO/qbit-quick-search) +[![License](https://img.shields.io/badge/license-GNU%20Affero%20GPL%20v3-green.svg)](https://www.gnu.org/licenses/agpl-3.0.en.html) +[![Java](https://img.shields.io/badge/java-21+-blue.svg)](https://adoptium.net/) -## What is an Extension QBit? +> **OpenSearch-Powered Full-Text Search for QQQ Applications** -Extension QBits extend or enhance how QQQ operates at the infrastructure/framework level. They provide: -- New backend implementations -- Authentication mechanisms -- Audit capabilities -- Table customizers -- Action handlers +This QBit provides global search across QQQ application tables via OpenSearch. Annotate entity classes, configure a connection, produce the QBit, and gain real-time indexing, scheduled basepull catch-up, per-field relevance boosting, and a pagination-aware search API. -## Quick Start +## Core Capabilities -1. Click "Use this template" to create your QBit -2. Run `scripts/customize_template.py` to rename the example -3. Implement your extension logic -4. Add tests -5. Publish to Maven repository +- **Real-Time Indexing**: Post-insert, post-update, and post-delete customizers index records immediately on write +- **Basepull Safety Net**: Scheduled process catches records missed during downtime or bulk imports +- **Field-Level Boosting**: `@QuickSearchField(weight = N)` controls relevance ranking per field +- **Edge-Ngram Typeahead**: Custom analyzer enables prefix matching ("ord" finds "order") +- **Paginated Search**: `QuickSearchAction` returns results with scores, highlights, total hits, and hasMore flag +- **Extensible Publisher**: `IndexEventPublisher` interface allows swapping in RabbitMQ, Kafka, or any async transport +- **Admin Observability**: Operational tables track index status and run history -## Structure +## Open Source & Full Control + +QBit Quick Search is 100% open source under AGPL v3. All data stays in your OpenSearch cluster. + +## Architecture + +### Design Principles + +1. **Real-time first, basepull as safety net**: Table customizers index records synchronously on every insert, update, and delete. The basepull process runs on a schedule as a catch-up mechanism for bulk imports, OpenSearch downtime, or any records the real-time path missed. + +2. **Annotation-driven discovery**: Entity classes marked with `@QuickSearchable` and `@QuickSearchField` are discovered at produce time. No manual registration of fields or tables required. + +3. **Extensible transport**: The `IndexEventPublisher` strategy interface decouples the write path from the indexing path. The default `SynchronousIndexEventPublisher` writes directly to OpenSearch. Consumers can provide a queue-backed implementation for async indexing. + +### Technology Stack + +- **Java 21** with QQQ backend modules +- **OpenSearch 2.x** for full-text search with custom edge-ngram analyzer +- **QQQ Framework**: Entities, processes, customizers, permissions, API layer + +### Module Organization ``` -src/main/java/com/kingsrook/qbits/example/ -├── ExampleExtensionQBitConfig.java # Configuration -├── ExampleExtensionQBitProducer.java # Entry point -├── customizers/ -│ └── ExampleTableCustomizer.java # Table customization -└── actions/ - └── ExampleActionCustomizer.java # Action handling +qbit-quick-search/ + src/main/java/com/kingsrook/qbits/quicksearch/ + QuickSearchQBitConfig.java -- Configuration and validation + QuickSearchQBitProducer.java -- Produces tables, processes, customizers + QuickSearchQBitContext.java -- Static runtime context + QuickSearchableTableConfig.java -- Per-table index configuration + annotations/ -- @QuickSearchable, @QuickSearchField + model/ -- QuickSearchIndex, QuickSearchIndexRun entities + opensearch/ -- OpenSearch client, document model, bulk results + actions/ -- QuickSearchAction, input/output DTOs + processes/ -- AbstractIndexingStep, BasepullIndexStep, FullReindexStep + customizers/ -- Post-insert, post-update, post-delete customizers + publisher/ -- IndexEventPublisher interface, SynchronousIndexEventPublisher ``` -## Key Characteristics +## Getting Started -- **No QAppSection** - Pure infrastructure -- **No user-facing tables** (or minimal operational) -- **Wraps existing tables/processes** with new behavior -- **Validates dependencies** on other QQQ components +### Prerequisites + +- **Java 21+** +- **Maven 3.8+** +- **OpenSearch 2.x** instance +- **QQQ Application** (this is a QBit, not a standalone application) + +### Usage + +#### Maven dependency + +```xml + + com.kingsrook.qbits + qbit-quick-search + 0.1.0-SNAPSHOT + +``` + +#### Annotate entity classes + +```java +@QuickSearchable(tableName = "customer") +public class Customer +{ + @QuickSearchField(weight = 3) + private String name; + + @QuickSearchField(weight = 1) + private String email; + + @QuickSearchField(weight = 2, includeLabel = true) + private String accountNumber; +} +``` + +#### Minimal setup + +```java +QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("yourBackendName") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("my-app-search") + .withSearchableEntityClasses(List.of(Customer.class, Order.class)); + +new QuickSearchQBitProducer() + .withConfig(config) + .produce(qInstance); +``` + +#### With authentication and SSL + +```java +QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("yourBackendName") + .withOpensearchHost("search.example.com") + .withOpensearchPort(443) + .withUseSsl(true) + .withOpensearchUsername("admin") + .withOpensearchPassword("secretPassword") + .withOpensearchIndexName("my-app-search") + .withSearchableEntityClasses(List.of(Customer.class)); +``` + +#### With custom publisher (async indexing) + +```java +QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("yourBackendName") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("my-app-search") + .withSearchableEntityClasses(List.of(Customer.class)) + .withIndexEventPublisher(new RabbitMqIndexEventPublisher(connectionFactory)); +``` + +## Data Model + +### Tables (2) + +| Table | Description | +|-------|-------------| +| `quickSearchIndex` | Per-table index configuration, status, and basepull tracking | +| `quickSearchIndexRun` | Individual indexing run history with record counts and error tracking | + +### Processes (2) + +| Process | Description | +|---------|-------------| +| Basepull Index | Scheduled: queries each source table for records modified since last run, indexes in batches | +| Full Reindex | On-demand: deletes and re-indexes all records for one or all tables | + +## Configuration Reference + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `backendName` | `String` | required | QQQ backend for operational tables | +| `opensearchHost` | `String` | required | OpenSearch hostname | +| `opensearchPort` | `Integer` | required | OpenSearch port | +| `opensearchIndexName` | `String` | required | OpenSearch index name | +| `searchableEntityClasses` | `List>` | required | Entity classes with `@QuickSearchable` | +| `opensearchUsername` | `String` | none | HTTP basic auth username | +| `opensearchPassword` | `String` | none | HTTP basic auth password | +| `useSsl` | `Boolean` | `false` | Use HTTPS | +| `tableNamePrefix` | `String` | none | Prefix for produced table names | +| `enableRealTimeIndexing` | `Boolean` | `true` | Register post-insert/update/delete customizers | +| `enableScheduledProcesses` | `Boolean` | `true` | Register scheduled basepull process | +| `defaultBasepullIntervalMinutes` | `Integer` | `5` | Default basepull schedule | +| `bulkBatchSize` | `Integer` | `500` | Documents per OpenSearch bulk request | +| `sourceBatchSize` | `Integer` | `1000` | Records per source table query batch | +| `indexEventPublisher` | `IndexEventPublisher` | sync | Custom publisher for async transport | + +## Annotations + +### `@QuickSearchable` + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `tableName` | `String` | required | QQQ table name | +| `fields` | `String[]` | `{}` | Fallback field list if no `@QuickSearchField` annotations | +| `basepullIntervalMinutes` | `int` | `5` | Basepull schedule for this table | +| `basepullTimestampField` | `String` | `"modifyDate"` | Field for change detection | +| `enabledByDefault` | `boolean` | `true` | Active on first produce | + +### `@QuickSearchField` + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `weight` | `int` | `1` | Boost factor for search relevance | +| `includeLabel` | `boolean` | `false` | Prefix value with field label in indexed text | + +## Companion QBit Integration + +### qbit-crm + +Index `crmContact`, `crmCompany`, and `crmDeal` for full-text global search across CRM records. + +### qbit-wms + +Index `wmsItem` and `wmsLocation` for warehouse item and location search. + +## Testing + +```bash +mvn test # Run all 174 unit tests +mvn verify # Run unit tests + integration tests (requires Docker) +mvn test -Dtest=ClassName # Run single test class +``` + +### Coverage + +- **70%+ instruction coverage** (JaCoCo enforced) +- **90%+ class coverage** (JaCoCo enforced) +- All unit tests run in-memory via MemoryRecordStore (no OpenSearch required) +- Integration tests use Testcontainers with real OpenSearch ## Documentation -- [Getting Started](docs/00-getting-started.md) -- [Extension Patterns](docs/01-extension-patterns.md) +- **[QQQ Wiki](https://github.com/Kingsrook/qqq/wiki)** - Framework documentation +- **[QBit Development Guide](https://github.com/Kingsrook/qqq/wiki/QBit-Development)** - How QBits work + +## Contributing + +QBit Quick Search is open source and welcomes contributions. + +- **[Report Issues](https://github.com/QRun-IO/qqq/issues)** - Bug reports and feature requests +- **[QQQ Contribution Guide](https://github.com/Kingsrook/qqq/wiki/Contribution-Guidelines)** - How to contribute + +## About Kingsrook + +QBit Quick Search is built by **[Kingsrook](https://qrun.io)** - making engineers more productive through intelligent automation and developer tools. + +- **Website**: [https://qrun.io](https://qrun.io) +- **Contact**: [contact@kingsrook.com](mailto:contact@kingsrook.com) +- **GitHub**: [https://github.com/QRun-IO](https://github.com/QRun-IO) ## License -AGPL-3.0 - See [LICENSE](LICENSE) +This project is licensed under the **GNU Affero General Public License v3.0** - see the [LICENSE](LICENSE) file for details. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..42e8129 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + + * See Project [Security Policy](https://github.com/Kingsrook/qqq/blob/develop/SECURITY.md) diff --git a/checkstyle/config.xml b/checkstyle/config.xml new file mode 100644 index 0000000..78c2f49 --- /dev/null +++ b/checkstyle/config.xml @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/checkstyle/license.txt b/checkstyle/license.txt new file mode 100644 index 0000000..af39d6f --- /dev/null +++ b/checkstyle/license.txt @@ -0,0 +1,20 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ diff --git a/docs/00-getting-started.md b/docs/00-getting-started.md deleted file mode 100644 index f019f1e..0000000 --- a/docs/00-getting-started.md +++ /dev/null @@ -1,42 +0,0 @@ -# Getting Started - -## Prerequisites - -- Java 17+ -- Maven 3.8+ -- QQQ 0.23.0+ - -## Creating Your Extension QBit - -1. Click "Use this template" on GitHub -2. Clone your new repository -3. Run the customization script: - ```bash - python scripts/customize_template.py - ``` -4. Enter your QBit name when prompted - -## Project Structure - -``` -src/main/java/com/kingsrook/qbits/yourextension/ -├── YourExtensionQBitConfig.java -├── YourExtensionQBitProducer.java -├── customizers/ -└── actions/ -``` - -## Using Your QBit - -```java -new YourExtensionQBitProducer() - .withConfig(new YourExtensionQBitConfig() - .withTargetTableName("order")) - .produce(qInstance, "myExtension"); -``` - -## Next Steps - -- Read [Extension Patterns](01-extension-patterns.md) -- Implement your customization logic -- Add tests diff --git a/docs/01-extension-patterns.md b/docs/01-extension-patterns.md deleted file mode 100644 index 5c90fdc..0000000 --- a/docs/01-extension-patterns.md +++ /dev/null @@ -1,58 +0,0 @@ -# Extension Patterns - -## Table Customizers - -Add behavior to existing tables: - -```java -public class AuditTableCustomizer -{ - public void customize(QTableMetaData table) - { - table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME)); - table.withField(new QFieldMetaData("modifiedAt", QFieldType.DATE_TIME)); - - table.withCustomizer( - TableCustomizers.PRE_INSERT_RECORD, - new QCodeReference(AuditPreInsert.class)); - } -} -``` - -## Action Customizers - -Wrap actions with additional behavior: - -```java -public class LoggingActionWrapper implements PreInsertRecordInterface -{ - @Override - public void preInsertRecord(InsertInput input, QRecord record) throws QException - { - LOG.info("Inserting record", logPair("table", input.getTableName())); - } -} -``` - -## Config Validation - -Validate that required components exist: - -```java -@Override -public void validate(QInstance qInstance, List errors) -{ - if(qInstance.getTable(targetTableName) == null) - { - errors.add("Target table required: " + targetTableName); - } -} -``` - -## When to Use Extension QBits - -- Adding audit logging to tables -- Implementing authentication/authorization -- Adding encryption to fields -- Custom validation rules -- Performance monitoring diff --git a/pom.xml b/pom.xml index 403769f..e61f946 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ - 0.1.0 + 0.1.0-SNAPSHOT 21 21 2.10.0 @@ -43,8 +43,8 @@ - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0 + GNU Affero General Public License, Version 3 + https://www.gnu.org/licenses/agpl-3.0.html repo @@ -91,12 +91,6 @@ qqq-backend-core - - com.kingsrook.qqq - qqq-middleware-javalin - true - - org.opensearch.client opensearch-java @@ -112,7 +106,7 @@ com.fasterxml.jackson.datatype jackson-datatype-jsr310 - 2.15.3 + 2.21.0 @@ -153,13 +147,13 @@ org.testcontainers testcontainers - 1.19.3 + 1.21.0 test org.testcontainers junit-jupiter - 1.19.3 + 1.21.0 test @@ -207,10 +201,6 @@ check - - - - BUNDLE @@ -218,12 +208,12 @@ INSTRUCTION COVEREDRATIO - 0.30 + 0.70 CLASS COVEREDRATIO - 0.80 + 0.90 diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfig.java b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfig.java index b895940..dfccdba 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfig.java @@ -24,26 +24,47 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* ** Configuration for the Quick Search QBit. + ** + ** Required fields: backendName, opensearchHost, opensearchPort, + ** opensearchIndexName, searchableEntityClasses. + ** + ** Optional: opensearchUsername and opensearchPassword (must be set together), + ** tableNamePrefix, useSsl (default false), enableScheduledProcesses (default + ** true), enableRealTimeIndexing (default true), defaultBasepullIntervalMinutes + ** (default 5), bulkBatchSize (default 500), sourceBatchSize (default 1000). + ** + ** Note: indexEventPublisher is typed as Object until the IndexEventPublisher + ** interface is created in Task 7, at which point it will be properly typed. *******************************************************************************/ public class QuickSearchQBitConfig implements QBitConfig { - private String backendName; - private String tableNamePrefix; - private String opensearchHost; - private Integer opensearchPort; - private String opensearchIndexName; - private String opensearchUsername; - private String opensearchPassword; - private Boolean useSsl; - private Boolean autoDiscoverAnnotations; - private Boolean enableScheduledProcesses; - private Integer defaultBasepullIntervalMinutes; + private String backendName; + private String tableNamePrefix; + private String opensearchHost; + private Integer opensearchPort; + private String opensearchIndexName; + private String opensearchUsername; + private String opensearchPassword; + private Boolean useSsl = false; + private Boolean autoDiscoverAnnotations; + private Boolean enableScheduledProcesses = true; + private Boolean enableRealTimeIndexing = true; + private Integer defaultBasepullIntervalMinutes = 5; + private Integer bulkBatchSize = 500; + private Integer sourceBatchSize = 1000; private List> searchableEntityClasses; + ///////////////////////////////////////////////////////////////////////////// + // Typed as Object until IndexEventPublisher interface exists (see Task 7) // + ///////////////////////////////////////////////////////////////////////////// + private Object indexEventPublisher; + /*************************************************************************** ** Validate configuration before QBit is produced. + ** + ** Collects all validation errors into the provided errors list. ***************************************************************************/ @Override public void validate(QInstance qInstance, List errors) @@ -66,25 +87,63 @@ public class QuickSearchQBitConfig implements QBitConfig { errors.add("opensearchPort is required for QuickSearchQBit"); } + else if(opensearchPort <= 0) + { + errors.add("opensearchPort must be positive for QuickSearchQBit"); + } if(!StringUtils.hasContent(opensearchIndexName)) { errors.add("opensearchIndexName is required for QuickSearchQBit"); } + + boolean hasUsername = StringUtils.hasContent(opensearchUsername); + boolean hasPassword = StringUtils.hasContent(opensearchPassword); + if(hasUsername && !hasPassword) + { + errors.add("opensearchPassword is required when opensearchUsername is set"); + } + else if(!hasUsername && hasPassword) + { + errors.add("opensearchUsername is required when opensearchPassword is set"); + } + + if(searchableEntityClasses == null || searchableEntityClasses.isEmpty()) + { + errors.add("searchableEntityClasses is required for QuickSearchQBit"); + } + + if(bulkBatchSize != null && bulkBatchSize <= 0) + { + errors.add("bulkBatchSize must be positive for QuickSearchQBit"); + } + + if(sourceBatchSize != null && sourceBatchSize <= 0) + { + errors.add("sourceBatchSize must be positive for QuickSearchQBit"); + } + + if(defaultBasepullIntervalMinutes != null && defaultBasepullIntervalMinutes <= 0) + { + errors.add("defaultBasepullIntervalMinutes must be positive for QuickSearchQBit"); + } } /*************************************************************************** ** Apply table name prefix if configured. + ** + ** If tableNamePrefix is non-null and non-empty, returns prefix + name. + ** Otherwise returns name unchanged. ***************************************************************************/ - public String applyPrefix(String tableName) + public String applyPrefix(String name) { if(StringUtils.hasContent(tableNamePrefix)) { - return tableNamePrefix + tableName; + return tableNamePrefix + name; } - return tableName; + return name; } @@ -399,6 +458,37 @@ public class QuickSearchQBitConfig implements QBitConfig + /*************************************************************************** + ** Getter for enableRealTimeIndexing + ***************************************************************************/ + public Boolean getEnableRealTimeIndexing() + { + return (this.enableRealTimeIndexing); + } + + + + /*************************************************************************** + ** Setter for enableRealTimeIndexing + ***************************************************************************/ + public void setEnableRealTimeIndexing(Boolean enableRealTimeIndexing) + { + this.enableRealTimeIndexing = enableRealTimeIndexing; + } + + + + /*************************************************************************** + ** Fluent setter for enableRealTimeIndexing + ***************************************************************************/ + public QuickSearchQBitConfig withEnableRealTimeIndexing(Boolean enableRealTimeIndexing) + { + this.enableRealTimeIndexing = enableRealTimeIndexing; + return (this); + } + + + /*************************************************************************** ** Getter for defaultBasepullIntervalMinutes ***************************************************************************/ @@ -430,6 +520,68 @@ public class QuickSearchQBitConfig implements QBitConfig + /*************************************************************************** + ** Getter for bulkBatchSize + ***************************************************************************/ + public Integer getBulkBatchSize() + { + return (this.bulkBatchSize); + } + + + + /*************************************************************************** + ** Setter for bulkBatchSize + ***************************************************************************/ + public void setBulkBatchSize(Integer bulkBatchSize) + { + this.bulkBatchSize = bulkBatchSize; + } + + + + /*************************************************************************** + ** Fluent setter for bulkBatchSize + ***************************************************************************/ + public QuickSearchQBitConfig withBulkBatchSize(Integer bulkBatchSize) + { + this.bulkBatchSize = bulkBatchSize; + return (this); + } + + + + /*************************************************************************** + ** Getter for sourceBatchSize + ***************************************************************************/ + public Integer getSourceBatchSize() + { + return (this.sourceBatchSize); + } + + + + /*************************************************************************** + ** Setter for sourceBatchSize + ***************************************************************************/ + public void setSourceBatchSize(Integer sourceBatchSize) + { + this.sourceBatchSize = sourceBatchSize; + } + + + + /*************************************************************************** + ** Fluent setter for sourceBatchSize + ***************************************************************************/ + public QuickSearchQBitConfig withSourceBatchSize(Integer sourceBatchSize) + { + this.sourceBatchSize = sourceBatchSize; + return (this); + } + + + /*************************************************************************** ** Getter for searchableEntityClasses ***************************************************************************/ @@ -459,4 +611,41 @@ public class QuickSearchQBitConfig implements QBitConfig return (this); } + + + /*************************************************************************** + ** Getter for indexEventPublisher + ** + ** Typed as Object until IndexEventPublisher interface exists (Task 7). + ***************************************************************************/ + public Object getIndexEventPublisher() + { + return (this.indexEventPublisher); + } + + + + /*************************************************************************** + ** Setter for indexEventPublisher + ** + ** Typed as Object until IndexEventPublisher interface exists (Task 7). + ***************************************************************************/ + public void setIndexEventPublisher(Object indexEventPublisher) + { + this.indexEventPublisher = indexEventPublisher; + } + + + + /*************************************************************************** + ** Fluent setter for indexEventPublisher + ** + ** Typed as Object until IndexEventPublisher interface exists (Task 7). + ***************************************************************************/ + public QuickSearchQBitConfig withIndexEventPublisher(Object indexEventPublisher) + { + this.indexEventPublisher = indexEventPublisher; + return (this); + } + } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContext.java b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContext.java index 8a46e38..d088a84 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContext.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContext.java @@ -20,13 +20,32 @@ import java.util.List; /******************************************************************************* - ** Static context for Quick Search QBit runtime state. + ** Static holder for Quick Search QBit runtime state. + ** + ** Provides access to the active configuration, the OpenSearch client, the + ** index-event publisher, and the list of tables discovered at startup. + ** + ** Note: client is typed as Object until QuickSearchOpenSearchClient is + ** wired in; publisher is typed as Object until IndexEventPublisher is + ** created in Task 7. + ** + ** Call clear() in test @AfterEach / @BeforeEach to reset all state. *******************************************************************************/ public class QuickSearchQBitContext { - private static QuickSearchQBitConfig config; + private static QuickSearchQBitConfig config; private static List discoveredTables; + ///////////////////////////////////////////////////////////////////////// + // Typed as Object until QuickSearchOpenSearchClient is set here. // + ///////////////////////////////////////////////////////////////////////// + private static Object client; + + ///////////////////////////////////////////////////////////////////////// + // Typed as Object until IndexEventPublisher interface exists (Task 7) // + ///////////////////////////////////////////////////////////////////////// + private static Object publisher; + /*************************************************************************** @@ -67,4 +86,85 @@ public class QuickSearchQBitContext QuickSearchQBitContext.discoveredTables = discoveredTables; } + + + /*************************************************************************** + ** Find a table config by table name from the discovered tables list. + ** + ** Returns null if discoveredTables is null or no match is found. + ***************************************************************************/ + public static QuickSearchableTableConfig getTableConfig(String tableName) + { + if(discoveredTables == null) + { + return null; + } + + for(QuickSearchableTableConfig tableConfig : discoveredTables) + { + if(tableName.equals(tableConfig.getTableName())) + { + return tableConfig; + } + } + + return null; + } + + + + /*************************************************************************** + ** Getter for client + ***************************************************************************/ + public static Object getClient() + { + return client; + } + + + + /*************************************************************************** + ** Setter for client + ***************************************************************************/ + public static void setClient(Object client) + { + QuickSearchQBitContext.client = client; + } + + + + /*************************************************************************** + ** Getter for publisher + ***************************************************************************/ + public static Object getPublisher() + { + return publisher; + } + + + + /*************************************************************************** + ** Setter for publisher + ***************************************************************************/ + public static void setPublisher(Object publisher) + { + QuickSearchQBitContext.publisher = publisher; + } + + + + /*************************************************************************** + ** Reset all static state to null. + ** + ** Call in test @BeforeEach or @AfterEach to prevent state leakage + ** between tests. + ***************************************************************************/ + public static void clear() + { + config = null; + discoveredTables = null; + client = null; + publisher = null; + } + } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java index 8f3c9ba..522d5b8 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java @@ -17,18 +17,12 @@ package com.kingsrook.qbits.quicksearch; import java.lang.reflect.Field; -import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; -import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField; -import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable; -import com.kingsrook.qbits.quicksearch.customizers.QuickSearchDeleteCustomizer; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; -import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; -import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep; -import com.kingsrook.qbits.quicksearch.processes.FullReindexStep; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -36,411 +30,49 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; -import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField; +import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable; +import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostDeleteCustomizer; +import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostInsertCustomizer; +import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostUpdateCustomizer; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import com.kingsrook.qbits.quicksearch.processes.BasepullIndexStep; +import com.kingsrook.qbits.quicksearch.processes.FullReindexStep; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import com.kingsrook.qbits.quicksearch.publisher.SynchronousIndexEventPublisher; /******************************************************************************* - ** Producer for the Quick Search QBit. + ** Central orchestrator for the Quick Search QBit. + ** + ** Validates config, discovers annotated tables, produces QQQ metadata + ** (tables, processes, possible value sources, app), initializes the + ** OpenSearch client, and registers real-time customizers on source tables. *******************************************************************************/ -public class QuickSearchQBitProducer implements QBitProducer +public class QuickSearchQBitProducer { private static final QLogger LOG = QLogger.getLogger(QuickSearchQBitProducer.class); - public static final String GROUP_ID = "com.kingsrook.qbits"; - public static final String ARTIFACT_ID = "qbit-quick-search"; - public static final String VERSION = "0.1.0"; - public static final String FULL_REINDEX_PROCESS_NAME = "quickSearchFullReindex"; - public static final String BASEPULL_PROCESS_NAME = "quickSearchBasepull"; - public static final String APP_NAME = "quickSearchApp"; + public static final String BASEPULL_PROCESS_NAME = "quickSearchBasepullIndex"; + public static final String FULL_REINDEX_PROCESS_NAME = "quickSearchFullReindex"; + public static final String APP_NAME = "quickSearchAdmin"; private QuickSearchQBitConfig config; - /*************************************************************************** - ** Produce this QBit into the given QInstance. - ***************************************************************************/ - @Override - public void produce(QInstance qInstance, String namespace) throws QException - { - ///////////////////////////// - // Validate configuration // - ///////////////////////////// - List errors = new ArrayList<>(); - config.validate(qInstance, errors); - if(!errors.isEmpty()) - { - throw new QException("QuickSearch QBit configuration errors: " + String.join(", ", errors)); - } - - /////////////////////////////// - // Register QBit identity // - /////////////////////////////// - QBitMetaData qBitMetaData = new QBitMetaData() - .withGroupId(GROUP_ID) - .withArtifactId(ARTIFACT_ID) - .withVersion(VERSION) - .withNamespace(namespace) - .withConfig(config); - - qInstance.addQBit(qBitMetaData); - - /////////////////////////////////////// - // Discover @QuickSearchable tables // - /////////////////////////////////////// - List discoveredTables = discoverSearchableTables(qInstance); - LOG.info("Discovered searchable tables", logPair("count", discoveredTables.size())); - - /////////////////////////////// - // Produce operational tables // - /////////////////////////////// - producePossibleValueSources(qInstance); - produceQuickSearchIndexTable(qInstance); - produceQuickSearchIndexRunTable(qInstance); - - /////////////////////////////// - // Produce processes // - /////////////////////////////// - produceFullReindexProcess(qInstance); - produceBasepullProcess(qInstance); - - /////////////////////////////// - // Produce app // - /////////////////////////////// - produceApp(qInstance); - - /////////////////////////////// - // Initialize OpenSearch // - /////////////////////////////// - initializeOpenSearch(); - - /////////////////////////////// - // Register delete customizers // - /////////////////////////////// - registerDeleteCustomizers(qInstance, discoveredTables); - - /////////////////////////////// - // Store discovered configs // - /////////////////////////////// - QuickSearchQBitContext.setConfig(config); - QuickSearchQBitContext.setDiscoveredTables(discoveredTables); - } - - - - /*************************************************************************** - ** Discover tables annotated with @QuickSearchable. - ***************************************************************************/ - private List discoverSearchableTables(QInstance qInstance) - { - List result = new ArrayList<>(); - - List> entityClasses = config.getSearchableEntityClasses(); - if(entityClasses == null || entityClasses.isEmpty()) - { - return result; - } - - for(Class entityClass : entityClasses) - { - if(!entityClass.isAnnotationPresent(QuickSearchable.class)) - { - LOG.warn("Entity class not annotated with @QuickSearchable", logPair("class", entityClass.getName())); - continue; - } - - QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class); - String tableName = annotation.tableName(); - - QTableMetaData table = qInstance.getTable(tableName); - if(table == null) - { - LOG.warn("Table not found for searchable entity", logPair("class", entityClass.getName()), logPair("tableName", tableName)); - continue; - } - - QuickSearchableTableConfig tableConfig = buildTableConfig(table, entityClass, annotation); - result.add(tableConfig); - - LOG.debug("Found searchable table", logPair("tableName", table.getName()), logPair("fields", tableConfig.getSearchableFields())); - } - - return result; - } - - - - /*************************************************************************** - ** Build configuration for a single searchable table. - ***************************************************************************/ - private QuickSearchableTableConfig buildTableConfig(QTableMetaData table, Class recordClass, QuickSearchable annotation) - { - List searchableFields = new ArrayList<>(); - - if(annotation.fields().length > 0) - { - searchableFields.addAll(Arrays.asList(annotation.fields())); - } - else - { - for(Field field : recordClass.getDeclaredFields()) - { - if(field.isAnnotationPresent(QuickSearchField.class)) - { - searchableFields.add(field.getName()); - } - } - } - - return new QuickSearchableTableConfig() - .withTableName(table.getName()) - .withSearchableFields(searchableFields) - .withBasepullIntervalMinutes(annotation.basepullIntervalMinutes()) - .withBasepullTimestampField(annotation.basepullTimestampField()) - .withEnabledByDefault(annotation.enabledByDefault()); - } - - - - /*************************************************************************** - ** Produce possible value sources for enum fields. - ***************************************************************************/ - private void producePossibleValueSources(QInstance qInstance) - { - String prefix = config.getTableNamePrefix(); - - qInstance.addPossibleValueSource(new QPossibleValueSource() - .withName(prefix + "quickSearchRunType") - .withType(QPossibleValueSourceType.ENUM) - .withEnumValues(List.of( - new QPossibleValue<>("FULL", "Full Reindex"), - new QPossibleValue<>("BASEPULL", "Basepull") - ))); - - qInstance.addPossibleValueSource(new QPossibleValueSource() - .withName(prefix + "quickSearchRunStatus") - .withType(QPossibleValueSourceType.ENUM) - .withEnumValues(List.of( - new QPossibleValue<>("RUNNING", "Running"), - new QPossibleValue<>("COMPLETE", "Complete"), - new QPossibleValue<>("ERROR", "Error") - ))); - } - - - - /*************************************************************************** - ** Produce the quickSearchIndex table. - ***************************************************************************/ - private void produceQuickSearchIndexTable(QInstance qInstance) - { - String tableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); - - QTableMetaData table = new QTableMetaData() - .withName(tableName) - .withLabel("Quick Search Index") - .withBackendName(config.getBackendName()) - .withPrimaryKeyField("id") - .withRecordLabelFormat("%s") - .withRecordLabelFields(List.of("tableName")) - .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) - .withField(new QFieldMetaData("tableName", QFieldType.STRING).withLabel("Table Name").withIsRequired(true).withIsEditable(false)) - .withField(new QFieldMetaData("isEnabled", QFieldType.BOOLEAN).withLabel("Enabled")) - .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER).withLabel("Basepull Interval (min)")) - .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING).withLabel("Timestamp Field")) - .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.STRING).withLabel("Searchable Fields")) - .withField(new QFieldMetaData("lastFullIndexTime", QFieldType.DATE_TIME).withLabel("Last Full Index").withIsEditable(false)) - .withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME).withLabel("Last Basepull").withIsEditable(false)) - .withField(new QFieldMetaData("indexedRecordCount", QFieldType.INTEGER).withLabel("Record Count").withIsEditable(false)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Create Date").withIsEditable(false)) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Modify Date").withIsEditable(false)); - - qInstance.addTable(table); - } - - - - /*************************************************************************** - ** Produce the quickSearchIndexRun table. - ***************************************************************************/ - private void produceQuickSearchIndexRunTable(QInstance qInstance) - { - String tableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); - String prefix = config.getTableNamePrefix(); - - QTableMetaData table = new QTableMetaData() - .withName(tableName) - .withLabel("Quick Search Index Run") - .withBackendName(config.getBackendName()) - .withPrimaryKeyField("id") - .withRecordLabelFormat("%s - %s") - .withRecordLabelFields(List.of("runType", "startTime")) - .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) - .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER).withLabel("Index").withIsRequired(true).withPossibleValueSourceName(indexTableName)) - .withField(new QFieldMetaData("runType", QFieldType.STRING).withLabel("Run Type").withIsRequired(true).withPossibleValueSourceName(prefix + "quickSearchRunType")) - .withField(new QFieldMetaData("status", QFieldType.STRING).withLabel("Status").withIsRequired(true).withPossibleValueSourceName(prefix + "quickSearchRunStatus")) - .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME).withLabel("Start Time")) - .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME).withLabel("End Time")) - .withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER).withLabel("Records Processed")) - .withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER).withLabel("Records Indexed")) - .withField(new QFieldMetaData("errorCount", QFieldType.INTEGER).withLabel("Error Count")) - .withField(new QFieldMetaData("errorMessage", QFieldType.STRING).withLabel("Error Message")); - - qInstance.addTable(table); - } - - - - /*************************************************************************** - ** Produce the full reindex process. - ***************************************************************************/ - private void produceFullReindexProcess(QInstance qInstance) - { - String processName = config.applyPrefix(FULL_REINDEX_PROCESS_NAME); - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); - - QProcessMetaData process = new QProcessMetaData() - .withName(processName) - .withLabel("Full Reindex") - .withTableName(indexTableName) - .withStepList(List.of( - new QFrontendStepMetaData() - .withName("input") - .withFormFields(List.of( - new QFieldMetaData("tableName", QFieldType.STRING) - .withLabel("Table Name (empty for all)") - .withIsRequired(false) - )), - new QBackendStepMetaData() - .withName("execute") - .withCode(new QCodeReference(FullReindexStep.class)) - )); - - qInstance.addProcess(process); - } - - - - /*************************************************************************** - ** Produce the basepull index process. - ***************************************************************************/ - private void produceBasepullProcess(QInstance qInstance) - { - String processName = config.applyPrefix(BASEPULL_PROCESS_NAME); - - QProcessMetaData process = new QProcessMetaData() - .withName(processName) - .withLabel("Basepull Index Update") - .withStepList(List.of( - new QBackendStepMetaData() - .withName("execute") - .withCode(new QCodeReference(BasepullIndexStep.class)) - )); - - if(!Boolean.FALSE.equals(config.getEnableScheduledProcesses())) - { - process.withSchedule(new QScheduleMetaData() - .withRepeatSeconds(60) - .withDescription("Quick Search Basepull - runs every minute")); - } - - qInstance.addProcess(process); - } - - - - /*************************************************************************** - ** Produce the maintenance app. - ***************************************************************************/ - private void produceApp(QInstance qInstance) - { - String appName = config.applyPrefix(APP_NAME); - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); - String runTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); - String reindexProcessName = config.applyPrefix(FULL_REINDEX_PROCESS_NAME); - - QAppSection mainSection = new QAppSection() - .withName("main") - .withLabel("Search Indexes") - .withIcon(new QIcon().withName("search")) - .withTables(List.of(indexTableName, runTableName)) - .withProcesses(List.of(reindexProcessName)); - - QAppMetaData app = new QAppMetaData() - .withName(appName) - .withLabel("Quick Search") - .withIcon(new QIcon().withName("search")) - .withSections(List.of(mainSection)); - - qInstance.addApp(app); - } - - - - /*************************************************************************** - ** Initialize the OpenSearch index. - ***************************************************************************/ - private void initializeOpenSearch() throws QException - { - try - { - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - client.ensureIndexExists(); - client.close(); - LOG.info("OpenSearch index initialized", logPair("indexName", config.getOpensearchIndexName())); - } - catch(Exception e) - { - LOG.warn("Failed to initialize OpenSearch index - search may not work", logPair("indexName", config.getOpensearchIndexName()), e); - } - } - - - - /*************************************************************************** - ** Register delete customizers on discovered tables. - ***************************************************************************/ - private void registerDeleteCustomizers(QInstance qInstance, List discoveredTables) - { - for(QuickSearchableTableConfig tableConfig : discoveredTables) - { - QTableMetaData table = qInstance.getTable(tableConfig.getTableName()); - if(table != null) - { - String primaryKeyField = table.getPrimaryKeyField(); - QuickSearchDeleteCustomizer customizer = new QuickSearchDeleteCustomizer(config, tableConfig.getTableName(), primaryKeyField); - - table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(customizer.getClass())); - - LOG.debug("Registered delete customizer", logPair("tableName", tableConfig.getTableName())); - } - } - } - - - /*************************************************************************** ** Fluent setter for config. ***************************************************************************/ public QuickSearchQBitProducer withConfig(QuickSearchQBitConfig config) { this.config = config; - return this; + return (this); } @@ -450,7 +82,355 @@ public class QuickSearchQBitProducer implements QBitProducer ***************************************************************************/ public QuickSearchQBitConfig getConfig() { - return config; + return (this.config); + } + + + + /*************************************************************************** + ** Produce all QQQ metadata for the Quick Search QBit and register it + ** into the given QInstance. + ** + ** Steps: + ** 1. Validate configuration + ** 2. Discover @QuickSearchable annotated entity classes + ** 3. Create and initialize the OpenSearch client + ** 4. Create the index event publisher + ** 5. Store everything in QuickSearchQBitContext + ** 6. Produce operational tables + ** 7. Produce processes + ** 8. Register customizers on source tables (if real-time indexing enabled) + ** 9. Produce an admin app + ** + ** @param qInstance the QQQ instance to register metadata into + ** @throws QException if validation fails or an unrecoverable error occurs + ***************************************************************************/ + public void produce(QInstance qInstance) throws QException + { + //////////////////////////////////////////////////// + // 1. Validate config // + //////////////////////////////////////////////////// + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + if(!errors.isEmpty()) + { + throw new QException("QuickSearchQBit configuration validation failed: " + String.join("; ", errors)); + } + + //////////////////////////////////////////////////// + // 2. Discover searchable tables from annotations // + //////////////////////////////////////////////////// + List discoveredTables = discoverSearchableTables(qInstance); + + //////////////////////////////////////////////////// + // 3. Create OpenSearch client // + //////////////////////////////////////////////////// + QuickSearchOpenSearchClient client = null; + try + { + client = new QuickSearchOpenSearchClient(config); + } + catch(Exception e) + { + LOG.warn("Failed to create OpenSearch client during produce; continuing without client", e); + } + + //////////////////////////////////////////////////// + // 4. Initialize index (best-effort) // + //////////////////////////////////////////////////// + if(client != null) + { + try + { + client.ensureIndexExists(); + } + catch(Exception e) + { + LOG.warn("Failed to ensure OpenSearch index exists during produce; continuing", e); + } + } + + //////////////////////////////////////////////////// + // 5. Create publisher // + //////////////////////////////////////////////////// + IndexEventPublisher publisher = null; + if(config.getIndexEventPublisher() != null) + { + publisher = (IndexEventPublisher) config.getIndexEventPublisher(); + } + else if(client != null) + { + publisher = new SynchronousIndexEventPublisher(client, discoveredTables, config.getBulkBatchSize()); + } + + //////////////////////////////////////////////////// + // 6. Store in context // + //////////////////////////////////////////////////// + QuickSearchQBitContext.setConfig(config); + QuickSearchQBitContext.setClient(client); + QuickSearchQBitContext.setPublisher(publisher); + QuickSearchQBitContext.setDiscoveredTables(discoveredTables); + + //////////////////////////////////////////////////// + // 7. Produce operational tables // + //////////////////////////////////////////////////// + String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); + String indexRunTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); + + QTableMetaData indexTable = buildQuickSearchIndexTable(indexTableName, config.getBackendName()); + QTableMetaData indexRunTable = buildQuickSearchIndexRunTable(indexRunTableName, config.getBackendName()); + + qInstance.addTable(indexTable); + qInstance.addTable(indexRunTable); + + //////////////////////////////////////////////////// + // 8. Produce processes // + //////////////////////////////////////////////////// + String basepullProcessName = config.applyPrefix(BASEPULL_PROCESS_NAME); + String fullReindexProcessName = config.applyPrefix(FULL_REINDEX_PROCESS_NAME); + + QProcessMetaData basepullProcess = buildBasepullProcess(basepullProcessName); + QProcessMetaData fullReindexProcess = buildFullReindexProcess(fullReindexProcessName); + + qInstance.addProcess(basepullProcess); + qInstance.addProcess(fullReindexProcess); + + //////////////////////////////////////////////////// + // 9. Register customizers on source tables // + //////////////////////////////////////////////////// + if(Boolean.TRUE.equals(config.getEnableRealTimeIndexing())) + { + registerCustomizers(qInstance, discoveredTables); + } + + //////////////////////////////////////////////////// + // 10. Produce admin app // + //////////////////////////////////////////////////// + String appName = config.applyPrefix(APP_NAME); + QAppMetaData app = new QAppMetaData() + .withName(appName) + .withLabel("Quick Search Admin") + .withChild(indexTable) + .withChild(indexRunTable) + .withChild(basepullProcess) + .withChild(fullReindexProcess); + + qInstance.addApp(app); + } + + + + /*************************************************************************** + ** Discover searchable table configurations by scanning annotated entity + ** classes from the config. + ** + ** For each class, extracts @QuickSearchable annotation attributes and + ** scans fields for @QuickSearchField annotations. Falls back to the + ** annotation's fields() array if no @QuickSearchField-annotated fields + ** are found. + ** + ** @param qInstance the QInstance to look up table primary keys + ** @return list of discovered table configurations + ***************************************************************************/ + List discoverSearchableTables(QInstance qInstance) + { + List tables = new ArrayList<>(); + List> entityClasses = config.getSearchableEntityClasses(); + + if(entityClasses == null || entityClasses.isEmpty()) + { + return (tables); + } + + for(Class entityClass : entityClasses) + { + QuickSearchable annotation = entityClass.getAnnotation(QuickSearchable.class); + if(annotation == null) + { + LOG.warn("Entity class does not have @QuickSearchable annotation; skipping", + "className", entityClass.getName()); + continue; + } + + String tableName = annotation.tableName(); + Integer basepullIntervalMinutes = annotation.basepullIntervalMinutes(); + String basepullTimestampField = annotation.basepullTimestampField(); + Boolean enabledByDefault = annotation.enabledByDefault(); + + ////////////////////////////////////////////////////////// + // Scan fields for @QuickSearchField annotations // + ////////////////////////////////////////////////////////// + List searchableFields = new ArrayList<>(); + Map fieldWeights = new LinkedHashMap<>(); + Map fieldIncludeLabels = new LinkedHashMap<>(); + + for(Field field : entityClass.getDeclaredFields()) + { + QuickSearchField fieldAnnotation = field.getAnnotation(QuickSearchField.class); + if(fieldAnnotation != null) + { + String fieldName = field.getName(); + searchableFields.add(fieldName); + fieldWeights.put(fieldName, fieldAnnotation.weight()); + fieldIncludeLabels.put(fieldName, fieldAnnotation.includeLabel()); + } + } + + ////////////////////////////////////////////////////////// + // Fall back to @QuickSearchable.fields() if no // + // @QuickSearchField-annotated fields were found // + ////////////////////////////////////////////////////////// + if(searchableFields.isEmpty()) + { + String[] annotationFields = annotation.fields(); + if(annotationFields != null) + { + for(String fieldName : annotationFields) + { + searchableFields.add(fieldName); + fieldWeights.put(fieldName, 1); + fieldIncludeLabels.put(fieldName, false); + } + } + } + + ////////////////////////////////////////////////////////// + // Get primary key field from QInstance table metadata // + ////////////////////////////////////////////////////////// + String primaryKeyField = "id"; + QTableMetaData table = qInstance.getTable(tableName); + if(table != null) + { + primaryKeyField = table.getPrimaryKeyField(); + } + + QuickSearchableTableConfig tableConfig = new QuickSearchableTableConfig() + .withTableName(tableName) + .withPrimaryKeyField(primaryKeyField) + .withSearchableFields(searchableFields) + .withFieldWeights(fieldWeights) + .withFieldIncludeLabels(fieldIncludeLabels) + .withBasepullIntervalMinutes(basepullIntervalMinutes) + .withBasepullTimestampField(basepullTimestampField) + .withEnabledByDefault(enabledByDefault); + + tables.add(tableConfig); + } + + return (tables); + } + + + + /*************************************************************************** + ** Build QTableMetaData for the quickSearchIndex operational table. + ***************************************************************************/ + private QTableMetaData buildQuickSearchIndexTable(String tableName, String backendName) + { + return new QTableMetaData() + .withName(tableName) + .withLabel("Quick Search Index") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("tableName", QFieldType.STRING)) + .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN)) + .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER)) + .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING)) + .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.STRING)) + .withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("recordCount", QFieldType.INTEGER)) + .withField(new QFieldMetaData("status", QFieldType.STRING)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)); + } + + + + /*************************************************************************** + ** Build QTableMetaData for the quickSearchIndexRun operational table. + ***************************************************************************/ + private QTableMetaData buildQuickSearchIndexRunTable(String tableName, String backendName) + { + return new QTableMetaData() + .withName(tableName) + .withLabel("Quick Search Index Run") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER)) + .withField(new QFieldMetaData("runType", QFieldType.STRING)) + .withField(new QFieldMetaData("status", QFieldType.STRING)) + .withField(new QFieldMetaData("startTime", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("endTime", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER)) + .withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER)) + .withField(new QFieldMetaData("errorCount", QFieldType.INTEGER)) + .withField(new QFieldMetaData("errorMessage", QFieldType.STRING)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)); + } + + + + /*************************************************************************** + ** Build the basepull indexing process metadata. + ***************************************************************************/ + private QProcessMetaData buildBasepullProcess(String processName) + { + return new QProcessMetaData() + .withName(processName) + .withLabel("Quick Search Basepull Index") + .withStep(new QBackendStepMetaData() + .withName("basepull") + .withCode(new QCodeReference(BasepullIndexStep.class))); + } + + + + /*************************************************************************** + ** Build the full reindex process metadata with optional tableName input. + ***************************************************************************/ + private QProcessMetaData buildFullReindexProcess(String processName) + { + return new QProcessMetaData() + .withName(processName) + .withLabel("Quick Search Full Reindex") + .withStep(new QBackendStepMetaData() + .withName("fullReindex") + .withCode(new QCodeReference(FullReindexStep.class))); + } + + + + /*************************************************************************** + ** Register post-insert, post-update, and post-delete customizers on each + ** discovered source table. + ***************************************************************************/ + private void registerCustomizers(QInstance qInstance, List discoveredTables) + { + for(QuickSearchableTableConfig tableConfig : discoveredTables) + { + String tableName = tableConfig.getTableName(); + QTableMetaData table = qInstance.getTable(tableName); + + if(table == null) + { + LOG.warn("Source table not found in QInstance; skipping customizer registration", + "tableName", tableName); + continue; + } + + table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, + new QCodeReference(QuickSearchPostInsertCustomizer.class)); + + table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, + new QCodeReference(QuickSearchPostUpdateCustomizer.class)); + + table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, + new QCodeReference(QuickSearchPostDeleteCustomizer.class)); + } } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java index 4365514..653e754 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfig.java @@ -17,18 +17,26 @@ package com.kingsrook.qbits.quicksearch; import java.util.List; +import java.util.Map; /******************************************************************************* ** Configuration for a table that is searchable via Quick Search. + ** + ** Describes which fields to index, optional per-field boost weights, + ** optional per-field label-inclusion flags, basepull scheduling, and + ** whether the table is enabled by default. *******************************************************************************/ public class QuickSearchableTableConfig { - private String tableName; - private List searchableFields; - private Integer basepullIntervalMinutes; - private String basepullTimestampField; - private Boolean enabledByDefault; + private String tableName; + private String primaryKeyField; + private List searchableFields; + private Map fieldWeights; + private Map fieldIncludeLabels; + private Integer basepullIntervalMinutes; + private String basepullTimestampField; + private Boolean enabledByDefault; @@ -63,6 +71,37 @@ public class QuickSearchableTableConfig + /*************************************************************************** + ** Getter for primaryKeyField + ***************************************************************************/ + public String getPrimaryKeyField() + { + return (this.primaryKeyField); + } + + + + /*************************************************************************** + ** Setter for primaryKeyField + ***************************************************************************/ + public void setPrimaryKeyField(String primaryKeyField) + { + this.primaryKeyField = primaryKeyField; + } + + + + /*************************************************************************** + ** Fluent setter for primaryKeyField + ***************************************************************************/ + public QuickSearchableTableConfig withPrimaryKeyField(String primaryKeyField) + { + this.primaryKeyField = primaryKeyField; + return (this); + } + + + /*************************************************************************** ** Getter for searchableFields ***************************************************************************/ @@ -94,6 +133,68 @@ public class QuickSearchableTableConfig + /*************************************************************************** + ** Getter for fieldWeights + ***************************************************************************/ + public Map getFieldWeights() + { + return (this.fieldWeights); + } + + + + /*************************************************************************** + ** Setter for fieldWeights + ***************************************************************************/ + public void setFieldWeights(Map fieldWeights) + { + this.fieldWeights = fieldWeights; + } + + + + /*************************************************************************** + ** Fluent setter for fieldWeights + ***************************************************************************/ + public QuickSearchableTableConfig withFieldWeights(Map fieldWeights) + { + this.fieldWeights = fieldWeights; + return (this); + } + + + + /*************************************************************************** + ** Getter for fieldIncludeLabels + ***************************************************************************/ + public Map getFieldIncludeLabels() + { + return (this.fieldIncludeLabels); + } + + + + /*************************************************************************** + ** Setter for fieldIncludeLabels + ***************************************************************************/ + public void setFieldIncludeLabels(Map fieldIncludeLabels) + { + this.fieldIncludeLabels = fieldIncludeLabels; + } + + + + /*************************************************************************** + ** Fluent setter for fieldIncludeLabels + ***************************************************************************/ + public QuickSearchableTableConfig withFieldIncludeLabels(Map fieldIncludeLabels) + { + this.fieldIncludeLabels = fieldIncludeLabels; + return (this); + } + + + /*************************************************************************** ** Getter for basepullIntervalMinutes ***************************************************************************/ diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java index d55f34f..3b90aeb 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchAction.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,77 +13,115 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.actions; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; import com.kingsrook.qbits.quicksearch.processes.IndexingUtils; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.Hit; /******************************************************************************* - ** Executes quick search queries against OpenSearch. + ** Action that executes a quick-search query against OpenSearch and returns + ** normalized, paginated results. + ** + ** Validates and normalizes the input, delegates to the + ** QuickSearchOpenSearchClient, maps each hit to a QuickSearchResult, and + ** computes pagination metadata (totalHits, hasMore). *******************************************************************************/ public class QuickSearchAction { - private static final QLogger LOG = QLogger.getLogger(QuickSearchAction.class); + private static final QLogger LOG = QLogger.getLogger(QuickSearchAction.class); + private static final int DEFAULT_LIMIT = 25; + private static final int DEFAULT_OFFSET = 0; - /*************************************************************************** - ** Execute a quick search. - ***************************************************************************/ + /******************************************************************************* + ** Execute a quick-search for the given input. + ** + ** Returns an empty output (totalHits=0, hasMore=false, empty results list) + ** when the search term is null or blank. Otherwise normalizes the term, + ** applies default pagination values, calls the OpenSearch client, and maps + ** hits to QuickSearchResult objects. + ** + ** @param input the search parameters; must not be null + ** @return populated QuickSearchOutput + ** @throws QException if the OpenSearch call fails + *******************************************************************************/ public QuickSearchOutput execute(QuickSearchInput input) throws QException { - String searchTerm = IndexingUtils.normalizeSearchText(input.getSearchTerm()); + String searchTerm = input.getSearchTerm(); - if(!StringUtils.hasContent(searchTerm)) + if(searchTerm == null || searchTerm.isBlank()) { - return new QuickSearchOutput() + LOG.debug("QuickSearch called with blank/null search term - returning empty output"); + return (new QuickSearchOutput() .withResults(Collections.emptyList()) - .withTotalCount(0); + .withTotalHits(0L) + .withHasMore(false)); } - Integer limit = input.getLimit(); - if(limit == null || limit <= 0) + String normalizedTerm = IndexingUtils.normalizeSearchText(searchTerm); + + int limit = (input.getLimit() == null || input.getLimit() <= 0) ? DEFAULT_LIMIT : input.getLimit(); + int offset = (input.getOffset() == null || input.getOffset() < 0) ? DEFAULT_OFFSET : input.getOffset(); + + QuickSearchOpenSearchClient client = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + List tableConfigs = QuickSearchQBitContext.getDiscoveredTables(); + + LOG.debug("Executing quick search", "term", normalizedTerm, "tableName", input.getTableName(), "limit", limit, "offset", offset); + + SearchResponse response = client.search(normalizedTerm, input.getTableName(), limit, offset, tableConfigs); + + List results = new ArrayList<>(); + + for(Hit hit : response.hits().hits()) { - limit = 50; - } + OpenSearchDocument source = hit.source(); - LOG.debug("Executing quick search", logPair("term", searchTerm), logPair("limit", limit), logPair("table", input.getTableName())); + String highlightSnippet = null; + Map> highlight = hit.highlight(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(QuickSearchQBitContext.getConfig()); - - try - { - List documents = client.search(searchTerm, input.getTableName(), limit); - - List results = new ArrayList<>(); - for(OpenSearchDocument doc : documents) + if(highlight != null) { - results.add(new QuickSearchResult() - .withTableName(doc.getSourceTable()) - .withRecordId(doc.getRecordId()) - .withRecordLabel(doc.getRecordLabel()) - .withMatchedText(doc.getSearchableText())); + List fragments = highlight.get("searchableText"); + + if(fragments != null && !fragments.isEmpty()) + { + highlightSnippet = String.join("...", fragments); + } } - return new QuickSearchOutput() - .withResults(results) - .withTotalCount(results.size()); - } - finally - { - client.close(); + Double rawScore = hit.score(); + Float score = rawScore != null ? rawScore.floatValue() : null; + + results.add(new QuickSearchResult() + .withTableName(source.getSourceTable()) + .withRecordId(source.getRecordId()) + .withRecordLabel(source.getRecordLabel()) + .withScore(score) + .withHighlightSnippet(highlightSnippet)); } + + long totalHits = response.hits().total().value(); + boolean hasMore = (offset + results.size()) < totalHits; + + return (new QuickSearchOutput() + .withResults(results) + .withTotalHits(totalHits) + .withHasMore(hasMore)); } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchInput.java b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchInput.java index a3decc0..1f869ed 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchInput.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchInput.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,109 +13,133 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.actions; /******************************************************************************* - ** Input parameters for a quick search request. + ** Input DTO for the QuickSearch action. + ** + ** Carries the search term plus optional filtering and pagination parameters. *******************************************************************************/ public class QuickSearchInput { + private String searchTerm; private String tableName; private Integer limit; + private Integer offset; - - /*************************************************************************** - ** Getter for searchTerm - ***************************************************************************/ + /******************************************************************************* + ** Getter for searchTerm. + *******************************************************************************/ public String getSearchTerm() { - return (this.searchTerm); + return searchTerm; } - - /*************************************************************************** - ** Setter for searchTerm - ***************************************************************************/ + /******************************************************************************* + ** Setter for searchTerm. + *******************************************************************************/ public void setSearchTerm(String searchTerm) { this.searchTerm = searchTerm; } - - /*************************************************************************** - ** Fluent setter for searchTerm - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for searchTerm. + *******************************************************************************/ public QuickSearchInput withSearchTerm(String searchTerm) { this.searchTerm = searchTerm; - return (this); + return this; } - - /*************************************************************************** - ** Getter for tableName - ***************************************************************************/ + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ public String getTableName() { - return (this.tableName); + return tableName; } - - /*************************************************************************** - ** Setter for tableName - ***************************************************************************/ + /******************************************************************************* + ** Setter for tableName. + *******************************************************************************/ public void setTableName(String tableName) { this.tableName = tableName; } - - /*************************************************************************** - ** Fluent setter for tableName - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ public QuickSearchInput withTableName(String tableName) { this.tableName = tableName; - return (this); + return this; } - - /*************************************************************************** - ** Getter for limit - ***************************************************************************/ + /******************************************************************************* + ** Getter for limit. + *******************************************************************************/ public Integer getLimit() { - return (this.limit); + return limit; } - - /*************************************************************************** - ** Setter for limit - ***************************************************************************/ + /******************************************************************************* + ** Setter for limit. + *******************************************************************************/ public void setLimit(Integer limit) { this.limit = limit; } - - /*************************************************************************** - ** Fluent setter for limit - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for limit. + *******************************************************************************/ public QuickSearchInput withLimit(Integer limit) { this.limit = limit; - return (this); + return this; + } + + + /******************************************************************************* + ** Getter for offset. + *******************************************************************************/ + public Integer getOffset() + { + return offset; + } + + + /******************************************************************************* + ** Setter for offset. + *******************************************************************************/ + public void setOffset(Integer offset) + { + this.offset = offset; + } + + + /******************************************************************************* + ** Fluent setter for offset. + *******************************************************************************/ + public QuickSearchInput withOffset(Integer offset) + { + this.offset = offset; + return this; } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchOutput.java b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchOutput.java index 0e2c350..fdc2fb1 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchOutput.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchOutput.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.actions; @@ -20,73 +21,100 @@ import java.util.List; /******************************************************************************* - ** Output of a quick search request. + ** Output DTO for the QuickSearch action. + ** + ** Carries the list of matching results, the total hit count from OpenSearch, + ** and a convenience flag indicating whether additional pages are available. *******************************************************************************/ public class QuickSearchOutput { + private List results; - private Integer totalCount; + private Long totalHits; + private Boolean hasMore; - - /*************************************************************************** - ** Getter for results - ***************************************************************************/ + /******************************************************************************* + ** Getter for results. + *******************************************************************************/ public List getResults() { - return (this.results); + return results; } - - /*************************************************************************** - ** Setter for results - ***************************************************************************/ + /******************************************************************************* + ** Setter for results. + *******************************************************************************/ public void setResults(List results) { this.results = results; } - - /*************************************************************************** - ** Fluent setter for results - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for results. + *******************************************************************************/ public QuickSearchOutput withResults(List results) { this.results = results; - return (this); + return this; } - - /*************************************************************************** - ** Getter for totalCount - ***************************************************************************/ - public Integer getTotalCount() + /******************************************************************************* + ** Getter for totalHits. + *******************************************************************************/ + public Long getTotalHits() { - return (this.totalCount); + return totalHits; } - - /*************************************************************************** - ** Setter for totalCount - ***************************************************************************/ - public void setTotalCount(Integer totalCount) + /******************************************************************************* + ** Setter for totalHits. + *******************************************************************************/ + public void setTotalHits(Long totalHits) { - this.totalCount = totalCount; + this.totalHits = totalHits; } - - /*************************************************************************** - ** Fluent setter for totalCount - ***************************************************************************/ - public QuickSearchOutput withTotalCount(Integer totalCount) + /******************************************************************************* + ** Fluent setter for totalHits. + *******************************************************************************/ + public QuickSearchOutput withTotalHits(Long totalHits) { - this.totalCount = totalCount; - return (this); + this.totalHits = totalHits; + return this; + } + + + /******************************************************************************* + ** Getter for hasMore. + *******************************************************************************/ + public Boolean getHasMore() + { + return hasMore; + } + + + /******************************************************************************* + ** Setter for hasMore. + *******************************************************************************/ + public void setHasMore(Boolean hasMore) + { + this.hasMore = hasMore; + } + + + /******************************************************************************* + ** Fluent setter for hasMore. + *******************************************************************************/ + public QuickSearchOutput withHasMore(Boolean hasMore) + { + this.hasMore = hasMore; + return this; } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java index ac97f43..e4812ae 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchResult.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,141 +13,161 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.actions; /******************************************************************************* - ** A single result from a quick search query. + ** A single search hit returned by a QuickSearch query, representing one + ** matching record with its relevance score and optional highlight snippet. *******************************************************************************/ public class QuickSearchResult { + private String tableName; private String recordId; private String recordLabel; - private String matchedText; + private Float score; + private String highlightSnippet; - - /*************************************************************************** - ** Getter for tableName - ***************************************************************************/ + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ public String getTableName() { - return (this.tableName); + return tableName; } - - /*************************************************************************** - ** Setter for tableName - ***************************************************************************/ + /******************************************************************************* + ** Setter for tableName. + *******************************************************************************/ public void setTableName(String tableName) { this.tableName = tableName; } - - /*************************************************************************** - ** Fluent setter for tableName - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ public QuickSearchResult withTableName(String tableName) { this.tableName = tableName; - return (this); + return this; } - - /*************************************************************************** - ** Getter for recordId - ***************************************************************************/ + /******************************************************************************* + ** Getter for recordId. + *******************************************************************************/ public String getRecordId() { - return (this.recordId); + return recordId; } - - /*************************************************************************** - ** Setter for recordId - ***************************************************************************/ + /******************************************************************************* + ** Setter for recordId. + *******************************************************************************/ public void setRecordId(String recordId) { this.recordId = recordId; } - - /*************************************************************************** - ** Fluent setter for recordId - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for recordId. + *******************************************************************************/ public QuickSearchResult withRecordId(String recordId) { this.recordId = recordId; - return (this); + return this; } - - /*************************************************************************** - ** Getter for recordLabel - ***************************************************************************/ + /******************************************************************************* + ** Getter for recordLabel. + *******************************************************************************/ public String getRecordLabel() { - return (this.recordLabel); + return recordLabel; } - - /*************************************************************************** - ** Setter for recordLabel - ***************************************************************************/ + /******************************************************************************* + ** Setter for recordLabel. + *******************************************************************************/ public void setRecordLabel(String recordLabel) { this.recordLabel = recordLabel; } - - /*************************************************************************** - ** Fluent setter for recordLabel - ***************************************************************************/ + /******************************************************************************* + ** Fluent setter for recordLabel. + *******************************************************************************/ public QuickSearchResult withRecordLabel(String recordLabel) { this.recordLabel = recordLabel; - return (this); + return this; } - - /*************************************************************************** - ** Getter for matchedText - ***************************************************************************/ - public String getMatchedText() + /******************************************************************************* + ** Getter for score. + *******************************************************************************/ + public Float getScore() { - return (this.matchedText); + return score; } - - /*************************************************************************** - ** Setter for matchedText - ***************************************************************************/ - public void setMatchedText(String matchedText) + /******************************************************************************* + ** Setter for score. + *******************************************************************************/ + public void setScore(Float score) { - this.matchedText = matchedText; + this.score = score; } - - /*************************************************************************** - ** Fluent setter for matchedText - ***************************************************************************/ - public QuickSearchResult withMatchedText(String matchedText) + /******************************************************************************* + ** Fluent setter for score. + *******************************************************************************/ + public QuickSearchResult withScore(Float score) { - this.matchedText = matchedText; - return (this); + this.score = score; + return this; + } + + + /******************************************************************************* + ** Getter for highlightSnippet. + *******************************************************************************/ + public String getHighlightSnippet() + { + return highlightSnippet; + } + + + /******************************************************************************* + ** Setter for highlightSnippet. + *******************************************************************************/ + public void setHighlightSnippet(String highlightSnippet) + { + this.highlightSnippet = highlightSnippet; + } + + + /******************************************************************************* + ** Fluent setter for highlightSnippet. + *******************************************************************************/ + public QuickSearchResult withHighlightSnippet(String highlightSnippet) + { + this.highlightSnippet = highlightSnippet; + return this; } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchDeleteCustomizer.java b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchDeleteCustomizer.java deleted file mode 100644 index 0d50779..0000000 --- a/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchDeleteCustomizer.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2024 Kingsrook, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kingsrook.qbits.quicksearch.customizers; - - -import java.util.List; -import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; -import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; -import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; - - -/******************************************************************************* - ** Customizer that removes deleted records from the Quick Search index. - *******************************************************************************/ -public class QuickSearchDeleteCustomizer implements TableCustomizerInterface -{ - private static final QLogger LOG = QLogger.getLogger(QuickSearchDeleteCustomizer.class); - - private final QuickSearchQBitConfig config; - private final String tableName; - private final String primaryKeyField; - - - - /*************************************************************************** - ** Constructor. - ***************************************************************************/ - public QuickSearchDeleteCustomizer(QuickSearchQBitConfig config, String tableName, String primaryKeyField) - { - this.config = config; - this.tableName = tableName; - this.primaryKeyField = primaryKeyField; - } - - - - /*************************************************************************** - ** Post-delete hook - removes documents from OpenSearch. - ***************************************************************************/ - @Override - public List postDelete(DeleteInput deleteInput, List records) throws QException - { - if(records == null || records.isEmpty()) - { - return records; - } - - QuickSearchOpenSearchClient searchClient = null; - - try - { - searchClient = new QuickSearchOpenSearchClient(config); - - for(QRecord record : records) - { - Object primaryKeyValue = record.getValue(primaryKeyField); - if(primaryKeyValue != null) - { - String recordId = String.valueOf(primaryKeyValue); - try - { - searchClient.deleteDocument(tableName, recordId); - LOG.debug("Deleted document from search index", logPair("tableName", tableName), logPair("recordId", recordId)); - } - catch(Exception e) - { - LOG.warn("Failed to delete document from search index", logPair("tableName", tableName), logPair("recordId", recordId), e); - } - } - } - } - catch(Exception e) - { - LOG.warn("Failed to create OpenSearch client for delete", logPair("tableName", tableName), e); - } - finally - { - if(searchClient != null) - { - searchClient.close(); - } - } - - return records; - } - -} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizer.java b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizer.java new file mode 100644 index 0000000..cd202bf --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizer.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.customizers; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.publisher.IndexEvent; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Post-delete table customizer that removes deleted records from the Quick + ** Search OpenSearch index. + ** + ** QQQ instantiates this class via reflection using the no-arg constructor. + ** The publisher and table configuration are obtained from the static + ** QuickSearchQBitContext at call time. + *******************************************************************************/ +public class QuickSearchPostDeleteCustomizer implements TableCustomizerInterface +{ + private static final QLogger LOG = QLogger.getLogger(QuickSearchPostDeleteCustomizer.class); + + + + /******************************************************************************* + ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism. + *******************************************************************************/ + public QuickSearchPostDeleteCustomizer() + { + } + + + + /******************************************************************************* + ** Post-delete hook: builds DELETE events for each deleted record and publishes + ** them via the configured IndexEventPublisher. + ** + ** If the publisher is null (QBit not yet initialized) or if publishing fails, + ** a warning is logged and the original records are returned without throwing, + ** so as never to block the underlying QQQ delete operation. + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + if(records == null || records.isEmpty()) + { + return records; + } + + Object publisherObject = QuickSearchQBitContext.getPublisher(); + if(publisherObject == null) + { + LOG.warn("QuickSearch publisher is null; skipping post-delete indexing", logPair("tableName", deleteInput.getTableName())); + return records; + } + + try + { + IndexEventPublisher publisher = (IndexEventPublisher) publisherObject; + String tableName = deleteInput.getTableName(); + String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); + + List events = new ArrayList<>(); + for(QRecord record : records) + { + events.add(new IndexEvent() + .withAction(IndexEventAction.DELETE) + .withTableName(tableName) + .withRecordId(record.getValueString(primaryKeyField))); + } + + publisher.publishDeleteEvents(events); + } + catch(Exception e) + { + LOG.warn("Failed to publish post-delete index events", e, logPair("tableName", deleteInput.getTableName())); + } + + return records; + } + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizer.java b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizer.java new file mode 100644 index 0000000..d9de3ba --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizer.java @@ -0,0 +1,107 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.customizers; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.publisher.IndexEvent; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Post-insert table customizer that indexes newly inserted records in the + ** Quick Search OpenSearch index. + ** + ** QQQ instantiates this class via reflection using the no-arg constructor. + ** The publisher and table configuration are obtained from the static + ** QuickSearchQBitContext at call time. + *******************************************************************************/ +public class QuickSearchPostInsertCustomizer implements TableCustomizerInterface +{ + private static final QLogger LOG = QLogger.getLogger(QuickSearchPostInsertCustomizer.class); + + + + /******************************************************************************* + ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism. + *******************************************************************************/ + public QuickSearchPostInsertCustomizer() + { + } + + + + /******************************************************************************* + ** Post-insert hook: builds INDEX events for each inserted record and publishes + ** them via the configured IndexEventPublisher. + ** + ** If the publisher is null (QBit not yet initialized) or if publishing fails, + ** a warning is logged and the original records are returned without throwing, + ** so as never to block the underlying QQQ insert operation. + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + if(records == null || records.isEmpty()) + { + return records; + } + + Object publisherObject = QuickSearchQBitContext.getPublisher(); + if(publisherObject == null) + { + LOG.warn("QuickSearch publisher is null; skipping post-insert indexing", logPair("tableName", insertInput.getTableName())); + return records; + } + + try + { + IndexEventPublisher publisher = (IndexEventPublisher) publisherObject; + String tableName = insertInput.getTableName(); + String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); + + List events = new ArrayList<>(); + for(QRecord record : records) + { + events.add(new IndexEvent() + .withAction(IndexEventAction.INDEX) + .withTableName(tableName) + .withRecordId(record.getValueString(primaryKeyField)) + .withRecord(record)); + } + + publisher.publishIndexEvents(events); + } + catch(Exception e) + { + LOG.warn("Failed to publish post-insert index events", e, logPair("tableName", insertInput.getTableName())); + } + + return records; + } + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizer.java b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizer.java new file mode 100644 index 0000000..7ba1fc4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizer.java @@ -0,0 +1,108 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.customizers; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.publisher.IndexEvent; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Post-update table customizer that re-indexes updated records in the Quick + ** Search OpenSearch index. + ** + ** QQQ instantiates this class via reflection using the no-arg constructor. + ** The publisher and table configuration are obtained from the static + ** QuickSearchQBitContext at call time. + *******************************************************************************/ +public class QuickSearchPostUpdateCustomizer implements TableCustomizerInterface +{ + private static final QLogger LOG = QLogger.getLogger(QuickSearchPostUpdateCustomizer.class); + + + + /******************************************************************************* + ** No-arg constructor required by QQQ's QCodeReference instantiation mechanism. + *******************************************************************************/ + public QuickSearchPostUpdateCustomizer() + { + } + + + + /******************************************************************************* + ** Post-update hook: builds INDEX events for each updated record and publishes + ** them via the configured IndexEventPublisher. + ** + ** If the publisher is null (QBit not yet initialized) or if publishing fails, + ** a warning is logged and the original records are returned without throwing, + ** so as never to block the underlying QQQ update operation. + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + if(records == null || records.isEmpty()) + { + return records; + } + + Object publisherObject = QuickSearchQBitContext.getPublisher(); + if(publisherObject == null) + { + LOG.warn("QuickSearch publisher is null; skipping post-update indexing", logPair("tableName", updateInput.getTableName())); + return records; + } + + try + { + IndexEventPublisher publisher = (IndexEventPublisher) publisherObject; + String tableName = updateInput.getTableName(); + String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); + + List events = new ArrayList<>(); + for(QRecord record : records) + { + events.add(new IndexEvent() + .withAction(IndexEventAction.INDEX) + .withTableName(tableName) + .withRecordId(record.getValueString(primaryKeyField)) + .withRecord(record)); + } + + publisher.publishIndexEvents(events); + } + catch(Exception e) + { + LOG.warn("Failed to publish post-update index events", e, logPair("tableName", updateInput.getTableName())); + } + + return records; + } + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndex.java b/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndex.java index 63f0bb3..60ad378 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndex.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndex.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.model; @@ -22,50 +23,57 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; /******************************************************************************* - ** Entity for tracking Quick Search index configuration and status per table. + ** Entity bean for the quickSearchIndex table. + ** + ** Tracks configuration for each table that participates in the quick search + ** index, including basepull scheduling, searchable field definitions, and + ** current indexing status. *******************************************************************************/ public class QuickSearchIndex extends QRecordEntity { public static final String TABLE_NAME = "quickSearchIndex"; - @QField(isPrimaryKey = true) + @QField() private Integer id; - @QField(label = "Table Name", isRequired = true, isEditable = false) + @QField() private String tableName; - @QField(label = "Enabled") - private Boolean isEnabled; + @QField() + private Boolean enabled; - @QField(label = "Basepull Interval (min)") + @QField() private Integer basepullIntervalMinutes; - @QField(label = "Timestamp Field") + @QField() private String basepullTimestampField; - @QField(label = "Searchable Fields") + @QField() private String searchableFieldsJson; - @QField(label = "Last Full Index Time", isEditable = false) - private Instant lastFullIndexTime; - - @QField(label = "Last Basepull Time", isEditable = false) + @QField() private Instant lastBasepullTime; - @QField(label = "Indexed Record Count", isEditable = false) - private Integer indexedRecordCount; + @QField() + private Instant lastFullReindexTime; - @QField(label = "Create Date", isEditable = false) + @QField() + private Integer recordCount; + + @QField() + private String status; + + @QField() private Instant createDate; - @QField(label = "Modify Date", isEditable = false) + @QField() private Instant modifyDate; - /*************************************************************************** + /******************************************************************************* ** Getter for id - ***************************************************************************/ + *******************************************************************************/ public Integer getId() { return (this.id); @@ -73,19 +81,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for id - ***************************************************************************/ - public void setId(Integer id) - { - this.id = id; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for id - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withId(Integer id) { this.id = id; @@ -94,9 +92,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for tableName - ***************************************************************************/ + *******************************************************************************/ public String getTableName() { return (this.tableName); @@ -104,19 +102,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for tableName - ***************************************************************************/ - public void setTableName(String tableName) - { - this.tableName = tableName; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for tableName - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withTableName(String tableName) { this.tableName = tableName; @@ -125,40 +113,30 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Getter for isEnabled - ***************************************************************************/ - public Boolean getIsEnabled() + /******************************************************************************* + ** Getter for enabled + *******************************************************************************/ + public Boolean getEnabled() { - return (this.isEnabled); + return (this.enabled); } - /*************************************************************************** - ** Setter for isEnabled - ***************************************************************************/ - public void setIsEnabled(Boolean isEnabled) + /******************************************************************************* + ** Fluent setter for enabled + *******************************************************************************/ + public QuickSearchIndex withEnabled(Boolean enabled) { - this.isEnabled = isEnabled; - } - - - - /*************************************************************************** - ** Fluent setter for isEnabled - ***************************************************************************/ - public QuickSearchIndex withIsEnabled(Boolean isEnabled) - { - this.isEnabled = isEnabled; + this.enabled = enabled; return (this); } - /*************************************************************************** + /******************************************************************************* ** Getter for basepullIntervalMinutes - ***************************************************************************/ + *******************************************************************************/ public Integer getBasepullIntervalMinutes() { return (this.basepullIntervalMinutes); @@ -166,19 +144,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for basepullIntervalMinutes - ***************************************************************************/ - public void setBasepullIntervalMinutes(Integer basepullIntervalMinutes) - { - this.basepullIntervalMinutes = basepullIntervalMinutes; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for basepullIntervalMinutes - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withBasepullIntervalMinutes(Integer basepullIntervalMinutes) { this.basepullIntervalMinutes = basepullIntervalMinutes; @@ -187,9 +155,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for basepullTimestampField - ***************************************************************************/ + *******************************************************************************/ public String getBasepullTimestampField() { return (this.basepullTimestampField); @@ -197,19 +165,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for basepullTimestampField - ***************************************************************************/ - public void setBasepullTimestampField(String basepullTimestampField) - { - this.basepullTimestampField = basepullTimestampField; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for basepullTimestampField - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withBasepullTimestampField(String basepullTimestampField) { this.basepullTimestampField = basepullTimestampField; @@ -218,9 +176,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for searchableFieldsJson - ***************************************************************************/ + *******************************************************************************/ public String getSearchableFieldsJson() { return (this.searchableFieldsJson); @@ -228,19 +186,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for searchableFieldsJson - ***************************************************************************/ - public void setSearchableFieldsJson(String searchableFieldsJson) - { - this.searchableFieldsJson = searchableFieldsJson; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for searchableFieldsJson - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withSearchableFieldsJson(String searchableFieldsJson) { this.searchableFieldsJson = searchableFieldsJson; @@ -249,40 +197,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Getter for lastFullIndexTime - ***************************************************************************/ - public Instant getLastFullIndexTime() - { - return (this.lastFullIndexTime); - } - - - - /*************************************************************************** - ** Setter for lastFullIndexTime - ***************************************************************************/ - public void setLastFullIndexTime(Instant lastFullIndexTime) - { - this.lastFullIndexTime = lastFullIndexTime; - } - - - - /*************************************************************************** - ** Fluent setter for lastFullIndexTime - ***************************************************************************/ - public QuickSearchIndex withLastFullIndexTime(Instant lastFullIndexTime) - { - this.lastFullIndexTime = lastFullIndexTime; - return (this); - } - - - - /*************************************************************************** + /******************************************************************************* ** Getter for lastBasepullTime - ***************************************************************************/ + *******************************************************************************/ public Instant getLastBasepullTime() { return (this.lastBasepullTime); @@ -290,19 +207,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for lastBasepullTime - ***************************************************************************/ - public void setLastBasepullTime(Instant lastBasepullTime) - { - this.lastBasepullTime = lastBasepullTime; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for lastBasepullTime - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withLastBasepullTime(Instant lastBasepullTime) { this.lastBasepullTime = lastBasepullTime; @@ -311,40 +218,72 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Getter for indexedRecordCount - ***************************************************************************/ - public Integer getIndexedRecordCount() + /******************************************************************************* + ** Getter for lastFullReindexTime + *******************************************************************************/ + public Instant getLastFullReindexTime() { - return (this.indexedRecordCount); + return (this.lastFullReindexTime); } - /*************************************************************************** - ** Setter for indexedRecordCount - ***************************************************************************/ - public void setIndexedRecordCount(Integer indexedRecordCount) + /******************************************************************************* + ** Fluent setter for lastFullReindexTime + *******************************************************************************/ + public QuickSearchIndex withLastFullReindexTime(Instant lastFullReindexTime) { - this.indexedRecordCount = indexedRecordCount; - } - - - - /*************************************************************************** - ** Fluent setter for indexedRecordCount - ***************************************************************************/ - public QuickSearchIndex withIndexedRecordCount(Integer indexedRecordCount) - { - this.indexedRecordCount = indexedRecordCount; + this.lastFullReindexTime = lastFullReindexTime; return (this); } - /*************************************************************************** + /******************************************************************************* + ** Getter for recordCount + *******************************************************************************/ + public Integer getRecordCount() + { + return (this.recordCount); + } + + + + /******************************************************************************* + ** Fluent setter for recordCount + *******************************************************************************/ + public QuickSearchIndex withRecordCount(Integer recordCount) + { + this.recordCount = recordCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for status + *******************************************************************************/ + public String getStatus() + { + return (this.status); + } + + + + /******************************************************************************* + ** Fluent setter for status + *******************************************************************************/ + public QuickSearchIndex withStatus(String status) + { + this.status = status; + return (this); + } + + + + /******************************************************************************* ** Getter for createDate - ***************************************************************************/ + *******************************************************************************/ public Instant getCreateDate() { return (this.createDate); @@ -352,19 +291,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for createDate - ***************************************************************************/ - public void setCreateDate(Instant createDate) - { - this.createDate = createDate; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for createDate - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withCreateDate(Instant createDate) { this.createDate = createDate; @@ -373,9 +302,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for modifyDate - ***************************************************************************/ + *******************************************************************************/ public Instant getModifyDate() { return (this.modifyDate); @@ -383,19 +312,9 @@ public class QuickSearchIndex extends QRecordEntity - /*************************************************************************** - ** Setter for modifyDate - ***************************************************************************/ - public void setModifyDate(Instant modifyDate) - { - this.modifyDate = modifyDate; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for modifyDate - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndex withModifyDate(Instant modifyDate) { this.modifyDate = modifyDate; diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRun.java b/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRun.java index cba106d..95dfb4c 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRun.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRun.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.model; @@ -22,47 +23,57 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; /******************************************************************************* - ** Entity for tracking Quick Search indexing job history. + ** Entity bean for the quickSearchIndexRun table. + ** + ** Records a single execution of the indexing process for a given + ** QuickSearchIndex, capturing run type, timing, record counts, and any errors + ** encountered during the run. *******************************************************************************/ public class QuickSearchIndexRun extends QRecordEntity { public static final String TABLE_NAME = "quickSearchIndexRun"; - @QField(isPrimaryKey = true) + @QField() private Integer id; - @QField(label = "Index", isRequired = true) + @QField() private Integer quickSearchIndexId; - @QField(label = "Run Type", isRequired = true) + @QField() private String runType; - @QField(label = "Status", isRequired = true) + @QField() private String status; - @QField(label = "Start Time") + @QField() private Instant startTime; - @QField(label = "End Time") + @QField() private Instant endTime; - @QField(label = "Records Processed") + @QField() private Integer recordsProcessed; - @QField(label = "Records Indexed") + @QField() private Integer recordsIndexed; - @QField(label = "Error Count") + @QField() private Integer errorCount; - @QField(label = "Error Message") + @QField() private String errorMessage; + @QField() + private Instant createDate; + + @QField() + private Instant modifyDate; - /*************************************************************************** + + /******************************************************************************* ** Getter for id - ***************************************************************************/ + *******************************************************************************/ public Integer getId() { return (this.id); @@ -70,19 +81,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for id - ***************************************************************************/ - public void setId(Integer id) - { - this.id = id; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for id - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withId(Integer id) { this.id = id; @@ -91,9 +92,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for quickSearchIndexId - ***************************************************************************/ + *******************************************************************************/ public Integer getQuickSearchIndexId() { return (this.quickSearchIndexId); @@ -101,19 +102,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for quickSearchIndexId - ***************************************************************************/ - public void setQuickSearchIndexId(Integer quickSearchIndexId) - { - this.quickSearchIndexId = quickSearchIndexId; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for quickSearchIndexId - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withQuickSearchIndexId(Integer quickSearchIndexId) { this.quickSearchIndexId = quickSearchIndexId; @@ -122,9 +113,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for runType - ***************************************************************************/ + *******************************************************************************/ public String getRunType() { return (this.runType); @@ -132,19 +123,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for runType - ***************************************************************************/ - public void setRunType(String runType) - { - this.runType = runType; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for runType - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withRunType(String runType) { this.runType = runType; @@ -153,9 +134,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for status - ***************************************************************************/ + *******************************************************************************/ public String getStatus() { return (this.status); @@ -163,19 +144,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for status - ***************************************************************************/ - public void setStatus(String status) - { - this.status = status; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for status - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withStatus(String status) { this.status = status; @@ -184,9 +155,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for startTime - ***************************************************************************/ + *******************************************************************************/ public Instant getStartTime() { return (this.startTime); @@ -194,19 +165,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for startTime - ***************************************************************************/ - public void setStartTime(Instant startTime) - { - this.startTime = startTime; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for startTime - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withStartTime(Instant startTime) { this.startTime = startTime; @@ -215,9 +176,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for endTime - ***************************************************************************/ + *******************************************************************************/ public Instant getEndTime() { return (this.endTime); @@ -225,19 +186,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for endTime - ***************************************************************************/ - public void setEndTime(Instant endTime) - { - this.endTime = endTime; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for endTime - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withEndTime(Instant endTime) { this.endTime = endTime; @@ -246,9 +197,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for recordsProcessed - ***************************************************************************/ + *******************************************************************************/ public Integer getRecordsProcessed() { return (this.recordsProcessed); @@ -256,19 +207,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for recordsProcessed - ***************************************************************************/ - public void setRecordsProcessed(Integer recordsProcessed) - { - this.recordsProcessed = recordsProcessed; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for recordsProcessed - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withRecordsProcessed(Integer recordsProcessed) { this.recordsProcessed = recordsProcessed; @@ -277,9 +218,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for recordsIndexed - ***************************************************************************/ + *******************************************************************************/ public Integer getRecordsIndexed() { return (this.recordsIndexed); @@ -287,19 +228,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for recordsIndexed - ***************************************************************************/ - public void setRecordsIndexed(Integer recordsIndexed) - { - this.recordsIndexed = recordsIndexed; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for recordsIndexed - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withRecordsIndexed(Integer recordsIndexed) { this.recordsIndexed = recordsIndexed; @@ -308,9 +239,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for errorCount - ***************************************************************************/ + *******************************************************************************/ public Integer getErrorCount() { return (this.errorCount); @@ -318,19 +249,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for errorCount - ***************************************************************************/ - public void setErrorCount(Integer errorCount) - { - this.errorCount = errorCount; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for errorCount - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withErrorCount(Integer errorCount) { this.errorCount = errorCount; @@ -339,9 +260,9 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** + /******************************************************************************* ** Getter for errorMessage - ***************************************************************************/ + *******************************************************************************/ public String getErrorMessage() { return (this.errorMessage); @@ -349,23 +270,55 @@ public class QuickSearchIndexRun extends QRecordEntity - /*************************************************************************** - ** Setter for errorMessage - ***************************************************************************/ - public void setErrorMessage(String errorMessage) - { - this.errorMessage = errorMessage; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for errorMessage - ***************************************************************************/ + *******************************************************************************/ public QuickSearchIndexRun withErrorMessage(String errorMessage) { this.errorMessage = errorMessage; return (this); } + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public QuickSearchIndexRun withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public QuickSearchIndexRun withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java b/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java new file mode 100644 index 0000000..662e057 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResult.java @@ -0,0 +1,128 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.opensearch; + + +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** Accumulates success and failure counts from a bulk OpenSearch index operation. + *******************************************************************************/ +public class BulkIndexResult +{ + + private Integer successCount = 0; + private Integer failureCount = 0; + private List errors = new ArrayList<>(); + + + + /******************************************************************************* + ** Increment the success counter by one. + *******************************************************************************/ + public void addSuccess() + { + successCount = successCount + 1; + } + + + + /******************************************************************************* + ** Increment the failure counter by one and record the error message. + *******************************************************************************/ + public void addFailure(String errorMessage) + { + failureCount = failureCount + 1; + errors.add(errorMessage); + } + + + + /******************************************************************************* + ** Returns true when no failures have been recorded. + *******************************************************************************/ + public Boolean isFullySuccessful() + { + return (failureCount == 0); + } + + + + /******************************************************************************* + ** Getter for successCount + *******************************************************************************/ + public Integer getSuccessCount() + { + return (successCount); + } + + + + /******************************************************************************* + ** Fluent setter for successCount + *******************************************************************************/ + public BulkIndexResult withSuccessCount(Integer successCount) + { + this.successCount = successCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for failureCount + *******************************************************************************/ + public Integer getFailureCount() + { + return (failureCount); + } + + + + /******************************************************************************* + ** Fluent setter for failureCount + *******************************************************************************/ + public BulkIndexResult withFailureCount(Integer failureCount) + { + this.failureCount = failureCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for errors + *******************************************************************************/ + public List getErrors() + { + return (errors); + } + + + + /******************************************************************************* + ** Fluent setter for errors + *******************************************************************************/ + public BulkIndexResult withErrors(List errors) + { + this.errors = errors; + return (this); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocument.java b/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocument.java index 852afe8..ab06255 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocument.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocument.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,61 +13,46 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.opensearch; import java.time.Instant; +import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; /******************************************************************************* - ** Document structure for Quick Search entries in OpenSearch. + ** Represents a document stored in OpenSearch for quick-search indexing. + ** + ** Each document maps to a single QQQ record and contains the denormalized + ** searchable text used for full-text queries. *******************************************************************************/ public class OpenSearchDocument { - private String sourceTable; - private String recordId; - private String recordLabel; - private String searchableText; + + private String sourceTable; + private String recordId; + private String recordLabel; + private String searchableText; private Instant indexedAt; + private Map fieldValues; - /*************************************************************************** - ** Generate a unique document ID for OpenSearch. - ** This is not stored in the document body - it's used as the document _id. - ***************************************************************************/ - @JsonIgnore - public String getDocumentId() - { - return sourceTable + ":" + recordId; - } - - - - /*************************************************************************** + /******************************************************************************* ** Getter for sourceTable - ***************************************************************************/ + *******************************************************************************/ public String getSourceTable() { - return (this.sourceTable); + return (sourceTable); } - /*************************************************************************** - ** Setter for sourceTable - ***************************************************************************/ - public void setSourceTable(String sourceTable) - { - this.sourceTable = sourceTable; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for sourceTable - ***************************************************************************/ + *******************************************************************************/ public OpenSearchDocument withSourceTable(String sourceTable) { this.sourceTable = sourceTable; @@ -76,29 +61,19 @@ public class OpenSearchDocument - /*************************************************************************** + /******************************************************************************* ** Getter for recordId - ***************************************************************************/ + *******************************************************************************/ public String getRecordId() { - return (this.recordId); + return (recordId); } - /*************************************************************************** - ** Setter for recordId - ***************************************************************************/ - public void setRecordId(String recordId) - { - this.recordId = recordId; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for recordId - ***************************************************************************/ + *******************************************************************************/ public OpenSearchDocument withRecordId(String recordId) { this.recordId = recordId; @@ -107,29 +82,19 @@ public class OpenSearchDocument - /*************************************************************************** + /******************************************************************************* ** Getter for recordLabel - ***************************************************************************/ + *******************************************************************************/ public String getRecordLabel() { - return (this.recordLabel); + return (recordLabel); } - /*************************************************************************** - ** Setter for recordLabel - ***************************************************************************/ - public void setRecordLabel(String recordLabel) - { - this.recordLabel = recordLabel; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for recordLabel - ***************************************************************************/ + *******************************************************************************/ public OpenSearchDocument withRecordLabel(String recordLabel) { this.recordLabel = recordLabel; @@ -138,29 +103,19 @@ public class OpenSearchDocument - /*************************************************************************** + /******************************************************************************* ** Getter for searchableText - ***************************************************************************/ + *******************************************************************************/ public String getSearchableText() { - return (this.searchableText); + return (searchableText); } - /*************************************************************************** - ** Setter for searchableText - ***************************************************************************/ - public void setSearchableText(String searchableText) - { - this.searchableText = searchableText; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for searchableText - ***************************************************************************/ + *******************************************************************************/ public OpenSearchDocument withSearchableText(String searchableText) { this.searchableText = searchableText; @@ -169,33 +124,56 @@ public class OpenSearchDocument - /*************************************************************************** + /******************************************************************************* ** Getter for indexedAt - ***************************************************************************/ + *******************************************************************************/ public Instant getIndexedAt() { - return (this.indexedAt); + return (indexedAt); } - /*************************************************************************** - ** Setter for indexedAt - ***************************************************************************/ - public void setIndexedAt(Instant indexedAt) - { - this.indexedAt = indexedAt; - } - - - - /*************************************************************************** + /******************************************************************************* ** Fluent setter for indexedAt - ***************************************************************************/ + *******************************************************************************/ public OpenSearchDocument withIndexedAt(Instant indexedAt) { this.indexedAt = indexedAt; return (this); } + + + /******************************************************************************* + ** Getter for fieldValues + *******************************************************************************/ + public Map getFieldValues() + { + return (fieldValues); + } + + + + /******************************************************************************* + ** Fluent setter for fieldValues + *******************************************************************************/ + public OpenSearchDocument withFieldValues(Map fieldValues) + { + this.fieldValues = fieldValues; + return (this); + } + + + + /******************************************************************************* + ** Returns the composite document ID used as the OpenSearch _id field. + ** Excluded from JSON serialization to avoid including it in the document body. + *******************************************************************************/ + @JsonIgnore + public String getDocumentId() + { + return (sourceTable + ":" + recordId); + } + } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java b/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java index 14da27a..abae50b 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClient.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,17 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.opensearch; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; @@ -31,8 +34,10 @@ import org.apache.hc.core5.http.HttpHost; import org.opensearch.client.json.jackson.JacksonJsonpMapper; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.opensearch._types.FieldValue; -import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.MultiMatchQuery; import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; import org.opensearch.client.opensearch.core.BulkRequest; import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.DeleteByQueryRequest; @@ -40,285 +45,433 @@ import org.opensearch.client.opensearch.core.DeleteRequest; import org.opensearch.client.opensearch.core.IndexRequest; import org.opensearch.client.opensearch.core.SearchRequest; import org.opensearch.client.opensearch.core.SearchResponse; -import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.search.Highlight; +import org.opensearch.client.opensearch.core.search.HighlightField; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.ExistsRequest; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5Transport; import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* - ** OpenSearch client wrapper for Quick Search operations. + ** OpenSearch client wrapper for the Quick Search QBit. + ** + ** Manages the lifecycle of an OpenSearchClient with optional SSL and basic-auth + ** support. Provides helpers for index creation with edge-ngram mappings, single + ** and bulk document indexing, deletion, and boosted multi-match search. *******************************************************************************/ public class QuickSearchOpenSearchClient { private static final QLogger LOG = QLogger.getLogger(QuickSearchOpenSearchClient.class); - private final OpenSearchClient client; - private final QuickSearchQBitConfig config; + private final OpenSearchClient client; + private final ApacheHttpClient5Transport transport; + private final String indexName; + private Boolean closed = false; - /*************************************************************************** - ** Constructor - creates OpenSearch client from QBit configuration. - ***************************************************************************/ + /******************************************************************************* + ** Construct a new client from the provided config. + ** + ** Creates an ApacheHttpClient5Transport pointed at the configured host/port. + ** When both username and password are non-null, a BasicCredentialsProvider + ** is configured for HTTP basic authentication. A JavaTimeModule-aware + ** ObjectMapper is used for JSON serialization. + ** + ** @param config the QBit configuration supplying connection details + ** @throws QException if transport construction fails + *******************************************************************************/ public QuickSearchOpenSearchClient(QuickSearchQBitConfig config) throws QException { - this.config = config; - try { - String scheme = Boolean.TRUE.equals(config.getUseSsl()) ? "https" : "http"; - HttpHost host = new HttpHost(scheme, config.getOpensearchHost(), config.getOpensearchPort()); - - ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder.builder(host); - - if(config.getOpensearchUsername() != null && config.getOpensearchPassword() != null) - { - BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials( - new AuthScope(host), - new UsernamePasswordCredentials( - config.getOpensearchUsername(), - config.getOpensearchPassword().toCharArray() - ) - ); - builder.setHttpClientConfigCallback(httpClientBuilder -> - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - ); - } + String scheme = Boolean.TRUE.equals(config.getUseSsl()) ? "https" : "http"; + HttpHost httpHost = new HttpHost(scheme, config.getOpensearchHost(), config.getOpensearchPort()); ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); - builder.setMapper(new JacksonJsonpMapper(objectMapper)); - this.client = new OpenSearchClient(builder.build()); + ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder + .builder(httpHost) + .setMapper(new JacksonJsonpMapper(objectMapper)); + + String username = config.getOpensearchUsername(); + String password = config.getOpensearchPassword(); + + if(username != null && password != null) + { + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + new AuthScope(httpHost), + new UsernamePasswordCredentials(username, password.toCharArray())); + + builder.setHttpClientConfigCallback(httpClientBuilder -> + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + transport = builder.build(); + client = new OpenSearchClient(transport); + indexName = config.getOpensearchIndexName(); } catch(Exception e) { - throw new QException("Failed to create OpenSearch client", e); + throw new QException("Failed to create OpenSearch transport: " + e.getMessage(), e); } } - /*************************************************************************** - ** Ensure the index exists, creating it if necessary. - ***************************************************************************/ + /******************************************************************************* + ** Ensure the index exists, creating it with explicit mappings if needed. + ** + ** Uses a custom edge-ngram analyzer (quick_search_analyzer) with min=2 max=20. + ** Field mappings: sourceTable and recordId as keyword; recordLabel as text; + ** searchableText as text with the custom analyzer; indexedAt as date; + ** fieldValues as a dynamic object. + ** + ** @throws QException if the OpenSearch call fails + *******************************************************************************/ public void ensureIndexExists() throws QException { try { - String indexName = config.getOpensearchIndexName(); - ExistsRequest existsRequest = new ExistsRequest.Builder().index(indexName).build(); + boolean exists = client.indices().exists(ExistsRequest.of(r -> r.index(indexName))).value(); - if(!client.indices().exists(existsRequest).value()) + if(exists) { - LOG.info("Creating OpenSearch index", logPair("indexName", indexName)); - - CreateIndexRequest createRequest = new CreateIndexRequest.Builder() - .index(indexName) - .build(); - - client.indices().create(createRequest); + LOG.info("OpenSearch index already exists", "indexName", indexName); + return; } + + client.indices().create(CreateIndexRequest.of(r -> r + .index(indexName) + .settings(s -> s + .analysis(a -> a + .tokenizer("edge_ngram_tokenizer", t -> t + .definition(d -> d + .edgeNgram(en -> en + .minGram(2) + .maxGram(20) + .tokenChars(List.of())))) + .analyzer("quick_search_analyzer", an -> an + .custom(c -> c + .tokenizer("standard") + .filter("lowercase", "edge_ngram_filter"))) + .filter("edge_ngram_filter", f -> f + .definition(d -> d + .edgeNgram(en -> en + .minGram(2) + .maxGram(20)))))) + .mappings(m -> m + .properties("sourceTable", p -> p.keyword(k -> k)) + .properties("recordId", p -> p.keyword(k -> k)) + .properties("recordLabel", p -> p.text(t -> t)) + .properties("searchableText", p -> p.text(t -> t.analyzer("quick_search_analyzer"))) + .properties("indexedAt", p -> p.date(d -> d)) + .properties("fieldValues", p -> p.object(o -> o))))); + + + LOG.info("Created OpenSearch index", "indexName", indexName); } catch(IOException e) { - throw new QException("Failed to ensure index exists", e); + throw new QException("Failed to ensure index exists [" + indexName + "]: " + e.getMessage(), e); } } - /*************************************************************************** - ** Index a single document. - ***************************************************************************/ - public void indexDocument(OpenSearchDocument document) throws QException + /******************************************************************************* + ** Index a single document using its composite document ID as the _id. + ** + ** @param doc the document to index + ** @throws QException if the OpenSearch call fails + *******************************************************************************/ + public void indexDocument(OpenSearchDocument doc) throws QException { try { - IndexRequest request = new IndexRequest.Builder() - .index(config.getOpensearchIndexName()) - .id(document.getDocumentId()) - .document(document) - .build(); - - client.index(request); + client.index(IndexRequest.of(r -> r + .index(indexName) + .id(doc.getDocumentId()) + .document(doc))); } catch(IOException e) { - throw new QException("Failed to index document: " + document.getDocumentId(), e); + throw new QException("Failed to index document [" + doc.getDocumentId() + "]: " + e.getMessage(), e); } } - /*************************************************************************** - ** Index multiple documents in bulk. - ***************************************************************************/ - public void indexDocuments(List documents) throws QException + /******************************************************************************* + ** Bulk-index a list of documents in batches of at most batchSize. + ** + ** An empty or null list is a no-op that returns an empty BulkIndexResult. + ** Documents within each batch are collected into a BulkRequest. Per-item + ** errors are recorded via BulkIndexResult.addFailure(); successes via + ** addSuccess(). + ** + ** @param docs the documents to index; null is treated as empty + ** @param batchSize maximum number of documents per bulk request + ** @return accumulated result across all batches + ** @throws QException if any batch request fails at the transport level + *******************************************************************************/ + public BulkIndexResult indexDocuments(List docs, int batchSize) throws QException { - if(documents.isEmpty()) + BulkIndexResult result = new BulkIndexResult(); + + if(docs == null || docs.isEmpty()) { - return; + return (result); } - try - { - BulkRequest.Builder bulkBuilder = new BulkRequest.Builder(); + int total = docs.size(); - for(OpenSearchDocument document : documents) + for(int start = 0; start < total; start += batchSize) + { + int end = Math.min(start + batchSize, total); + List batch = docs.subList(start, end); + + List operations = new ArrayList<>(); + + for(OpenSearchDocument doc : batch) { - bulkBuilder.operations(op -> op - .index(idx -> idx - .index(config.getOpensearchIndexName()) - .id(document.getDocumentId()) - .document(document) - ) - ); + String docId = doc.getDocumentId(); + + operations.add(BulkOperation.of(o -> o + .index(IndexOperation.of(i -> i + .index(indexName) + .id(docId) + .document(doc))))); } - BulkResponse response = client.bulk(bulkBuilder.build()); + BulkRequest bulkRequest = BulkRequest.of(r -> r.operations(operations)); - if(response.errors()) + try { - LOG.warn("Bulk indexing completed with errors", logPair("errorCount", response.items().stream().filter(i -> i.error() != null).count())); + BulkResponse response = client.bulk(bulkRequest); + + for(BulkResponseItem item : response.items()) + { + if(item.error() != null) + { + result.addFailure(item.id() + ": " + item.error().reason()); + } + else + { + result.addSuccess(); + } + } + } + catch(IOException e) + { + throw new QException("Bulk index request failed: " + e.getMessage(), e); } } - catch(IOException e) - { - throw new QException("Failed to bulk index documents", e); - } + + return (result); } - /*************************************************************************** - ** Delete a single document by its ID. - ***************************************************************************/ + /******************************************************************************* + ** Delete a single document identified by sourceTable and recordId. + ** + ** @param sourceTable the source table component of the document ID + ** @param recordId the record ID component of the document ID + ** @throws QException if the OpenSearch call fails + *******************************************************************************/ public void deleteDocument(String sourceTable, String recordId) throws QException { + String documentId = sourceTable + ":" + recordId; + try { - String documentId = sourceTable + ":" + recordId; - - DeleteRequest request = new DeleteRequest.Builder() - .index(config.getOpensearchIndexName()) - .id(documentId) - .build(); - - client.delete(request); + client.delete(DeleteRequest.of(r -> r + .index(indexName) + .id(documentId))); } catch(IOException e) { - throw new QException("Failed to delete document: " + sourceTable + ":" + recordId, e); + throw new QException("Failed to delete document [" + documentId + "]: " + e.getMessage(), e); } } - /*************************************************************************** - ** Delete all documents for a specific source table. - ***************************************************************************/ + /******************************************************************************* + ** Delete all documents whose sourceTable field matches the given table name. + ** + ** @param sourceTable the table name to delete all documents for + ** @throws QException if the OpenSearch call fails + *******************************************************************************/ public void deleteDocumentsForTable(String sourceTable) throws QException { try { - Query query = new Query.Builder() - .term(t -> t.field("sourceTable").value(FieldValue.of(sourceTable))) - .build(); - - DeleteByQueryRequest request = new DeleteByQueryRequest.Builder() - .index(config.getOpensearchIndexName()) - .query(query) - .build(); - - client.deleteByQuery(request); + client.deleteByQuery(DeleteByQueryRequest.of(r -> r + .index(indexName) + .query(Query.of(q -> q + .term(TermQuery.of(t -> t + .field("sourceTable") + .value(FieldValue.of(sourceTable)))))))); } catch(IOException e) { - throw new QException("Failed to delete documents for table: " + sourceTable, e); + throw new QException("Failed to delete documents for table [" + sourceTable + "]: " + e.getMessage(), e); } } - /*************************************************************************** - ** Search for documents matching the given query text. - ***************************************************************************/ - public List search(String queryText, Integer limit) throws QException - { - return search(queryText, null, limit); - } - - - - /*************************************************************************** - ** Search for documents matching the given query text, optionally filtered by table. - ***************************************************************************/ - public List search(String queryText, String sourceTable, Integer limit) throws QException + /******************************************************************************* + ** Execute a boosted multi-match search across the index. + ** + ** Builds a bool query with: + ** - a must multi_match on searchableText + ** - per-table should clauses on fieldValues.{fieldName} using configured weights + ** An optional term filter on sourceTable is applied when tableName is non-null. + ** Pagination via from/size. Highlights on searchableText. + ** + ** @param searchTerm the user-supplied search string + ** @param tableName when non-null, restricts results to this source table + ** @param limit maximum number of hits to return + ** @param offset number of hits to skip (for pagination) + ** @param tableConfigs per-table config providing searchable fields and weights + ** @return raw SearchResponse from OpenSearch + ** @throws QException if the OpenSearch call fails + *******************************************************************************/ + public SearchResponse search(String searchTerm, String tableName, int limit, int offset, List tableConfigs) throws QException { try { - Query matchQuery = new Query.Builder() - .match(new MatchQuery.Builder() - .field("searchableText") - .query(FieldValue.of(queryText)) - .build()) - .build(); + List shouldClauses = buildFieldBoostQueries(searchTerm, tableConfigs); - Query finalQuery; - if(sourceTable != null) - { - Query tableFilter = new Query.Builder() - .term(t -> t.field("sourceTable").value(FieldValue.of(sourceTable))) - .build(); + Query mustQuery = Query.of(q -> q + .multiMatch(MultiMatchQuery.of(mm -> mm + .query(searchTerm) + .fields("searchableText")))); - finalQuery = new Query.Builder() - .bool(b -> b - .must(matchQuery) - .filter(tableFilter)) - .build(); - } - else + BoolQuery.Builder boolBuilder = new BoolQuery.Builder() + .must(mustQuery) + .should(shouldClauses); + + if(tableName != null) { - finalQuery = matchQuery; + boolBuilder.filter(Query.of(q -> q + .term(TermQuery.of(t -> t + .field("sourceTable") + .value(FieldValue.of(tableName)))))); } - SearchRequest request = new SearchRequest.Builder() - .index(config.getOpensearchIndexName()) + Query finalQuery = Query.of(q -> q.bool(boolBuilder.build())); + + Highlight highlight = Highlight.of(h -> h + .fields("searchableText", HighlightField.of(hf -> hf))); + + int capturedOffset = offset; + int capturedLimit = limit; + + SearchRequest request = SearchRequest.of(r -> r + .index(indexName) .query(finalQuery) - .size(limit != null ? limit : 50) - .build(); + .highlight(highlight) + .from(capturedOffset) + .size(capturedLimit)); - SearchResponse response = client.search(request, OpenSearchDocument.class); - - List results = new ArrayList<>(); - for(Hit hit : response.hits().hits()) - { - if(hit.source() != null) - { - results.add(hit.source()); - } - } - - return results; + return (client.search(request, OpenSearchDocument.class)); } catch(IOException e) { - throw new QException("Failed to search documents", e); + throw new QException("Search failed for term [" + searchTerm + "]: " + e.getMessage(), e); } } - /*************************************************************************** - ** Close the client connection. - ***************************************************************************/ + /******************************************************************************* + ** Build should-clause boost queries from tableConfigs field weights. + ** + ** For each table config, emits a multi-match query for each field in + ** fieldWeights (if present) or searchableFields otherwise, targeting + ** fieldValues.{fieldName} with the configured boost. + ** + ** @param searchTerm the search string to match + ** @param tableConfigs the per-table configuration + ** @return list of boost queries, may be empty + *******************************************************************************/ + private List buildFieldBoostQueries(String searchTerm, List tableConfigs) + { + List queries = new ArrayList<>(); + + if(tableConfigs == null) + { + return (queries); + } + + for(QuickSearchableTableConfig tableConfig : tableConfigs) + { + if(tableConfig.getFieldWeights() != null) + { + for(Map.Entry entry : tableConfig.getFieldWeights().entrySet()) + { + String fieldPath = "fieldValues." + entry.getKey(); + double boostValue = entry.getValue().doubleValue(); + + queries.add(Query.of(q -> q + .multiMatch(MultiMatchQuery.of(mm -> mm + .query(searchTerm) + .fields(fieldPath) + .boost((float) boostValue))))); + } + } + else if(tableConfig.getSearchableFields() != null) + { + for(String field : tableConfig.getSearchableFields()) + { + String fieldPath = "fieldValues." + field; + + queries.add(Query.of(q -> q + .multiMatch(MultiMatchQuery.of(mm -> mm + .query(searchTerm) + .fields(fieldPath))))); + } + } + } + + return (queries); + } + + + + /******************************************************************************* + ** Close the underlying transport. + ** + ** Safe to call multiple times; subsequent calls after the first are no-ops. + ** Exceptions during close are logged but not re-thrown. + *******************************************************************************/ public void close() { - // The ApacheHttpClient5Transport handles cleanup + if(Boolean.TRUE.equals(closed)) + { + return; + } + + closed = true; + + try + { + transport.close(); + } + catch(Exception e) + { + LOG.warn("Error closing OpenSearch transport", e); + } } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java b/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java new file mode 100644 index 0000000..0113e6a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java @@ -0,0 +1,351 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.kingsrook.qbits.quicksearch.processes; + + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; + + +/******************************************************************************* + ** Abstract base class for Quick Search indexing process steps. + ** + ** Implements {@link BackendStep} but does NOT provide a run() implementation; + ** that is left to concrete subclasses (BasepullIndexStep, FullReindexStep). + ** + ** Provides protected helper methods for: + ** - Accessing QBit config, OpenSearch client, and discovered tables + ** - Ensuring quickSearchIndex rows exist for each table + ** - Creating and completing quickSearchIndexRun records + ** - Querying batches of records from source tables + *******************************************************************************/ +public abstract class AbstractIndexingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(AbstractIndexingStep.class); + + + + /******************************************************************************* + ** Return the active QuickSearchQBitConfig from context. + ** + ** @throws QException if config is null (QBit not initialized) + *******************************************************************************/ + protected QuickSearchQBitConfig getConfig() throws QException + { + QuickSearchQBitConfig config = QuickSearchQBitContext.getConfig(); + if(config == null) + { + throw new QException("QuickSearchQBitConfig is not initialized in QuickSearchQBitContext"); + } + return (config); + } + + + + /******************************************************************************* + ** Return the active OpenSearch client cast from context. + ** + ** @throws QException if client is null (not initialized) + *******************************************************************************/ + protected QuickSearchOpenSearchClient getClient() throws QException + { + Object client = QuickSearchQBitContext.getClient(); + if(client == null) + { + throw new QException("QuickSearchOpenSearchClient is not initialized in QuickSearchQBitContext"); + } + return ((QuickSearchOpenSearchClient) client); + } + + + + /******************************************************************************* + ** Return the list of discovered table configs from context. + ** + ** Returns an empty list if discoveredTables is null. + *******************************************************************************/ + protected List getDiscoveredTables() + { + List tables = QuickSearchQBitContext.getDiscoveredTables(); + if(tables == null) + { + return (Collections.emptyList()); + } + return (tables); + } + + + + /******************************************************************************* + ** Return the table config for the given table name from context. + ** + ** Delegates to QuickSearchQBitContext.getTableConfig(). + ** Returns null if not found. + *******************************************************************************/ + protected QuickSearchableTableConfig getTableConfig(String tableName) + { + return (QuickSearchQBitContext.getTableConfig(tableName)); + } + + + + /******************************************************************************* + ** Ensure a quickSearchIndex row exists for the given table. + ** + ** Queries the quickSearchIndex table for a row matching tableName. + ** If no row is found, inserts one with: + ** - tableName + ** - enabled = tableConfig.getEnabledByDefault() + ** - basepullIntervalMinutes from tableConfig + ** - basepullTimestampField from tableConfig + ** - searchableFieldsJson built from tableConfig's fields/weights/labels + ** - status = "ACTIVE" + ** + ** @param tableName the name of the source table + ** @param tableConfig the configuration for that table + ** @throws QException if the query or insert fails + *******************************************************************************/ + protected void ensureIndexRowExists(String tableName, QuickSearchableTableConfig tableConfig) throws QException + { + ////////////////////////////////////////////////////////////////////////////// + // Query for an existing index row // + ////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndex.TABLE_NAME); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + if(!queryOutput.getRecords().isEmpty()) + { + return; + } + + ////////////////////////////////////////////////////////////////////////////// + // Build and insert a new index row // + ////////////////////////////////////////////////////////////////////////////// + String searchableFieldsJson = buildSearchableFieldsJson(tableConfig); + + QRecord record = new QRecord() + .withValue("tableName", tableName) + .withValue("enabled", tableConfig.getEnabledByDefault()) + .withValue("basepullIntervalMinutes", tableConfig.getBasepullIntervalMinutes()) + .withValue("basepullTimestampField", tableConfig.getBasepullTimestampField()) + .withValue("searchableFieldsJson", searchableFieldsJson) + .withValue("status", "ACTIVE"); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(QuickSearchIndex.TABLE_NAME); + insertInput.setRecords(List.of(record)); + + new InsertAction().execute(insertInput); + } + + + + /******************************************************************************* + ** Build a JSON array string representing the searchable fields configuration + ** for a table. + ** + ** Each element looks like: + ** {"fieldName":"name","weight":1,"includeLabel":false} + ** + ** @param tableConfig the table configuration containing field lists/maps + ** @return JSON string, e.g. [{"fieldName":"name","weight":1,"includeLabel":false}] + *******************************************************************************/ + private String buildSearchableFieldsJson(QuickSearchableTableConfig tableConfig) + { + if(tableConfig.getSearchableFields() == null || tableConfig.getSearchableFields().isEmpty()) + { + return ("[]"); + } + + StringBuilder sb = new StringBuilder("["); + boolean first = true; + + for(String fieldName : tableConfig.getSearchableFields()) + { + if(!first) + { + sb.append(","); + } + first = false; + + Integer weight = 1; + if(tableConfig.getFieldWeights() != null && tableConfig.getFieldWeights().containsKey(fieldName)) + { + weight = tableConfig.getFieldWeights().get(fieldName); + } + + Boolean includeLabel = false; + if(tableConfig.getFieldIncludeLabels() != null && tableConfig.getFieldIncludeLabels().containsKey(fieldName)) + { + includeLabel = tableConfig.getFieldIncludeLabels().get(fieldName); + } + + sb.append("{\"fieldName\":\"").append(fieldName).append("\"") + .append(",\"weight\":").append(weight) + .append(",\"includeLabel\":").append(includeLabel) + .append("}"); + } + + sb.append("]"); + return (sb.toString()); + } + + + + /******************************************************************************* + ** Create a quickSearchIndexRun record and return it with its generated ID. + ** + ** Inserts a new record with status=RUNNING and startTime=now, then reads + ** back the inserted record to capture the auto-generated primary key. + ** + ** @param indexId the ID of the parent quickSearchIndex row + ** @param runType the type label for this run (e.g. "BASEPULL", "FULL_REINDEX") + ** @return a QuickSearchIndexRun entity with id populated + ** @throws QException if insert fails or no record is returned + *******************************************************************************/ + protected QuickSearchIndexRun createRunRecord(Integer indexId, String runType) throws QException + { + QRecord record = new QRecord() + .withValue("quickSearchIndexId", indexId) + .withValue("runType", runType) + .withValue("status", "RUNNING") + .withValue("startTime", Instant.now()); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(QuickSearchIndexRun.TABLE_NAME); + insertInput.setRecords(List.of(record)); + + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + if(insertOutput.getRecords().isEmpty()) + { + throw new QException("Failed to create quickSearchIndexRun record: no record returned from insert"); + } + + QRecord insertedRecord = insertOutput.getRecords().get(0); + + QuickSearchIndexRun run = new QuickSearchIndexRun() + .withId(insertedRecord.getValueInteger("id")) + .withQuickSearchIndexId(insertedRecord.getValueInteger("quickSearchIndexId")) + .withRunType(insertedRecord.getValueString("runType")) + .withStatus(insertedRecord.getValueString("status")) + .withStartTime(insertedRecord.getValueInstant("startTime")); + return (run); + } + + + + /******************************************************************************* + ** Mark a quickSearchIndexRun as complete by updating its status, endTime, + ** record counts, and error information. + ** + ** Wrapped in try/catch so that failures to update the run record do not + ** mask original exceptions when called from finally blocks. + ** + ** @param run the run record to update + ** @param status the final status (e.g. "SUCCESS", "ERROR") + ** @param recordsProcessed number of records examined + ** @param recordsIndexed number of records successfully indexed + ** @param errorCount number of errors encountered + ** @param errorMessage summary error message, or null + *******************************************************************************/ + protected void completeRunRecord(QuickSearchIndexRun run, String status, Integer recordsProcessed, + Integer recordsIndexed, Integer errorCount, String errorMessage) + { + try + { + QRecord record = new QRecord() + .withValue("id", run.getId()) + .withValue("status", status) + .withValue("endTime", Instant.now()) + .withValue("recordsProcessed", recordsProcessed) + .withValue("recordsIndexed", recordsIndexed) + .withValue("errorCount", errorCount) + .withValue("errorMessage", errorMessage); + + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(QuickSearchIndexRun.TABLE_NAME); + updateInput.setRecords(List.of(record)); + + new UpdateAction().execute(updateInput); + } + catch(Exception e) + { + LOG.warn("Failed to complete run record", e, "runId", run.getId(), "status", status); + } + } + + + + /******************************************************************************* + ** Query a batch of records from the given source table. + ** + ** @param tableName the name of the source table to query + ** @param filter the filter to apply (may be null for no filtering) + ** @param limit maximum number of records to return + ** @param offset number of records to skip (for pagination) + ** @return list of QRecord from the query + ** @throws QException if the query fails + *******************************************************************************/ + protected List querySourceTableBatch(String tableName, QQueryFilter filter, Integer limit, Integer offset) throws QException + { + QQueryFilter effectiveFilter = (filter != null) ? filter : new QQueryFilter(); + effectiveFilter.setLimit(limit); + effectiveFilter.setSkip(offset); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + queryInput.setFilter(effectiveFilter); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + if(records == null) + { + return (new ArrayList<>()); + } + + return (records); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStep.java b/src/main/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStep.java index 39f8518..c542a72 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStep.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStep.java @@ -20,21 +20,12 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; -import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; -import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; -import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; -import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -42,329 +33,269 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; +import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; /******************************************************************************* - ** Step that performs incremental basepull indexing of Quick Search indexes. + ** Incremental basepull indexing process step. + ** + ** Queries all enabled QuickSearchIndex rows, checks whether each is due for a + ** basepull run, and for those that are, queries the source table for records + ** modified since the last basepull, builds OpenSearch documents, and bulk-indexes + ** them. + ** + ** Run records are created at the start of each per-table run and updated with + ** final status and counts on completion. Per-table failures are caught and + ** logged so that one bad table does not prevent others from being processed. *******************************************************************************/ -public class BasepullIndexStep implements BackendStep +public class BasepullIndexStep extends AbstractIndexingStep { private static final QLogger LOG = QLogger.getLogger(BasepullIndexStep.class); - public static final String FIELD_CONFIG = "quickSearchConfig"; - - /*************************************************************************** - ** Execute the basepull indexing. - ***************************************************************************/ + /******************************************************************************* + ** Main entry point called by the QQQ process engine. + ** + ** 1. For each discovered table, ensure a quickSearchIndex row exists. + ** 2. Query all enabled quickSearchIndex rows. + ** 3. For each enabled row, check isDueForBasepull and run basepullIndex. + ** + ** @param input process step input (not directly used) + ** @param output process step output (not directly used) + ** @throws QException if an unrecoverable error occurs (per-table errors are caught) + *******************************************************************************/ @Override public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException { - QuickSearchQBitConfig config = (QuickSearchQBitConfig) input.getValue(FIELD_CONFIG); - - if(config == null) + //////////////////////////////////////////////////// + // 1. Ensure index rows exist for all tables // + //////////////////////////////////////////////////// + List discoveredTables = getDiscoveredTables(); + for(QuickSearchableTableConfig tableConfig : discoveredTables) { - throw new QException("QuickSearchQBitConfig is required"); + ensureIndexRowExists(tableConfig.getTableName(), tableConfig); } - List indexes = loadEnabledIndexesDueForBasepull(config); + //////////////////////////////////////////////////// + // 2. Query all enabled quickSearchIndex rows // + //////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndex.TABLE_NAME); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true))); - if(indexes.isEmpty()) + QueryOutput queryOutput = new QueryAction().execute(queryInput); + List indexRecords = queryOutput.getRecords(); + + if(indexRecords == null || indexRecords.isEmpty()) { - LOG.debug("No indexes due for basepull"); + LOG.info("No enabled quickSearchIndex rows found; nothing to basepull"); return; } - QuickSearchOpenSearchClient searchClient = new QuickSearchOpenSearchClient(config); + //////////////////////////////////////////////////// + // 3. Process each enabled index row // + //////////////////////////////////////////////////// + for(QRecord indexRecord : indexRecords) + { + QuickSearchIndex index = new QuickSearchIndex() + .withId(indexRecord.getValueInteger("id")) + .withTableName(indexRecord.getValueString("tableName")) + .withEnabled(indexRecord.getValueBoolean("enabled")) + .withBasepullIntervalMinutes(indexRecord.getValueInteger("basepullIntervalMinutes")) + .withBasepullTimestampField(indexRecord.getValueString("basepullTimestampField")) + .withLastBasepullTime(indexRecord.getValueInstant("lastBasepullTime")); - try - { - for(QuickSearchIndex index : indexes) + if(!isDueForBasepull(index)) { - try - { - basepullIndex(config, searchClient, index); - } - catch(Exception e) - { - LOG.warn("Basepull failed for index", logPair("tableName", index.getTableName()), e); - } + LOG.info("Index not due for basepull; skipping", + "tableName", index.getTableName(), + "lastBasepullTime", index.getLastBasepullTime()); + continue; + } + + try + { + basepullIndex(index); + } + catch(Exception e) + { + LOG.warn("Error during basepull for table; continuing with next table", + e, "tableName", index.getTableName()); } - } - finally - { - searchClient.close(); } } - /*************************************************************************** - ** Load enabled indexes that are due for basepull. - ***************************************************************************/ - private List loadEnabledIndexesDueForBasepull(QuickSearchQBitConfig config) throws QException + /******************************************************************************* + ** Determine whether this index is due for a basepull run. + ** + ** Returns true when lastBasepullTime is null (never run), or when the elapsed + ** time since the last run has exceeded the configured interval. + ** + ** @param index the QuickSearchIndex row to evaluate + ** @return true if a basepull should be executed for this index + *******************************************************************************/ + boolean isDueForBasepull(QuickSearchIndex index) { - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); + Instant lastBasepullTime = index.getLastBasepullTime(); - QueryInput queryInput = new QueryInput() - .withTableName(indexTableName) - .withFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria("isEnabled", QCriteriaOperator.EQUALS, true))); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - Instant now = Instant.now(); - List dueIndexes = new ArrayList<>(); - - for(QRecord record : queryOutput.getRecords()) + if(lastBasepullTime == null) { - QuickSearchIndex index = mapRecordToIndex(record); - if(isDueForBasepull(index, now)) - { - dueIndexes.add(index); - } - } - - return dueIndexes; - } - - - - /*************************************************************************** - ** Check if an index is due for basepull based on its interval. - ***************************************************************************/ - private boolean isDueForBasepull(QuickSearchIndex index, Instant now) - { - if(index.getLastBasepullTime() == null) - { - return true; + return (true); } Integer intervalMinutes = index.getBasepullIntervalMinutes(); - if(intervalMinutes == null || intervalMinutes <= 0) + if(intervalMinutes == null) { - intervalMinutes = 5; + return (true); } - Instant nextDue = index.getLastBasepullTime().plus(intervalMinutes, ChronoUnit.MINUTES); - return now.isAfter(nextDue); + return (lastBasepullTime.plus(intervalMinutes, ChronoUnit.MINUTES).isBefore(Instant.now())); } - /*************************************************************************** - ** Perform basepull indexing for a single index. - ***************************************************************************/ - private void basepullIndex(QuickSearchQBitConfig config, QuickSearchOpenSearchClient searchClient, QuickSearchIndex index) throws QException + /******************************************************************************* + ** Execute a basepull indexing run for one QuickSearchIndex row. + ** + ** Steps: + ** 1. Resolve table config and client. + ** 2. Create a run record (status=RUNNING). + ** 3. Build an optional QQueryFilter using lastBasepullTime. + ** 4. Paginate through the source table, building and indexing documents. + ** 5. Update lastBasepullTime on the QuickSearchIndex row. + ** 6. Complete the run record with final status and counts. + ** + ** @param index the QuickSearchIndex row driving this run + ** @throws QException if an error occurs that cannot be handled locally + *******************************************************************************/ + void basepullIndex(QuickSearchIndex index) throws QException { - String tableName = index.getTableName(); - LOG.debug("Starting basepull", logPair("tableName", tableName)); + String tableName = index.getTableName(); + QuickSearchableTableConfig tableConfig = getTableConfig(tableName); + QuickSearchOpenSearchClient client = getClient(); + QuickSearchQBitConfig config = getConfig(); - Instant startTime = Instant.now(); - QuickSearchIndexRun run = createRunRecord(config, index, "BASEPULL", "RUNNING", startTime); + QuickSearchIndexRun run = createRunRecord(index.getId(), "BASEPULL"); - int recordsProcessed = 0; - int recordsIndexed = 0; - String errorMessage = null; + Integer totalProcessed = 0; + Integer totalIndexed = 0; + Integer totalErrors = 0; + String errorMessage = null; try { - QTableMetaData table = QContext.getQInstance().getTable(tableName); - if(table == null) + /////////////////////////////////////////////////////////////////// + // Build filter: timestamp field > lastBasepullTime (if set) // + /////////////////////////////////////////////////////////////////// + QQueryFilter filter = null; + + if(index.getLastBasepullTime() != null && tableConfig != null + && tableConfig.getBasepullTimestampField() != null) { - throw new QException("Table not found: " + tableName); + filter = new QQueryFilter() + .withCriteria(new QFilterCriteria( + tableConfig.getBasepullTimestampField(), + QCriteriaOperator.GREATER_THAN, + index.getLastBasepullTime())); } - List searchableFields = parseFieldsJson(index.getSearchableFieldsJson()); - String timestampField = index.getBasepullTimestampField(); + Integer sourceBatchSize = config.getSourceBatchSize(); + Integer bulkBatchSize = config.getBulkBatchSize(); - if(!StringUtils.hasContent(timestampField)) + /////////////////////////////////////////////////////////////////// + // Paginate through the source table // + /////////////////////////////////////////////////////////////////// + int offset = 0; + + while(true) { - timestampField = "modifyDate"; - } + List batch = querySourceTableBatch(tableName, filter, sourceBatchSize, offset); - ///////////////////////////////////////////// - // Build query for records since last run // - ///////////////////////////////////////////// - QQueryFilter filter = new QQueryFilter(); - Instant lastRun = index.getLastBasepullTime(); - - if(lastRun != null) - { - filter.withCriteria(new QFilterCriteria(timestampField, QCriteriaOperator.GREATER_THAN, lastRun)); - } - filter.withCriteria(new QFilterCriteria(timestampField, QCriteriaOperator.LESS_THAN_OR_EQUALS, startTime)); - - QueryInput queryInput = new QueryInput() - .withTableName(tableName) - .withFilter(filter); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - recordsProcessed = queryOutput.getRecords().size(); - - if(recordsProcessed > 0) - { - ///////////////////////////////////// - // Build and index documents // - ///////////////////////////////////// - List documents = new ArrayList<>(); - for(QRecord record : queryOutput.getRecords()) + if(batch.isEmpty()) { - OpenSearchDocument doc = IndexingUtils.buildDocument(record, table, searchableFields); - if(StringUtils.hasContent(doc.getSearchableText())) + break; + } + + //////////////////////////////////////////////////// + // Build documents for this batch // + //////////////////////////////////////////////////// + List documents = new ArrayList<>(); + + for(QRecord record : batch) + { + if(tableConfig != null) { + OpenSearchDocument doc = IndexingUtils.buildDocument( + record, + tableName, + tableConfig.getPrimaryKeyField(), + tableConfig.getSearchableFields(), + tableConfig.getFieldWeights(), + tableConfig.getFieldIncludeLabels() != null ? tableConfig.getFieldIncludeLabels() : java.util.Collections.emptyMap()); documents.add(doc); } } + //////////////////////////////////////////////////// + // Bulk-index the batch // + //////////////////////////////////////////////////// if(!documents.isEmpty()) { - searchClient.indexDocuments(documents); - recordsIndexed = documents.size(); + BulkIndexResult result = client.indexDocuments(documents, bulkBatchSize); + totalIndexed = totalIndexed + result.getSuccessCount(); + totalErrors = totalErrors + result.getFailureCount(); + + if(!result.getErrors().isEmpty() && errorMessage == null) + { + errorMessage = result.getErrors().get(0); + } } - LOG.info("Basepull complete", logPair("tableName", tableName), logPair("recordsIndexed", recordsIndexed)); + totalProcessed = totalProcessed + batch.size(); + offset = offset + batch.size(); } - ///////////////////////////////////// - // Update index status // - ///////////////////////////////////// - updateLastBasepullTime(config, index, startTime); + //////////////////////////////////////////////////// + // Update lastBasepullTime on the index row // + //////////////////////////////////////////////////// + QRecord updateRecord = new QRecord() + .withValue("id", index.getId()) + .withValue("lastBasepullTime", Instant.now()); + + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(QuickSearchIndex.TABLE_NAME); + updateInput.setRecords(List.of(updateRecord)); + + new UpdateAction().execute(updateInput); + + //////////////////////////////////////////////////// + // Complete the run record // + //////////////////////////////////////////////////// + String finalStatus = (totalErrors > 0) ? "FAILED" : "COMPLETED"; + completeRunRecord(run, finalStatus, totalProcessed, totalIndexed, totalErrors, errorMessage); + + LOG.info("Basepull complete", + "tableName", tableName, + "totalProcessed", totalProcessed, + "totalIndexed", totalIndexed, + "totalErrors", totalErrors); } catch(Exception e) { - errorMessage = e.getMessage(); - LOG.error("Basepull failed", logPair("tableName", tableName), e); - throw new QException("Basepull failed for table: " + tableName, e); + LOG.warn("Basepull failed for table", e, "tableName", tableName); + completeRunRecord(run, "FAILED", totalProcessed, totalIndexed, totalErrors, e.getMessage()); + throw e; } - finally - { - completeRunRecord(config, run, recordsProcessed, recordsIndexed, errorMessage); - } - } - - - - /*************************************************************************** - ** Parse searchable fields from JSON. - ***************************************************************************/ - @SuppressWarnings("unchecked") - private List parseFieldsJson(String json) - { - if(!StringUtils.hasContent(json)) - { - return new ArrayList<>(); - } - - try - { - return JsonUtils.toObject(json, List.class); - } - catch(Exception e) - { - LOG.warn("Failed to parse searchable fields JSON", logPair("json", json), e); - return new ArrayList<>(); - } - } - - - - /*************************************************************************** - ** Create a run record for tracking. - ***************************************************************************/ - private QuickSearchIndexRun createRunRecord(QuickSearchQBitConfig config, QuickSearchIndex index, String runType, String status, Instant startTime) throws QException - { - String runTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); - - QRecord runRecord = new QRecord() - .withValue("quickSearchIndexId", index.getId()) - .withValue("runType", runType) - .withValue("status", status) - .withValue("startTime", startTime); - - InsertInput insertInput = new InsertInput() - .withTableName(runTableName) - .withRecords(List.of(runRecord)); - - new InsertAction().execute(insertInput); - - QuickSearchIndexRun run = new QuickSearchIndexRun() - .withQuickSearchIndexId(index.getId()) - .withRunType(runType) - .withStatus(status) - .withStartTime(startTime); - return run; - } - - - - /*************************************************************************** - ** Map a QRecord to a QuickSearchIndex entity. - ***************************************************************************/ - private QuickSearchIndex mapRecordToIndex(QRecord record) - { - return new QuickSearchIndex() - .withId(record.getValueInteger("id")) - .withTableName(record.getValueString("tableName")) - .withIsEnabled(record.getValueBoolean("isEnabled")) - .withBasepullIntervalMinutes(record.getValueInteger("basepullIntervalMinutes")) - .withBasepullTimestampField(record.getValueString("basepullTimestampField")) - .withSearchableFieldsJson(record.getValueString("searchableFieldsJson")) - .withLastFullIndexTime(record.getValueInstant("lastFullIndexTime")) - .withLastBasepullTime(record.getValueInstant("lastBasepullTime")) - .withIndexedRecordCount(record.getValueInteger("indexedRecordCount")); - } - - - - /*************************************************************************** - ** Complete a run record with final status. - ***************************************************************************/ - private void completeRunRecord(QuickSearchQBitConfig config, QuickSearchIndexRun run, int recordsProcessed, int recordsIndexed, String errorMessage) throws QException - { - String runTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); - - String status = errorMessage == null ? "COMPLETE" : "ERROR"; - - QRecord updateRecord = new QRecord() - .withValue("id", run.getId()) - .withValue("status", status) - .withValue("endTime", Instant.now()) - .withValue("recordsProcessed", recordsProcessed) - .withValue("recordsIndexed", recordsIndexed) - .withValue("errorMessage", errorMessage); - - UpdateInput updateInput = new UpdateInput() - .withTableName(runTableName) - .withRecords(List.of(updateRecord)); - - new UpdateAction().execute(updateInput); - } - - - - /*************************************************************************** - ** Update the last basepull time on the index. - ***************************************************************************/ - private void updateLastBasepullTime(QuickSearchQBitConfig config, QuickSearchIndex index, Instant basepullTime) throws QException - { - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); - - QRecord updateRecord = new QRecord() - .withValue("id", index.getId()) - .withValue("lastBasepullTime", basepullTime) - .withValue("modifyDate", Instant.now()); - - UpdateInput updateInput = new UpdateInput() - .withTableName(indexTableName) - .withRecords(List.of(updateRecord)); - - new UpdateAction().execute(updateInput); } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStep.java b/src/main/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStep.java index 2f6bbb1..12441ea 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStep.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStep.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,29 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.processes; -import java.io.Serializable; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; -import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; -import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -43,287 +33,190 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; +import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; /******************************************************************************* - ** Step that performs full reindexing of Quick Search indexes. + ** Process step that performs a full reindex of one or all tables. + ** + ** Deletes all existing OpenSearch documents for each target table, then + ** re-fetches and re-indexes every record in batches. Updates the + ** QuickSearchIndex row with lastFullReindexTime and recordCount when done. + ** + ** An optional "tableName" input field narrows the run to a single table. + ** When absent, all discovered tables are reindexed. *******************************************************************************/ -public class FullReindexStep implements BackendStep +public class FullReindexStep extends AbstractIndexingStep { private static final QLogger LOG = QLogger.getLogger(FullReindexStep.class); - public static final String FIELD_TABLE_NAME = "tableName"; - public static final String FIELD_CONFIG = "quickSearchConfig"; - - /*************************************************************************** - ** Execute the full reindex. - ***************************************************************************/ + /******************************************************************************* + ** Run the full reindex step. + ** + ** Reads the optional "tableName" field from the input. When present, only + ** that table is reindexed; otherwise every discovered table is processed. + ** + ** @param input the backend step input (may contain "tableName") + ** @param output the backend step output (unused but required by interface) + ** @throws QException if a fatal error occurs preventing any reindex + *******************************************************************************/ @Override public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException { - String targetTableName = input.getValueString(FIELD_TABLE_NAME); - QuickSearchQBitConfig config = (QuickSearchQBitConfig) input.getValue(FIELD_CONFIG); + String tableNameFilter = input.getValueString("tableName"); - if(config == null) + List tables = getDiscoveredTables(); + + if(tableNameFilter != null && !tableNameFilter.isBlank()) { - throw new QException("QuickSearchQBitConfig is required"); - } - - List indexes = loadIndexes(config, targetTableName); - - if(indexes.isEmpty()) - { - LOG.info("No indexes found to reindex", logPair("targetTableName", targetTableName)); - return; - } - - QuickSearchOpenSearchClient searchClient = new QuickSearchOpenSearchClient(config); - - try - { - for(QuickSearchIndex index : indexes) + List filtered = new ArrayList<>(); + for(QuickSearchableTableConfig tableConfig : tables) { - reindexTable(config, searchClient, index); + if(tableNameFilter.equals(tableConfig.getTableName())) + { + filtered.add(tableConfig); + } } + tables = filtered; } - finally + + for(QuickSearchableTableConfig tableConfig : tables) { - searchClient.close(); + String tableName = tableConfig.getTableName(); + + ensureIndexRowExists(tableName, tableConfig); + reindexTable(tableName, tableConfig); } } - /*************************************************************************** - ** Load indexes to process. - ***************************************************************************/ - private List loadIndexes(QuickSearchQBitConfig config, String targetTableName) throws QException + /******************************************************************************* + ** Reindex all records for a single table. + ** + ** Flow: + ** 1. Obtain the QuickSearchIndex row for this table (to get its ID). + ** 2. Create a FULL_REINDEX run record. + ** 3. Delete all existing OpenSearch documents for the table. + ** 4. Paginate through all source records and index them in batches. + ** 5. Update lastFullReindexTime and recordCount on the QuickSearchIndex row. + ** 6. Complete the run record with final stats. + ** + ** On any exception, the run record is marked FAILED before re-throwing. + ** + ** @param tableName the name of the source table + ** @param tableConfig the configuration for that table + ** @throws QException if a fatal indexing error occurs + *******************************************************************************/ + void reindexTable(String tableName, QuickSearchableTableConfig tableConfig) throws QException { - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); + QuickSearchOpenSearchClient client = getClient(); + QuickSearchQBitConfig config = getConfig(); - QQueryFilter filter = new QQueryFilter(); - if(StringUtils.hasContent(targetTableName)) - { - filter.withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, targetTableName)); - } - - QueryInput queryInput = new QueryInput() - .withTableName(indexTableName) - .withFilter(filter); + //////////////////////////////////////////////////// + // Query for the index row to get its ID // + //////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndex.TABLE_NAME); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName))); QueryOutput queryOutput = new QueryAction().execute(queryInput); - List indexes = new ArrayList<>(); - for(QRecord record : queryOutput.getRecords()) + if(queryOutput.getRecords().isEmpty()) { - indexes.add(mapRecordToIndex(record)); + throw new QException("No QuickSearchIndex row found for table: " + tableName); } - return indexes; - } + QRecord indexRecord = queryOutput.getRecords().get(0); + Integer indexId = indexRecord.getValueInteger("id"); + QuickSearchIndexRun run = createRunRecord(indexId, "FULL_REINDEX"); - - /*************************************************************************** - ** Reindex a single table. - ***************************************************************************/ - private void reindexTable(QuickSearchQBitConfig config, QuickSearchOpenSearchClient searchClient, QuickSearchIndex index) throws QException - { - String tableName = index.getTableName(); - LOG.info("Starting full reindex", logPair("tableName", tableName)); - - Instant startTime = Instant.now(); - QuickSearchIndexRun run = createRunRecord(config, index, "FULL", "RUNNING", startTime); - - int recordsProcessed = 0; - int recordsIndexed = 0; - String errorMessage = null; + Integer totalProcessed = 0; + Integer totalIndexed = 0; + Integer totalErrors = 0; try { - /////////////////////////////////////// - // Delete existing documents for table // - /////////////////////////////////////// - searchClient.deleteDocumentsForTable(tableName); + //////////////////////////////////////////////////// + // Delete existing documents for this table // + //////////////////////////////////////////////////// + client.deleteDocumentsForTable(tableName); - /////////////////////////////////// - // Query all records from source // - /////////////////////////////////// - QTableMetaData table = QContext.getQInstance().getTable(tableName); - if(table == null) + //////////////////////////////////////////////////// + // Paginate through all records and index them // + //////////////////////////////////////////////////// + Integer sourceBatchSize = config.getSourceBatchSize(); + Integer offset = 0; + + while(true) { - throw new QException("Table not found: " + tableName); - } + List batch = querySourceTableBatch(tableName, new QQueryFilter(), sourceBatchSize, offset); - List searchableFields = parseFieldsJson(index.getSearchableFieldsJson()); - - QueryInput queryInput = new QueryInput().withTableName(tableName); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - recordsProcessed = queryOutput.getRecords().size(); - - ///////////////////////////////////// - // Build and index documents // - ///////////////////////////////////// - List documents = new ArrayList<>(); - for(QRecord record : queryOutput.getRecords()) - { - OpenSearchDocument doc = IndexingUtils.buildDocument(record, table, searchableFields); - if(StringUtils.hasContent(doc.getSearchableText())) + if(batch.isEmpty()) { + break; + } + + List documents = new ArrayList<>(); + for(QRecord record : batch) + { + OpenSearchDocument doc = IndexingUtils.buildDocument( + record, + tableName, + tableConfig.getPrimaryKeyField(), + tableConfig.getSearchableFields(), + tableConfig.getFieldWeights(), + tableConfig.getFieldIncludeLabels() != null ? tableConfig.getFieldIncludeLabels() : java.util.Collections.emptyMap()); documents.add(doc); } + + BulkIndexResult result = client.indexDocuments(documents, config.getBulkBatchSize()); + + totalProcessed += batch.size(); + totalIndexed += result.getSuccessCount(); + totalErrors += result.getFailureCount(); + + offset += batch.size(); } - if(!documents.isEmpty()) - { - searchClient.indexDocuments(documents); - recordsIndexed = documents.size(); - } + //////////////////////////////////////////////////// + // Update the QuickSearchIndex row // + //////////////////////////////////////////////////// + QRecord updateRecord = new QRecord() + .withValue("id", indexId) + .withValue("lastFullReindexTime", Instant.now()) + .withValue("recordCount", totalProcessed); - ///////////////////////////////////// - // Update index status // - ///////////////////////////////////// - updateIndexStatus(config, index, startTime, recordsIndexed); + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(QuickSearchIndex.TABLE_NAME); + updateInput.setRecords(List.of(updateRecord)); - LOG.info("Full reindex complete", logPair("tableName", tableName), logPair("recordsIndexed", recordsIndexed)); + new UpdateAction().execute(updateInput); + + //////////////////////////////////////////////////// + // Complete the run record as success // + //////////////////////////////////////////////////// + completeRunRecord(run, "SUCCESS", totalProcessed, totalIndexed, totalErrors, null); + + LOG.info("Full reindex complete", "tableName", tableName, "totalProcessed", totalProcessed, + "totalIndexed", totalIndexed, "totalErrors", totalErrors); } catch(Exception e) { - errorMessage = e.getMessage(); - LOG.error("Full reindex failed", logPair("tableName", tableName), e); - throw new QException("Full reindex failed for table: " + tableName, e); + LOG.warn("Full reindex failed", e, "tableName", tableName); + completeRunRecord(run, "FAILED", totalProcessed, totalIndexed, totalErrors, e.getMessage()); + throw new QException("Full reindex failed for table [" + tableName + "]: " + e.getMessage(), e); } - finally - { - completeRunRecord(config, run, recordsProcessed, recordsIndexed, errorMessage); - } - } - - - - /*************************************************************************** - ** Parse searchable fields from JSON. - ***************************************************************************/ - @SuppressWarnings("unchecked") - private List parseFieldsJson(String json) - { - if(!StringUtils.hasContent(json)) - { - return new ArrayList<>(); - } - - try - { - return JsonUtils.toObject(json, List.class); - } - catch(Exception e) - { - LOG.warn("Failed to parse searchable fields JSON", logPair("json", json), e); - return new ArrayList<>(); - } - } - - - - /*************************************************************************** - ** Create a run record for tracking. - ***************************************************************************/ - private QuickSearchIndexRun createRunRecord(QuickSearchQBitConfig config, QuickSearchIndex index, String runType, String status, Instant startTime) throws QException - { - String runTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); - - QRecord runRecord = new QRecord() - .withValue("quickSearchIndexId", index.getId()) - .withValue("runType", runType) - .withValue("status", status) - .withValue("startTime", startTime); - - InsertInput insertInput = new InsertInput() - .withTableName(runTableName) - .withRecords(List.of(runRecord)); - - new InsertAction().execute(insertInput); - - QuickSearchIndexRun run = new QuickSearchIndexRun() - .withQuickSearchIndexId(index.getId()) - .withRunType(runType) - .withStatus(status) - .withStartTime(startTime); - return run; - } - - - - /*************************************************************************** - ** Map a QRecord to a QuickSearchIndex entity. - ***************************************************************************/ - private QuickSearchIndex mapRecordToIndex(QRecord record) - { - return new QuickSearchIndex() - .withId(record.getValueInteger("id")) - .withTableName(record.getValueString("tableName")) - .withIsEnabled(record.getValueBoolean("isEnabled")) - .withBasepullIntervalMinutes(record.getValueInteger("basepullIntervalMinutes")) - .withBasepullTimestampField(record.getValueString("basepullTimestampField")) - .withSearchableFieldsJson(record.getValueString("searchableFieldsJson")) - .withLastFullIndexTime(record.getValueInstant("lastFullIndexTime")) - .withLastBasepullTime(record.getValueInstant("lastBasepullTime")) - .withIndexedRecordCount(record.getValueInteger("indexedRecordCount")); - } - - - - /*************************************************************************** - ** Complete a run record with final status. - ***************************************************************************/ - private void completeRunRecord(QuickSearchQBitConfig config, QuickSearchIndexRun run, int recordsProcessed, int recordsIndexed, String errorMessage) throws QException - { - String runTableName = config.applyPrefix(QuickSearchIndexRun.TABLE_NAME); - - String status = errorMessage == null ? "COMPLETE" : "ERROR"; - - QRecord updateRecord = new QRecord() - .withValue("id", run.getId()) - .withValue("status", status) - .withValue("endTime", Instant.now()) - .withValue("recordsProcessed", recordsProcessed) - .withValue("recordsIndexed", recordsIndexed) - .withValue("errorMessage", errorMessage); - - UpdateInput updateInput = new UpdateInput() - .withTableName(runTableName) - .withRecords(List.of(updateRecord)); - - new UpdateAction().execute(updateInput); - } - - - - /*************************************************************************** - ** Update the index status after successful reindex. - ***************************************************************************/ - private void updateIndexStatus(QuickSearchQBitConfig config, QuickSearchIndex index, Instant fullIndexTime, int recordCount) throws QException - { - String indexTableName = config.applyPrefix(QuickSearchIndex.TABLE_NAME); - - QRecord updateRecord = new QRecord() - .withValue("id", index.getId()) - .withValue("lastFullIndexTime", fullIndexTime) - .withValue("indexedRecordCount", recordCount) - .withValue("modifyDate", Instant.now()); - - UpdateInput updateInput = new UpdateInput() - .withTableName(indexTableName) - .withRecords(List.of(updateRecord)); - - new UpdateAction().execute(updateInput); } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtils.java b/src/main/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtils.java index 7140c27..f5d2c53 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtils.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtils.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,88 +13,151 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.processes; import java.time.Instant; +import java.util.HashMap; import java.util.List; -import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import java.util.Map; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; /******************************************************************************* - ** Utilities for building searchable content from QQQ records. + ** Static utility class for text normalization and OpenSearch document building. + ** + ** Provides helpers used during the indexing pipeline to normalize field values + ** into searchable text and to assemble OpenSearchDocument instances from + ** QRecord data. *******************************************************************************/ public class IndexingUtils { - /*************************************************************************** - ** Build searchable text blob from a record's fields. - ***************************************************************************/ - public static String buildSearchableText(QRecord record, List fieldNames) + /******************************************************************************* + ** Private constructor to prevent instantiation of this utility class. + *******************************************************************************/ + private IndexingUtils() { + } + + + + /******************************************************************************* + ** Normalize text for search indexing. + ** + ** Converts to lowercase, trims leading/trailing whitespace, and collapses + ** runs of internal whitespace to a single space. Returns empty string for + ** null or blank input. + ** + ** @param text the raw text to normalize + ** @return the normalized text, never null + *******************************************************************************/ + public static String normalizeSearchText(String text) + { + if(text == null || text.isBlank()) + { + return (""); + } + + return (text.toLowerCase().trim().replaceAll("\\s+", " ")); + } + + + + /******************************************************************************* + ** Build a concatenated searchable text string from selected record fields. + ** + ** Iterates over each field name in {@code fields}, retrieves the string value + ** from the record, and appends it to the result. Fields with a null or blank + ** value are skipped. When {@code includeLabels} maps a field name to + ** {@code true}, the value is prefixed with {@code "fieldName: "}. + ** + ** @param record the QRecord to read field values from + ** @param fields ordered list of field names to include + ** @param includeLabels map of field name to whether its label should be prefixed + ** @return the concatenated text, trimmed; empty string if fields is null or empty + *******************************************************************************/ + public static String buildSearchableText(QRecord record, List fields, Map includeLabels) + { + if(fields == null || fields.isEmpty()) + { + return (""); + } + StringBuilder sb = new StringBuilder(); - for(String fieldName : fieldNames) + for(String fieldName : fields) { - Object value = record.getValue(fieldName); - if(value != null) + String value = record.getValueString(fieldName); + + if(value == null || value.isBlank()) { - String textValue = String.valueOf(value); - if(!textValue.isBlank()) + continue; + } + + if(sb.length() > 0) + { + sb.append(" "); + } + + if(Boolean.TRUE.equals(includeLabels.get(fieldName))) + { + sb.append(fieldName).append(": "); + } + + sb.append(value); + } + + return (sb.toString().trim()); + } + + + + /******************************************************************************* + ** Build an OpenSearchDocument from a QRecord. + ** + ** Delegates to {@link #buildSearchableText} for the searchable text, reads the + ** record ID from {@code primaryKeyField}, and populates {@code fieldValues} with + ** only those entries where the record's value is non-null. + ** + ** @param record the source QRecord + ** @param tableName the QQQ table name used as the document's sourceTable + ** @param primaryKeyField the field name whose value becomes the document's recordId + ** @param fields ordered list of field names for searchable text + ** @param fieldWeights map of field name to search weight (reserved for future use) + ** @param includeLabels map of field name to whether its label should be prefixed + ** @return a fully populated OpenSearchDocument with indexedAt set to now + *******************************************************************************/ + public static OpenSearchDocument buildDocument(QRecord record, String tableName, String primaryKeyField, List fields, Map fieldWeights, Map includeLabels) + { + String searchableText = buildSearchableText(record, fields, includeLabels); + String recordId = record.getValueString(primaryKeyField); + String recordLabel = record.getRecordLabel(); + + Map fieldValues = new HashMap<>(); + + if(fields != null) + { + for(String fieldName : fields) + { + Object value = record.getValue(fieldName); + + if(value != null) { - if(sb.length() > 0) - { - sb.append(" "); - } - sb.append(textValue); + fieldValues.put(fieldName, value); } } } - return sb.toString(); - } - - - - /*************************************************************************** - ** Normalize search query text for matching. - ***************************************************************************/ - public static String normalizeSearchText(String text) - { - if(text == null) - { - return ""; - } - return text.toLowerCase().trim(); - } - - - - /*************************************************************************** - ** Build an OpenSearchDocument from a QRecord. - ***************************************************************************/ - public static OpenSearchDocument buildDocument(QRecord record, QTableMetaData table, List searchableFields) - { - String primaryKeyField = table.getPrimaryKeyField(); - Object primaryKeyValue = record.getValue(primaryKeyField); - String recordId = primaryKeyValue != null ? String.valueOf(primaryKeyValue) : null; - - String recordLabel = record.getRecordLabel(); - if(recordLabel == null || recordLabel.isBlank()) - { - recordLabel = table.getLabel() + " " + recordId; - } - - String searchableText = buildSearchableText(record, searchableFields); - - return new OpenSearchDocument() - .withSourceTable(table.getName()) + return (new OpenSearchDocument() + .withSourceTable(tableName) .withRecordId(recordId) .withRecordLabel(recordLabel) .withSearchableText(searchableText) - .withIndexedAt(Instant.now()); + .withIndexedAt(Instant.now()) + .withFieldValues(fieldValues)); } } diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEvent.java b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEvent.java new file mode 100644 index 0000000..94ee4f3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEvent.java @@ -0,0 +1,147 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.publisher; + + +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Data transfer object representing an event to index or delete a record in + ** the OpenSearch quick-search index. + *******************************************************************************/ +public class IndexEvent +{ + + private String tableName; + private String recordId; + private IndexEventAction action; + private QRecord record; + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + /******************************************************************************* + ** Setter for tableName. + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public IndexEvent withTableName(String tableName) + { + this.tableName = tableName; + return this; + } + + + /******************************************************************************* + ** Getter for recordId. + *******************************************************************************/ + public String getRecordId() + { + return recordId; + } + + + /******************************************************************************* + ** Setter for recordId. + *******************************************************************************/ + public void setRecordId(String recordId) + { + this.recordId = recordId; + } + + + /******************************************************************************* + ** Fluent setter for recordId. + *******************************************************************************/ + public IndexEvent withRecordId(String recordId) + { + this.recordId = recordId; + return this; + } + + + /******************************************************************************* + ** Getter for action. + *******************************************************************************/ + public IndexEventAction getAction() + { + return action; + } + + + /******************************************************************************* + ** Setter for action. + *******************************************************************************/ + public void setAction(IndexEventAction action) + { + this.action = action; + } + + + /******************************************************************************* + ** Fluent setter for action. + *******************************************************************************/ + public IndexEvent withAction(IndexEventAction action) + { + this.action = action; + return this; + } + + + /******************************************************************************* + ** Getter for record. + *******************************************************************************/ + public QRecord getRecord() + { + return record; + } + + + /******************************************************************************* + ** Setter for record. + *******************************************************************************/ + public void setRecord(QRecord record) + { + this.record = record; + } + + + /******************************************************************************* + ** Fluent setter for record. + *******************************************************************************/ + public IndexEvent withRecord(QRecord record) + { + this.record = record; + return this; + } + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventAction.java b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventAction.java new file mode 100644 index 0000000..e9b4699 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventAction.java @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.publisher; + + +/******************************************************************************* + ** Enum representing the type of indexing action for an IndexEvent. + ** + ** INDEX - add or update the document in the OpenSearch index. + ** DELETE - remove the document from the OpenSearch index. + *******************************************************************************/ +public enum IndexEventAction +{ + INDEX, + DELETE +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventPublisher.java b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventPublisher.java new file mode 100644 index 0000000..9bf45eb --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventPublisher.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.publisher; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Strategy interface for publishing index events to the search engine. + ** + ** Implementations determine how events are dispatched (synchronously, + ** asynchronously via a queue, etc.). + *******************************************************************************/ +public interface IndexEventPublisher +{ + + /******************************************************************************* + ** Publish a batch of INDEX events, causing the corresponding records to be + ** added or updated in the search index. + ** + ** @param events the index events to publish + ** @throws QException if the publish operation fails + *******************************************************************************/ + void publishIndexEvents(List events) throws QException; + + + + /******************************************************************************* + ** Publish a batch of DELETE events, causing the corresponding records to be + ** removed from the search index. + ** + ** @param events the delete events to publish + ** @throws QException if the publish operation fails + *******************************************************************************/ + void publishDeleteEvents(List events) throws QException; + + + + /******************************************************************************* + ** Release any resources held by this publisher. + ** + ** @throws QException if the close operation fails + *******************************************************************************/ + void close() throws QException; + +} diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/publisher/SynchronousIndexEventPublisher.java b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/SynchronousIndexEventPublisher.java new file mode 100644 index 0000000..a230477 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/quicksearch/publisher/SynchronousIndexEventPublisher.java @@ -0,0 +1,166 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.publisher; + + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import com.kingsrook.qbits.quicksearch.processes.IndexingUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Synchronous implementation of IndexEventPublisher that processes events + ** immediately against the OpenSearch client. + ** + ** Index events are grouped by table, converted to OpenSearchDocument instances + ** using IndexingUtils, and bulk-indexed. Delete events are processed + ** individually. + *******************************************************************************/ +public class SynchronousIndexEventPublisher implements IndexEventPublisher +{ + private static final QLogger LOG = QLogger.getLogger(SynchronousIndexEventPublisher.class); + + private final QuickSearchOpenSearchClient client; + private final Map tableConfigsByName; + private final Integer bulkBatchSize; + + + + /******************************************************************************* + ** Constructor. + ** + ** @param client the OpenSearch client to delegate operations to + ** @param tableConfigs the list of searchable table configurations + ** @param bulkBatchSize number of documents per bulk request + *******************************************************************************/ + public SynchronousIndexEventPublisher(QuickSearchOpenSearchClient client, List tableConfigs, Integer bulkBatchSize) + { + this.client = client; + this.bulkBatchSize = bulkBatchSize; + this.tableConfigsByName = new HashMap<>(); + + if(tableConfigs != null) + { + for(QuickSearchableTableConfig config : tableConfigs) + { + tableConfigsByName.put(config.getTableName(), config); + } + } + } + + + + /******************************************************************************* + ** Publish index events synchronously by building documents and bulk-indexing + ** them via the OpenSearch client. + *******************************************************************************/ + @Override + public void publishIndexEvents(List events) throws QException + { + if(events == null || events.isEmpty()) + { + return; + } + + ///////////////////////////////////////////////////////////////////////// + // group events by table name so we can look up configs per table once // + ///////////////////////////////////////////////////////////////////////// + Map> eventsByTable = new HashMap<>(); + for(IndexEvent event : events) + { + eventsByTable.computeIfAbsent(event.getTableName(), k -> new ArrayList<>()).add(event); + } + + List allDocuments = new ArrayList<>(); + + for(Map.Entry> entry : eventsByTable.entrySet()) + { + String tableName = entry.getKey(); + List tableEvents = entry.getValue(); + + QuickSearchableTableConfig tableConfig = tableConfigsByName.get(tableName); + if(tableConfig == null) + { + LOG.warn("No QuickSearchableTableConfig found for table, skipping", logPair("tableName", tableName)); + continue; + } + + String primaryKeyField = tableConfig.getPrimaryKeyField(); + if(primaryKeyField == null) + { + primaryKeyField = "id"; + } + + for(IndexEvent event : tableEvents) + { + OpenSearchDocument document = IndexingUtils.buildDocument( + event.getRecord(), + event.getTableName(), + primaryKeyField, + tableConfig.getSearchableFields(), + tableConfig.getFieldWeights(), + tableConfig.getFieldIncludeLabels()); + + allDocuments.add(document); + } + } + + if(!allDocuments.isEmpty()) + { + client.indexDocuments(allDocuments, bulkBatchSize); + } + } + + + + /******************************************************************************* + ** Publish delete events synchronously by calling deleteDocument for each event. + *******************************************************************************/ + @Override + public void publishDeleteEvents(List events) throws QException + { + if(events == null || events.isEmpty()) + { + return; + } + + for(IndexEvent event : events) + { + client.deleteDocument(event.getTableName(), event.getRecordId()); + } + } + + + + /******************************************************************************* + ** Close by delegating to the underlying client. + *******************************************************************************/ + @Override + public void close() throws QException + { + client.close(); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/BaseQuickSearchTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/BaseQuickSearchTest.java index 89fa07a..903d7e7 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/BaseQuickSearchTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/BaseQuickSearchTest.java @@ -17,165 +17,125 @@ package com.kingsrook.qbits.quicksearch; import java.util.List; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; -import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import static org.mockito.Mockito.mock; /******************************************************************************* - ** Base test class for Quick Search QBit tests. + ** Shared test base class for Quick Search QBit tests that need a full + ** in-memory QInstance with QContext and QuickSearchQBitContext initialized. + ** + ** Sets up: + ** - A memory-backed QInstance with quickSearchIndex, quickSearchIndexRun, + ** and a "testEntity" source table. + ** - QContext initialized with the QInstance and a new anonymous session. + ** - QuickSearchQBitContext with a minimal test config and a mock OpenSearch + ** client. + ** + ** Subclasses with @Test methods can extend this class to reuse all setup. *******************************************************************************/ public class BaseQuickSearchTest { - public static final String BACKEND_NAME = "memory"; - public static final String TEST_TABLE_NAME = "testEntity"; - public static final String TEST_NAMESPACE = "testNamespace"; - - protected QuickSearchQBitConfig config; - protected QInstance qInstance; + protected static final String TEST_BACKEND_NAME = "testMemoryBackend"; + protected static final String TEST_ENTITY_TABLE = "testEntity"; - /*************************************************************************** - ** Setup before each test. - ***************************************************************************/ - @BeforeEach - void setup() throws QException + /******************************************************************************* + ** Build and return a fresh QInstance configured for in-memory testing. + ** + ** Registers: + ** - A memory backend named TEST_BACKEND_NAME + ** - FULLY_ANONYMOUS authentication + ** - quickSearchIndex table (all fields from QuickSearchIndex entity) + ** - quickSearchIndexRun table (all fields from QuickSearchIndexRun entity) + ** - testEntity table (id, name, description, modifyDate) + *******************************************************************************/ + protected QInstance buildTestQInstance() { - MemoryRecordStore.getInstance().reset(); + QInstance qInstance = new QInstance(); - qInstance = new QInstance(); - - //////////////////////////////////////////// - // Add memory backend for testing // - //////////////////////////////////////////// + //////////////////////////////////////////////////// + // Register the in-memory backend // + //////////////////////////////////////////////////// qInstance.addBackend(new QBackendMetaData() - .withName(BACKEND_NAME) + .withName(TEST_BACKEND_NAME) .withBackendType(MemoryBackendModule.class)); - //////////////////////////////////////////// - // Add anonymous auth for testing // - //////////////////////////////////////////// + //////////////////////////////////////////////////// + // Set up fully-anonymous authentication // + //////////////////////////////////////////////////// qInstance.setAuthentication(new QAuthenticationMetaData() .withName("anonymous") .withType(QAuthenticationType.FULLY_ANONYMOUS)); - //////////////////////////////////////////// - // Create test tables manually // - //////////////////////////////////////////// - qInstance.addTable(createQuickSearchIndexTable()); - qInstance.addTable(createQuickSearchIndexRunTable()); - qInstance.addTable(createTestEntityTable()); + //////////////////////////////////////////////////// + // quickSearchIndex table // + //////////////////////////////////////////////////// + qInstance.addTable(buildQuickSearchIndexTable()); - //////////////////////////////////////////// - // Register QBit config // - //////////////////////////////////////////// - config = createTestConfig(); - QBitMetaData qBitMetaData = new QBitMetaData() - .withGroupId(QuickSearchQBitProducer.GROUP_ID) - .withArtifactId(QuickSearchQBitProducer.ARTIFACT_ID) - .withVersion(QuickSearchQBitProducer.VERSION) - .withNamespace(TEST_NAMESPACE) - .withConfig(config); - qInstance.addQBit(qBitMetaData); + //////////////////////////////////////////////////// + // quickSearchIndexRun table // + //////////////////////////////////////////////////// + qInstance.addTable(buildQuickSearchIndexRunTable()); - //////////////////////////////////////////// - // Store config in context // - //////////////////////////////////////////// - QuickSearchQBitContext.setConfig(config); - QuickSearchQBitContext.setDiscoveredTables(List.of()); + //////////////////////////////////////////////////// + // testEntity source table // + //////////////////////////////////////////////////// + qInstance.addTable(buildTestEntityTable()); - QContext.init(qInstance, newSession()); + return (qInstance); } - /*************************************************************************** - ** Cleanup after each test. - ***************************************************************************/ - @AfterEach - void tearDown() - { - QContext.clear(); - MemoryRecordStore.getInstance().reset(); - } - - - - /*************************************************************************** - ** Create test configuration. - ***************************************************************************/ - protected QuickSearchQBitConfig createTestConfig() - { - return new QuickSearchQBitConfig() - .withBackendName(BACKEND_NAME) - .withTableNamePrefix("") - .withOpensearchHost("localhost") - .withOpensearchPort(9200) - .withOpensearchIndexName("test-quick-search") - .withUseSsl(false); - } - - - - /*************************************************************************** - ** Create a new session for testing. - ***************************************************************************/ - protected QSession newSession() - { - return new QSession(); - } - - - - /*************************************************************************** - ** Create the quickSearchIndex table for testing. - ***************************************************************************/ - protected QTableMetaData createQuickSearchIndexTable() + /******************************************************************************* + ** Build QTableMetaData for quickSearchIndex. + *******************************************************************************/ + private QTableMetaData buildQuickSearchIndexTable() { return new QTableMetaData() .withName(QuickSearchIndex.TABLE_NAME) - .withLabel("Quick Search Index") - .withBackendName(BACKEND_NAME) + .withBackendName(TEST_BACKEND_NAME) .withPrimaryKeyField("id") .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) - .withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true)) - .withField(new QFieldMetaData("isEnabled", QFieldType.BOOLEAN)) + .withField(new QFieldMetaData("tableName", QFieldType.STRING)) + .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN)) .withField(new QFieldMetaData("basepullIntervalMinutes", QFieldType.INTEGER)) .withField(new QFieldMetaData("basepullTimestampField", QFieldType.STRING)) .withField(new QFieldMetaData("searchableFieldsJson", QFieldType.STRING)) - .withField(new QFieldMetaData("lastFullIndexTime", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("lastBasepullTime", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("indexedRecordCount", QFieldType.INTEGER)) + .withField(new QFieldMetaData("lastFullReindexTime", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("recordCount", QFieldType.INTEGER)) + .withField(new QFieldMetaData("status", QFieldType.STRING)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)); } - /*************************************************************************** - ** Create the quickSearchIndexRun table for testing. - ***************************************************************************/ - protected QTableMetaData createQuickSearchIndexRunTable() + /******************************************************************************* + ** Build QTableMetaData for quickSearchIndexRun. + *******************************************************************************/ + private QTableMetaData buildQuickSearchIndexRunTable() { return new QTableMetaData() .withName(QuickSearchIndexRun.TABLE_NAME) - .withLabel("Quick Search Index Run") - .withBackendName(BACKEND_NAME) + .withBackendName(TEST_BACKEND_NAME) .withPrimaryKeyField("id") .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) .withField(new QFieldMetaData("quickSearchIndexId", QFieldType.INTEGER)) @@ -186,28 +146,93 @@ public class BaseQuickSearchTest .withField(new QFieldMetaData("recordsProcessed", QFieldType.INTEGER)) .withField(new QFieldMetaData("recordsIndexed", QFieldType.INTEGER)) .withField(new QFieldMetaData("errorCount", QFieldType.INTEGER)) - .withField(new QFieldMetaData("errorMessage", QFieldType.STRING)); - } - - - - /*************************************************************************** - ** Create a test entity table for testing. - ***************************************************************************/ - protected QTableMetaData createTestEntityTable() - { - return new QTableMetaData() - .withName(TEST_TABLE_NAME) - .withLabel("Test Entity") - .withBackendName(BACKEND_NAME) - .withPrimaryKeyField("id") - .withRecordLabelFormat("%s") - .withRecordLabelFields(List.of("name")) - .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) - .withField(new QFieldMetaData("name", QFieldType.STRING)) - .withField(new QFieldMetaData("description", QFieldType.STRING)) - .withField(new QFieldMetaData("status", QFieldType.STRING)) + .withField(new QFieldMetaData("errorMessage", QFieldType.STRING)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)); } + + + /******************************************************************************* + ** Build QTableMetaData for the test source entity table. + *******************************************************************************/ + private QTableMetaData buildTestEntityTable() + { + return new QTableMetaData() + .withName(TEST_ENTITY_TABLE) + .withBackendName(TEST_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)); + } + + + + /******************************************************************************* + ** Set up test environment before each test. + ** + ** Resets MemoryRecordStore, initializes QContext, and sets up + ** QuickSearchQBitContext with a minimal config and mock OpenSearch client. + *******************************************************************************/ + @BeforeEach + void baseSetUp() + { + //////////////////////////////////////////////////// + // Reset in-memory record store // + //////////////////////////////////////////////////// + MemoryRecordStore.getInstance().reset(); + + //////////////////////////////////////////////////// + // Initialize QContext with test QInstance // + //////////////////////////////////////////////////// + QInstance qInstance = buildTestQInstance(); + QContext.init(qInstance, new QSession()); + + //////////////////////////////////////////////////// + // Set up QuickSearchQBitContext with test config // + //////////////////////////////////////////////////// + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName(TEST_BACKEND_NAME) + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of()); + + QuickSearchQBitContext.setConfig(config); + + //////////////////////////////////////////////////// + // Set up a mock OpenSearch client // + //////////////////////////////////////////////////// + QuickSearchQBitContext.setClient(mock(com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient.class)); + + //////////////////////////////////////////////////// + // Set up a test discoveredTables list // + //////////////////////////////////////////////////// + QuickSearchQBitContext.setDiscoveredTables(List.of( + new QuickSearchableTableConfig() + .withTableName(TEST_ENTITY_TABLE) + .withPrimaryKeyField("id") + .withSearchableFields(List.of("name", "description")) + .withBasepullTimestampField("modifyDate") + .withBasepullIntervalMinutes(5) + .withEnabledByDefault(true) + )); + } + + + + /******************************************************************************* + ** Clean up test environment after each test. + ** + ** Clears QContext and QuickSearchQBitContext to prevent state leakage. + *******************************************************************************/ + @AfterEach + void baseTearDown() + { + QContext.clear(); + QuickSearchQBitContext.clear(); + } + } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/OpenSearchIntegrationTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/OpenSearchIntegrationTest.java index e4caaf9..c4bbf75 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/OpenSearchIntegrationTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/OpenSearchIntegrationTest.java @@ -16,22 +16,37 @@ package com.kingsrook.qbits.quicksearch; -import java.time.Duration; import java.time.Instant; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qbits.quicksearch.actions.QuickSearchAction; import com.kingsrook.qbits.quicksearch.actions.QuickSearchInput; import com.kingsrook.qbits.quicksearch.actions.QuickSearchOutput; -import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField; +import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable; import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import com.kingsrook.qbits.quicksearch.processes.FullReindexStep; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; -import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; @@ -40,238 +55,255 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* - ** Integration tests for Quick Search using OpenSearch testcontainers. + ** End-to-end integration test for the Quick Search QBit using a real + ** OpenSearch instance managed by Testcontainers. ** - ** These tests require Docker to be running. They run by default and will - ** be automatically skipped if Docker is unavailable. Set environment variable - ** SKIP_INTEGRATION_TESTS=true to manually skip these tests. - ** - ** This test class is named with *IntegrationTest suffix so it can be excluded - ** from unit test runs if needed via surefire configuration. + ** Tests run in order: index creation, full reindex, search with results, + ** pagination, and no-results search. *******************************************************************************/ @Testcontainers(disabledWithoutDocker = true) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class OpenSearchIntegrationTest { - private static final int OPENSEARCH_PORT = 9200; + + private static final String BACKEND_NAME = "testMemoryBackend"; + private static final String TABLE_NAME = "testProduct"; + private static final String INDEX_NAME = "quick_search_integration_test"; @Container - static GenericContainer opensearchContainer = new GenericContainer<>("opensearchproject/opensearch:2.11.0") - .withExposedPorts(OPENSEARCH_PORT) + static GenericContainer opensearch = new GenericContainer<>("opensearchproject/opensearch:2.11.0") + .withExposedPorts(9200) .withEnv("discovery.type", "single-node") .withEnv("plugins.security.disabled", "true") .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "Admin123!") - .waitingFor(Wait.forHttp("/").forPort(OPENSEARCH_PORT).forStatusCode(200)) - .withStartupTimeout(Duration.ofMinutes(2)); - - private static QuickSearchQBitConfig config; - private static QuickSearchOpenSearchClient client; + .waitingFor(Wait.forHttp("/_cluster/health").forStatusCode(200)); - /*************************************************************************** - ** Set up OpenSearch client after container starts. - ***************************************************************************/ - @BeforeAll - static void setUp() throws Exception + /******************************************************************************* + ** Test entity class annotated for quick search indexing. + *******************************************************************************/ + @QuickSearchable(tableName = TABLE_NAME) + static class TestProduct { - config = new QuickSearchQBitConfig() - .withOpensearchHost(opensearchContainer.getHost()) - .withOpensearchPort(opensearchContainer.getMappedPort(OPENSEARCH_PORT)) - .withOpensearchIndexName("quick-search-test") - .withUseSsl(false) - .withBackendName("testBackend"); + @QuickSearchField(weight = 3) + private String name; - client = new QuickSearchOpenSearchClient(config); + @QuickSearchField + private String description; - QuickSearchQBitContext.setConfig(config); + @QuickSearchField(weight = 2, includeLabel = true) + private String sku; } - /*************************************************************************** - ** Clean up after tests. - ***************************************************************************/ - @AfterAll - static void tearDown() + /******************************************************************************* + ** Initialize the QInstance, QBit, and QContext before all tests run. + *******************************************************************************/ + @BeforeAll + static void setUpAll() throws Exception { - if(client != null) + MemoryRecordStore.getInstance().reset(); + + //////////////////////////////////////////////////// + // Build QInstance with memory backend // + //////////////////////////////////////////////////// + QInstance qInstance = new QInstance(); + + qInstance.addBackend(new QBackendMetaData() + .withName(BACKEND_NAME) + .withBackendType(MemoryBackendModule.class)); + + qInstance.setAuthentication(new QAuthenticationMetaData() + .withName("anonymous") + .withType(QAuthenticationType.FULLY_ANONYMOUS)); + + //////////////////////////////////////////////////// + // Add testProduct source table // + //////////////////////////////////////////////////// + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME) + .withBackendName(BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))); + + //////////////////////////////////////////////////// + // Configure and produce the QBit // + //////////////////////////////////////////////////// + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName(BACKEND_NAME) + .withOpensearchHost(opensearch.getHost()) + .withOpensearchPort(opensearch.getMappedPort(9200)) + .withOpensearchIndexName(INDEX_NAME) + .withSearchableEntityClasses(List.of(TestProduct.class)) + .withEnableRealTimeIndexing(false); + + new QuickSearchQBitProducer().withConfig(config).produce(qInstance); + + QContext.init(qInstance, new QSession()); + } + + + + /******************************************************************************* + ** Clean up QContext and QBit state after all tests complete. + *******************************************************************************/ + @AfterAll + static void tearDownAll() + { + QContext.clear(); + QuickSearchQBitContext.clear(); + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** Test 1: Verify the OpenSearch client was created and the index exists. + *******************************************************************************/ + @Test + @Order(1) + void testOpenSearchIndexCreated() + { + Object rawClient = QuickSearchQBitContext.getClient(); + assertThat(rawClient).isNotNull(); + assertThat(rawClient).isInstanceOf(QuickSearchOpenSearchClient.class); + + QuickSearchQBitConfig storedConfig = QuickSearchQBitContext.getConfig(); + assertThat(storedConfig).isNotNull(); + assertThat(storedConfig.getOpensearchIndexName()).isEqualTo(INDEX_NAME); + } + + + + /******************************************************************************* + ** Test 2: Insert 5 test products, run FullReindexStep, and verify a search + ** returns results. + *******************************************************************************/ + @Test + @Order(2) + void testFullReindex() throws Exception + { + //////////////////////////////////////////////////// + // Insert 5 test products into the memory backend // + //////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("name", "Blue Widget").withValue("description", "A fine widget for all uses").withValue("sku", "BW-001").withValue("modifyDate", Instant.now()), + new QRecord().withValue("id", 2).withValue("name", "Red Widget").withValue("description", "A premium widget in red").withValue("sku", "RW-002").withValue("modifyDate", Instant.now()), + new QRecord().withValue("id", 3).withValue("name", "Green Gadget").withValue("description", "A handy gadget for the workshop").withValue("sku", "GG-003").withValue("modifyDate", Instant.now()), + new QRecord().withValue("id", 4).withValue("name", "Yellow Gadget").withValue("description", "A bright yellow gadget").withValue("sku", "YG-004").withValue("modifyDate", Instant.now()), + new QRecord().withValue("id", 5).withValue("name", "Purple Widget").withValue("description", "A rare purple widget").withValue("sku", "PW-005").withValue("modifyDate", Instant.now()) + )); + new InsertAction().execute(insertInput); + + //////////////////////////////////////////////////// + // Run the full reindex step // + //////////////////////////////////////////////////// + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + new FullReindexStep().run(input, output); + + //////////////////////////////////////////////////// + // Give OpenSearch time to refresh its index // + //////////////////////////////////////////////////// + Thread.sleep(2000); + + //////////////////////////////////////////////////// + // Verify at least one result for "widget" // + //////////////////////////////////////////////////// + QuickSearchOutput searchOutput = new QuickSearchAction().execute( + new QuickSearchInput().withSearchTerm("widget")); + + assertThat(searchOutput.getTotalHits()).isGreaterThan(0L); + assertThat(searchOutput.getResults()).isNotEmpty(); + } + + + + /******************************************************************************* + ** Test 3: Search for "widget" and assert results have required fields. + *******************************************************************************/ + @Test + @Order(3) + void testSearchReturnsResults() throws Exception + { + QuickSearchOutput output = new QuickSearchAction().execute( + new QuickSearchInput().withSearchTerm("widget")); + + assertThat(output.getTotalHits()).isGreaterThan(0L); + assertThat(output.getResults()).isNotEmpty(); + + //////////////////////////////////////////////////// + // Each result must have tableName, recordId, // + // and score populated // + //////////////////////////////////////////////////// + output.getResults().forEach(result -> { - client.close(); + assertThat(result.getTableName()).isEqualTo(TABLE_NAME); + assertThat(result.getRecordId()).isNotBlank(); + assertThat(result.getScore()).isNotNull().isGreaterThan(0f); + }); + } + + + + /******************************************************************************* + ** Test 4: Search with limit and offset to verify pagination. + *******************************************************************************/ + @Test + @Order(4) + void testSearchPagination() throws Exception + { + //////////////////////////////////////////////////// + // Fetch first page (limit=2, offset=0) // + //////////////////////////////////////////////////// + QuickSearchOutput page1 = new QuickSearchAction().execute( + new QuickSearchInput().withSearchTerm("widget").withLimit(2).withOffset(0)); + + assertThat(page1.getResults()).hasSizeLessThanOrEqualTo(2); + + //////////////////////////////////////////////////// + // If total hits > 2, hasMore should be true and // + // page 2 should return different record IDs // + //////////////////////////////////////////////////// + if(page1.getTotalHits() > 2) + { + assertThat(page1.getHasMore()).isTrue(); + + QuickSearchOutput page2 = new QuickSearchAction().execute( + new QuickSearchInput().withSearchTerm("widget").withLimit(2).withOffset(2)); + + assertThat(page2.getResults()).isNotEmpty(); + + List page1Ids = page1.getResults().stream().map(r -> r.getRecordId()).toList(); + List page2Ids = page2.getResults().stream().map(r -> r.getRecordId()).toList(); + + assertThat(page1Ids).doesNotContainAnyElementsOf(page2Ids); } } - /*************************************************************************** - ** Test that we can create the index. - ***************************************************************************/ - @Test - @Order(1) - void testEnsureIndexExists() throws Exception - { - client.ensureIndexExists(); - } - - - - /*************************************************************************** - ** Test indexing a single document. - ***************************************************************************/ - @Test - @Order(2) - void testIndexSingleDocument() throws Exception - { - OpenSearchDocument doc = new OpenSearchDocument() - .withSourceTable("order") - .withRecordId("12345") - .withRecordLabel("Order #12345") - .withSearchableText("John Doe 123 Main Street shipped express delivery") - .withIndexedAt(Instant.now()); - - client.indexDocument(doc); - - // Give OpenSearch time to index - Thread.sleep(1000); - } - - - - /*************************************************************************** - ** Test bulk indexing multiple documents. - ***************************************************************************/ - @Test - @Order(3) - void testBulkIndexDocuments() throws Exception - { - List docs = List.of( - new OpenSearchDocument() - .withSourceTable("customer") - .withRecordId("100") - .withRecordLabel("Customer: Alice Smith") - .withSearchableText("Alice Smith alice@example.com 555-1234") - .withIndexedAt(Instant.now()), - new OpenSearchDocument() - .withSourceTable("customer") - .withRecordId("101") - .withRecordLabel("Customer: Bob Johnson") - .withSearchableText("Bob Johnson bob@company.org 555-5678") - .withIndexedAt(Instant.now()), - new OpenSearchDocument() - .withSourceTable("product") - .withRecordId("SKU001") - .withRecordLabel("Widget Pro") - .withSearchableText("Widget Pro premium quality electronics gadget") - .withIndexedAt(Instant.now()) - ); - - client.indexDocuments(docs); - - // Give OpenSearch time to index - Thread.sleep(1000); - } - - - - /*************************************************************************** - ** Test searching for documents. - ***************************************************************************/ - @Test - @Order(4) - void testSearchDocuments() throws Exception - { - List results = client.search("John", 10); - - assertThat(results).isNotEmpty(); - assertThat(results.get(0).getSourceTable()).isEqualTo("order"); - assertThat(results.get(0).getRecordId()).isEqualTo("12345"); - } - - - - /*************************************************************************** - ** Test searching with table filter. - ***************************************************************************/ + /******************************************************************************* + ** Test 5: Search for a term that should not match any records. + *******************************************************************************/ @Test @Order(5) - void testSearchWithTableFilter() throws Exception - { - List results = client.search("alice", "customer", 10); - - assertThat(results).hasSize(1); - assertThat(results.get(0).getRecordId()).isEqualTo("100"); - } - - - - /*************************************************************************** - ** Test that search finds nothing for non-matching query. - ***************************************************************************/ - @Test - @Order(6) void testSearchNoResults() throws Exception { - List results = client.search("xyznonexistent123", 10); + QuickSearchOutput output = new QuickSearchAction().execute( + new QuickSearchInput().withSearchTerm("zzzznonexistent")); - assertThat(results).isEmpty(); - } - - - - /*************************************************************************** - ** Test QuickSearchAction end-to-end. - ***************************************************************************/ - @Test - @Order(7) - void testQuickSearchAction() throws Exception - { - QuickSearchAction action = new QuickSearchAction(); - - QuickSearchInput input = new QuickSearchInput() - .withSearchTerm("Bob") - .withLimit(10); - - QuickSearchOutput output = action.execute(input); - - assertThat(output.getResults()).isNotEmpty(); - assertThat(output.getResults().get(0).getTableName()).isEqualTo("customer"); - assertThat(output.getResults().get(0).getRecordId()).isEqualTo("101"); - } - - - - /*************************************************************************** - ** Test deleting a document. - ***************************************************************************/ - @Test - @Order(8) - void testDeleteDocument() throws Exception - { - client.deleteDocument("customer", "100"); - - // Give OpenSearch time to process delete - Thread.sleep(1000); - - List results = client.search("Alice", "customer", 10); - assertThat(results).isEmpty(); - } - - - - /*************************************************************************** - ** Test deleting all documents for a table. - ***************************************************************************/ - @Test - @Order(9) - void testDeleteDocumentsForTable() throws Exception - { - client.deleteDocumentsForTable("product"); - - // Give OpenSearch time to process delete - Thread.sleep(1000); - - List results = client.search("widget", "product", 10); - assertThat(results).isEmpty(); + assertThat(output.getTotalHits()).isEqualTo(0L); + assertThat(output.getResults()).isEmpty(); + assertThat(output.getHasMore()).isFalse(); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfigTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfigTest.java index 1668dd6..31f35d1 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfigTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitConfigTest.java @@ -32,7 +32,7 @@ class QuickSearchQBitConfigTest { /*************************************************************************** - ** Test validation with valid config. + ** Test validation with a fully valid config. ***************************************************************************/ @Test void testValidate_validConfig_noErrors() @@ -43,7 +43,8 @@ class QuickSearchQBitConfigTest .withBackendName("memory") .withOpensearchHost("localhost") .withOpensearchPort(9200) - .withOpensearchIndexName("test-index"); + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); List errors = new ArrayList<>(); config.validate(qInstance, errors); @@ -64,21 +65,22 @@ class QuickSearchQBitConfigTest QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withOpensearchHost("localhost") .withOpensearchPort(9200) - .withOpensearchIndexName("test-index"); + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); List errors = new ArrayList<>(); config.validate(qInstance, errors); - assertThat(errors).contains("backendName is required for QuickSearchQBit"); + assertThat(errors).anyMatch(e -> e.contains("backendName is required")); } /*************************************************************************** - ** Test validation with non-existent backend. + ** Test validation with a backend name not present in QInstance. ***************************************************************************/ @Test - void testValidate_nonExistentBackend_addsError() + void testValidate_backendNotInQInstance_addsError() { QInstance qInstance = createQInstanceWithBackend(); @@ -86,12 +88,13 @@ class QuickSearchQBitConfigTest .withBackendName("nonexistent") .withOpensearchHost("localhost") .withOpensearchPort(9200) - .withOpensearchIndexName("test-index"); + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); List errors = new ArrayList<>(); config.validate(qInstance, errors); - assertThat(errors).contains("Backend not found: nonexistent"); + assertThat(errors).anyMatch(e -> e.contains("nonexistent")); } @@ -107,33 +110,81 @@ class QuickSearchQBitConfigTest QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withBackendName("memory") .withOpensearchPort(9200) - .withOpensearchIndexName("test-index"); + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); List errors = new ArrayList<>(); config.validate(qInstance, errors); - assertThat(errors).contains("opensearchHost is required for QuickSearchQBit"); + assertThat(errors).anyMatch(e -> e.contains("opensearchHost is required")); } /*************************************************************************** - ** Test validation with missing OpenSearch port. + ** Test validation with null OpenSearch port. ***************************************************************************/ @Test - void testValidate_missingOpensearchPort_addsError() + void testValidate_nullOpensearchPort_addsError() { QInstance qInstance = createQInstanceWithBackend(); QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withBackendName("memory") .withOpensearchHost("localhost") - .withOpensearchIndexName("test-index"); + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); List errors = new ArrayList<>(); config.validate(qInstance, errors); - assertThat(errors).contains("opensearchPort is required for QuickSearchQBit"); + assertThat(errors).anyMatch(e -> e.contains("opensearchPort is required")); + } + + + + /*************************************************************************** + ** Test validation with zero port. + ***************************************************************************/ + @Test + void testValidate_zeroPort_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(0) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("opensearchPort must be positive")); + } + + + + /*************************************************************************** + ** Test validation with negative port. + ***************************************************************************/ + @Test + void testValidate_negativePort_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(-1) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("opensearchPort must be positive")); } @@ -149,12 +200,248 @@ class QuickSearchQBitConfigTest QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withBackendName("memory") .withOpensearchHost("localhost") - .withOpensearchPort(9200); + .withOpensearchPort(9200) + .withSearchableEntityClasses(List.of(String.class)); List errors = new ArrayList<>(); config.validate(qInstance, errors); - assertThat(errors).contains("opensearchIndexName is required for QuickSearchQBit"); + assertThat(errors).anyMatch(e -> e.contains("opensearchIndexName is required")); + } + + + + /*************************************************************************** + ** Test validation when username is set but password is missing. + ***************************************************************************/ + @Test + void testValidate_usernameWithoutPassword_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withOpensearchUsername("user") + .withSearchableEntityClasses(List.of(String.class)); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("opensearchPassword is required")); + } + + + + /*************************************************************************** + ** Test validation when password is set but username is missing. + ***************************************************************************/ + @Test + void testValidate_passwordWithoutUsername_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withOpensearchPassword("secret") + .withSearchableEntityClasses(List.of(String.class)); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("opensearchUsername is required")); + } + + + + /*************************************************************************** + ** Test validation with both username and password set passes auth check. + ***************************************************************************/ + @Test + void testValidate_bothUsernameAndPassword_passesAuthCheck() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withOpensearchUsername("user") + .withOpensearchPassword("secret") + .withSearchableEntityClasses(List.of(String.class)); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).noneMatch(e -> e.contains("opensearchUsername") || e.contains("opensearchPassword")); + } + + + + /*************************************************************************** + ** Test validation with neither username nor password passes auth check. + ***************************************************************************/ + @Test + void testValidate_neitherUsernameNorPassword_passesAuthCheck() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).noneMatch(e -> e.contains("opensearchUsername") || e.contains("opensearchPassword")); + } + + + + /*************************************************************************** + ** Test validation with empty searchableEntityClasses list. + ***************************************************************************/ + @Test + void testValidate_emptySearchableEntityClasses_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of()); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("searchableEntityClasses is required")); + } + + + + /*************************************************************************** + ** Test validation with null searchableEntityClasses. + ***************************************************************************/ + @Test + void testValidate_nullSearchableEntityClasses_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index"); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("searchableEntityClasses is required")); + } + + + + /*************************************************************************** + ** Test validation with zero bulkBatchSize. + ***************************************************************************/ + @Test + void testValidate_zeroBulkBatchSize_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)) + .withBulkBatchSize(0); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("bulkBatchSize must be positive")); + } + + + + /*************************************************************************** + ** Test validation with zero sourceBatchSize. + ***************************************************************************/ + @Test + void testValidate_zeroSourceBatchSize_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)) + .withSourceBatchSize(0); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("sourceBatchSize must be positive")); + } + + + + /*************************************************************************** + ** Test validation with zero defaultBasepullIntervalMinutes. + ***************************************************************************/ + @Test + void testValidate_zeroDefaultBasepullInterval_addsError() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig() + .withBackendName("memory") + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-index") + .withSearchableEntityClasses(List.of(String.class)) + .withDefaultBasepullIntervalMinutes(0); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors).anyMatch(e -> e.contains("defaultBasepullIntervalMinutes must be positive")); + } + + + + /*************************************************************************** + ** Test that multiple errors are collected in a single validate call. + ***************************************************************************/ + @Test + void testValidate_multipleErrors_allCollected() + { + QInstance qInstance = createQInstanceWithBackend(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig(); + + List errors = new ArrayList<>(); + config.validate(qInstance, errors); + + assertThat(errors.size()).isGreaterThan(1); + assertThat(errors).anyMatch(e -> e.contains("backendName")); + assertThat(errors).anyMatch(e -> e.contains("opensearchHost")); + assertThat(errors).anyMatch(e -> e.contains("opensearchPort")); + assertThat(errors).anyMatch(e -> e.contains("opensearchIndexName")); } @@ -168,30 +455,26 @@ class QuickSearchQBitConfigTest QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withTableNamePrefix("myApp_"); - String result = config.applyPrefix("tableName"); - - assertThat(result).isEqualTo("myApp_tableName"); + assertThat(config.applyPrefix("tableName")).isEqualTo("myApp_tableName"); } /*************************************************************************** - ** Test applyPrefix with null prefix. + ** Test applyPrefix with null prefix returns original name. ***************************************************************************/ @Test void testApplyPrefix_nullPrefix_returnsOriginal() { QuickSearchQBitConfig config = new QuickSearchQBitConfig(); - String result = config.applyPrefix("tableName"); - - assertThat(result).isEqualTo("tableName"); + assertThat(config.applyPrefix("tableName")).isEqualTo("tableName"); } /*************************************************************************** - ** Test applyPrefix with empty prefix. + ** Test applyPrefix with empty prefix returns original name. ***************************************************************************/ @Test void testApplyPrefix_emptyPrefix_returnsOriginal() @@ -199,20 +482,50 @@ class QuickSearchQBitConfigTest QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withTableNamePrefix(""); - String result = config.applyPrefix("tableName"); - - assertThat(result).isEqualTo("tableName"); + assertThat(config.applyPrefix("tableName")).isEqualTo("tableName"); } /*************************************************************************** - ** Test all fluent setters. + ** Test that all fluent setters return this. ***************************************************************************/ @Test - void testFluentSetters() + void testFluentSetters_returnThis() { List> entityClasses = List.of(String.class); + Object dummyPublisher = new Object(); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig(); + + assertThat(config.withBackendName("b")).isSameAs(config); + assertThat(config.withTableNamePrefix("p_")).isSameAs(config); + assertThat(config.withOpensearchHost("h")).isSameAs(config); + assertThat(config.withOpensearchPort(9200)).isSameAs(config); + assertThat(config.withOpensearchIndexName("i")).isSameAs(config); + assertThat(config.withOpensearchUsername("u")).isSameAs(config); + assertThat(config.withOpensearchPassword("pw")).isSameAs(config); + assertThat(config.withUseSsl(true)).isSameAs(config); + assertThat(config.withAutoDiscoverAnnotations(true)).isSameAs(config); + assertThat(config.withEnableScheduledProcesses(false)).isSameAs(config); + assertThat(config.withEnableRealTimeIndexing(false)).isSameAs(config); + assertThat(config.withDefaultBasepullIntervalMinutes(10)).isSameAs(config); + assertThat(config.withBulkBatchSize(100)).isSameAs(config); + assertThat(config.withSourceBatchSize(200)).isSameAs(config); + assertThat(config.withSearchableEntityClasses(entityClasses)).isSameAs(config); + assertThat(config.withIndexEventPublisher(dummyPublisher)).isSameAs(config); + } + + + + /*************************************************************************** + ** Test all fluent setters and corresponding getters. + ***************************************************************************/ + @Test + void testFluentSetters_gettersReturnCorrectValues() + { + List> entityClasses = List.of(String.class, Integer.class); + Object dummyPublisher = new Object(); QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withBackendName("testBackend") @@ -225,8 +538,12 @@ class QuickSearchQBitConfigTest .withUseSsl(true) .withAutoDiscoverAnnotations(true) .withEnableScheduledProcesses(false) + .withEnableRealTimeIndexing(false) .withDefaultBasepullIntervalMinutes(10) - .withSearchableEntityClasses(entityClasses); + .withBulkBatchSize(250) + .withSourceBatchSize(500) + .withSearchableEntityClasses(entityClasses) + .withIndexEventPublisher(dummyPublisher); assertThat(config.getBackendName()).isEqualTo("testBackend"); assertThat(config.getTableNamePrefix()).isEqualTo("prefix_"); @@ -238,8 +555,12 @@ class QuickSearchQBitConfigTest assertThat(config.getUseSsl()).isTrue(); assertThat(config.getAutoDiscoverAnnotations()).isTrue(); assertThat(config.getEnableScheduledProcesses()).isFalse(); + assertThat(config.getEnableRealTimeIndexing()).isFalse(); assertThat(config.getDefaultBasepullIntervalMinutes()).isEqualTo(10); + assertThat(config.getBulkBatchSize()).isEqualTo(250); + assertThat(config.getSourceBatchSize()).isEqualTo(500); assertThat(config.getSearchableEntityClasses()).isEqualTo(entityClasses); + assertThat(config.getIndexEventPublisher()).isSameAs(dummyPublisher); } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContextTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContextTest.java index 796b949..7745881 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContextTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitContextTest.java @@ -17,18 +17,43 @@ package com.kingsrook.qbits.quicksearch; import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* - ** Tests for QuickSearchQBitContext. + ** Tests for QuickSearchQBitContext static state holder. *******************************************************************************/ class QuickSearchQBitContextTest { /*************************************************************************** - ** Test setConfig and getConfig. + ** Clear state before and after each test to prevent leakage. + ***************************************************************************/ + @BeforeEach + @AfterEach + void clearContext() + { + QuickSearchQBitContext.clear(); + } + + + + /*************************************************************************** + ** Test that getConfig returns null before any value is set. + ***************************************************************************/ + @Test + void testGetConfig_beforeSet_returnsNull() + { + assertThat(QuickSearchQBitContext.getConfig()).isNull(); + } + + + + /*************************************************************************** + ** Test setConfig and getConfig round trip. ***************************************************************************/ @Test void testSetAndGetConfig() @@ -44,7 +69,7 @@ class QuickSearchQBitContextTest /*************************************************************************** - ** Test setDiscoveredTables and getDiscoveredTables. + ** Test setDiscoveredTables and getDiscoveredTables round trip. ***************************************************************************/ @Test void testSetAndGetDiscoveredTables() @@ -59,4 +84,95 @@ class QuickSearchQBitContextTest assertThat(QuickSearchQBitContext.getDiscoveredTables().get(0).getTableName()).isEqualTo("orders"); } + + + /*************************************************************************** + ** Test setClient and getClient round trip. + ***************************************************************************/ + @Test + void testSetAndGetClient() + { + Object dummyClient = new Object(); + QuickSearchQBitContext.setClient(dummyClient); + + assertThat(QuickSearchQBitContext.getClient()).isSameAs(dummyClient); + } + + + + /*************************************************************************** + ** Test setPublisher and getPublisher round trip. + ***************************************************************************/ + @Test + void testSetAndGetPublisher() + { + Object dummyPublisher = new Object(); + QuickSearchQBitContext.setPublisher(dummyPublisher); + + assertThat(QuickSearchQBitContext.getPublisher()).isSameAs(dummyPublisher); + } + + + + /*************************************************************************** + ** Test getTableConfig finds a table by name. + ***************************************************************************/ + @Test + void testGetTableConfig_findsTableByName() + { + QuickSearchableTableConfig orders = new QuickSearchableTableConfig().withTableName("orders"); + QuickSearchableTableConfig customers = new QuickSearchableTableConfig().withTableName("customers"); + QuickSearchQBitContext.setDiscoveredTables(List.of(orders, customers)); + + QuickSearchableTableConfig found = QuickSearchQBitContext.getTableConfig("customers"); + + assertThat(found).isSameAs(customers); + } + + + + /*************************************************************************** + ** Test getTableConfig returns null for an unknown table name. + ***************************************************************************/ + @Test + void testGetTableConfig_unknownTableName_returnsNull() + { + QuickSearchableTableConfig orders = new QuickSearchableTableConfig().withTableName("orders"); + QuickSearchQBitContext.setDiscoveredTables(List.of(orders)); + + assertThat(QuickSearchQBitContext.getTableConfig("nonexistent")).isNull(); + } + + + + /*************************************************************************** + ** Test getTableConfig when discoveredTables is null. + ***************************************************************************/ + @Test + void testGetTableConfig_nullDiscoveredTables_returnsNull() + { + assertThat(QuickSearchQBitContext.getTableConfig("orders")).isNull(); + } + + + + /*************************************************************************** + ** Test clear() resets all fields to null. + ***************************************************************************/ + @Test + void testClear_resetsAllFieldsToNull() + { + QuickSearchQBitContext.setConfig(new QuickSearchQBitConfig()); + QuickSearchQBitContext.setDiscoveredTables(List.of(new QuickSearchableTableConfig())); + QuickSearchQBitContext.setClient(new Object()); + QuickSearchQBitContext.setPublisher(new Object()); + + QuickSearchQBitContext.clear(); + + assertThat(QuickSearchQBitContext.getConfig()).isNull(); + assertThat(QuickSearchQBitContext.getDiscoveredTables()).isNull(); + assertThat(QuickSearchQBitContext.getClient()).isNull(); + assertThat(QuickSearchQBitContext.getPublisher()).isNull(); + } + } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducerTest.java new file mode 100644 index 0000000..2224ef5 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducerTest.java @@ -0,0 +1,538 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.kingsrook.qbits.quicksearch; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField; +import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable; +import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostDeleteCustomizer; +import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostInsertCustomizer; +import com.kingsrook.qbits.quicksearch.customizers.QuickSearchPostUpdateCustomizer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Tests for QuickSearchQBitProducer. + ** + ** Note: OpenSearch client creation will fail in these tests because no + ** OpenSearch instance is running. The producer is designed to log a warning + ** and continue in this case, so we can still verify metadata production. + *******************************************************************************/ +class QuickSearchQBitProducerTest +{ + private static final String BACKEND_NAME = "memory"; + private static final String SOURCE_TABLE = "person"; + + + + /*************************************************************************** + ** Test annotated entity: person table with @QuickSearchField annotations. + ***************************************************************************/ + @QuickSearchable(tableName = "person", basepullIntervalMinutes = 10, basepullTimestampField = "modifyDate") + static class PersonEntity + { + @QuickSearchField(weight = 3, includeLabel = true) + private String firstName; + + @QuickSearchField(weight = 2) + private String lastName; + + @QuickSearchField + private String email; + + private String internalField; + } + + + + /*************************************************************************** + ** Test annotated entity using fields() array instead of @QuickSearchField. + ***************************************************************************/ + @QuickSearchable(tableName = "order", fields = {"orderNumber", "customerName"}, basepullIntervalMinutes = 15) + static class OrderEntity + { + private String orderNumber; + private String customerName; + } + + + + /*************************************************************************** + ** Entity class without @QuickSearchable annotation. + ***************************************************************************/ + static class UnannotatedEntity + { + private String name; + } + + + + @BeforeEach + void setUp() + { + MemoryRecordStore.getInstance().reset(); + QuickSearchQBitContext.clear(); + } + + + + @AfterEach + void tearDown() + { + QContext.clear(); + QuickSearchQBitContext.clear(); + } + + + + /*************************************************************************** + ** Test that produce creates operational tables in the QInstance. + ***************************************************************************/ + @Test + void testProduce_createsOperationalTables() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(buildValidConfig()); + + producer.produce(qInstance); + + assertThat(qInstance.getTable("quickSearchIndex")).isNotNull(); + assertThat(qInstance.getTable("quickSearchIndex").getBackendName()).isEqualTo(BACKEND_NAME); + assertThat(qInstance.getTable("quickSearchIndex").getPrimaryKeyField()).isEqualTo("id"); + + assertThat(qInstance.getTable("quickSearchIndexRun")).isNotNull(); + assertThat(qInstance.getTable("quickSearchIndexRun").getBackendName()).isEqualTo(BACKEND_NAME); + assertThat(qInstance.getTable("quickSearchIndexRun").getPrimaryKeyField()).isEqualTo("id"); + } + + + + /*************************************************************************** + ** Test that tableNamePrefix is applied to operational table names. + ***************************************************************************/ + @Test + void testProduce_tableNamePrefixApplied() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = buildValidConfig() + .withTableNamePrefix("myApp_"); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + producer.produce(qInstance); + + assertThat(qInstance.getTable("myApp_quickSearchIndex")).isNotNull(); + assertThat(qInstance.getTable("myApp_quickSearchIndexRun")).isNotNull(); + assertThat(qInstance.getProcess("myApp_" + QuickSearchQBitProducer.BASEPULL_PROCESS_NAME)).isNotNull(); + assertThat(qInstance.getProcess("myApp_" + QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME)).isNotNull(); + } + + + + /*************************************************************************** + ** Test annotation discovery extracts correct field names and weights. + ***************************************************************************/ + @Test + void testAnnotationDiscovery_fieldWeightsAndLabels() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(buildValidConfig()); + + List discovered = producer.discoverSearchableTables(qInstance); + + assertThat(discovered).hasSize(1); + QuickSearchableTableConfig personConfig = discovered.get(0); + + assertThat(personConfig.getTableName()).isEqualTo("person"); + assertThat(personConfig.getPrimaryKeyField()).isEqualTo("id"); + assertThat(personConfig.getBasepullIntervalMinutes()).isEqualTo(10); + assertThat(personConfig.getBasepullTimestampField()).isEqualTo("modifyDate"); + assertThat(personConfig.getEnabledByDefault()).isTrue(); + + assertThat(personConfig.getSearchableFields()) + .containsExactly("firstName", "lastName", "email"); + + assertThat(personConfig.getFieldWeights()) + .containsEntry("firstName", 3) + .containsEntry("lastName", 2) + .containsEntry("email", 1); + + assertThat(personConfig.getFieldIncludeLabels()) + .containsEntry("firstName", true) + .containsEntry("lastName", false) + .containsEntry("email", false); + } + + + + /*************************************************************************** + ** Test annotation discovery falls back to fields() array when no + ** @QuickSearchField annotations are found. + ***************************************************************************/ + @Test + void testAnnotationDiscovery_fieldsArrayFallback() throws QException + { + QInstance qInstance = buildQInstance(); + addOrderTable(qInstance); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = buildValidConfig() + .withSearchableEntityClasses(List.of(OrderEntity.class)); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + List discovered = producer.discoverSearchableTables(qInstance); + + assertThat(discovered).hasSize(1); + QuickSearchableTableConfig orderConfig = discovered.get(0); + + assertThat(orderConfig.getTableName()).isEqualTo("order"); + assertThat(orderConfig.getSearchableFields()) + .containsExactly("orderNumber", "customerName"); + assertThat(orderConfig.getFieldWeights()) + .containsEntry("orderNumber", 1) + .containsEntry("customerName", 1); + assertThat(orderConfig.getBasepullIntervalMinutes()).isEqualTo(15); + } + + + + /*************************************************************************** + ** Test that unannotated classes are skipped during discovery. + ***************************************************************************/ + @Test + void testAnnotationDiscovery_unannotatedClassSkipped() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = buildValidConfig() + .withSearchableEntityClasses(List.of(UnannotatedEntity.class)); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + List discovered = producer.discoverSearchableTables(qInstance); + + assertThat(discovered).isEmpty(); + } + + + + /*************************************************************************** + ** Test that produce registers post-insert/update/delete customizers + ** on source tables when enableRealTimeIndexing is true. + ***************************************************************************/ + @Test + void testProduce_registersCustomizers() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = buildValidConfig() + .withEnableRealTimeIndexing(true); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + producer.produce(qInstance); + + QTableMetaData personTable = qInstance.getTable(SOURCE_TABLE); + assertThat(personTable).isNotNull(); + + Map customizers = + personTable.getCustomizers(); + assertThat(customizers).isNotNull(); + assertThat(customizers).containsKey("postInsertRecord"); + assertThat(customizers).containsKey("postUpdateRecord"); + assertThat(customizers).containsKey("postDeleteRecord"); + + assertThat(customizers.get("postInsertRecord").getName()) + .isEqualTo(QuickSearchPostInsertCustomizer.class.getName()); + assertThat(customizers.get("postUpdateRecord").getName()) + .isEqualTo(QuickSearchPostUpdateCustomizer.class.getName()); + assertThat(customizers.get("postDeleteRecord").getName()) + .isEqualTo(QuickSearchPostDeleteCustomizer.class.getName()); + } + + + + /*************************************************************************** + ** Test that no customizers are registered when enableRealTimeIndexing + ** is false. + ***************************************************************************/ + @Test + void testProduce_realTimeIndexingDisabled_noCustomizers() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = buildValidConfig() + .withEnableRealTimeIndexing(false); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + producer.produce(qInstance); + + QTableMetaData personTable = qInstance.getTable(SOURCE_TABLE); + assertThat(personTable).isNotNull(); + + Map customizers = + personTable.getCustomizers(); + assertThat(customizers == null || customizers.isEmpty()).isTrue(); + } + + + + /*************************************************************************** + ** Test that produce creates processes in the QInstance. + ***************************************************************************/ + @Test + void testProduce_createsProcesses() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(buildValidConfig()); + + producer.produce(qInstance); + + assertThat(qInstance.getProcess(QuickSearchQBitProducer.BASEPULL_PROCESS_NAME)).isNotNull(); + assertThat(qInstance.getProcess(QuickSearchQBitProducer.FULL_REINDEX_PROCESS_NAME)).isNotNull(); + } + + + + /*************************************************************************** + ** Test that produce creates an admin app in the QInstance. + ***************************************************************************/ + @Test + void testProduce_createsAdminApp() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(buildValidConfig()); + + producer.produce(qInstance); + + assertThat(qInstance.getApp(QuickSearchQBitProducer.APP_NAME)).isNotNull(); + assertThat(qInstance.getApp(QuickSearchQBitProducer.APP_NAME).getLabel()) + .isEqualTo("Quick Search Admin"); + } + + + + /*************************************************************************** + ** Test that produce populates QuickSearchQBitContext with discovered + ** tables and config. + ***************************************************************************/ + @Test + void testProduce_populatesContext() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = buildValidConfig(); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + producer.produce(qInstance); + + assertThat(QuickSearchQBitContext.getConfig()).isSameAs(config); + assertThat(QuickSearchQBitContext.getDiscoveredTables()).isNotNull(); + assertThat(QuickSearchQBitContext.getDiscoveredTables()).hasSize(1); + assertThat(QuickSearchQBitContext.getDiscoveredTables().get(0).getTableName()).isEqualTo("person"); + } + + + + /*************************************************************************** + ** Test that validation failure throws an exception with all error messages. + ***************************************************************************/ + @Test + void testProduce_validationFailure_throwsWithMessages() + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitConfig config = new QuickSearchQBitConfig(); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(config); + + assertThatThrownBy(() -> producer.produce(qInstance)) + .isInstanceOf(QException.class) + .hasMessageContaining("validation failed") + .hasMessageContaining("backendName is required"); + } + + + + /*************************************************************************** + ** Test the fluent withConfig setter returns this. + ***************************************************************************/ + @Test + void testFluentSetter_returnsThis() + { + QuickSearchQBitProducer producer = new QuickSearchQBitProducer(); + QuickSearchQBitConfig config = new QuickSearchQBitConfig(); + + assertThat(producer.withConfig(config)).isSameAs(producer); + assertThat(producer.getConfig()).isSameAs(config); + } + + + + /*************************************************************************** + ** Test that the operational tables have all expected fields from the + ** QuickSearchIndex and QuickSearchIndexRun entities. + ***************************************************************************/ + @Test + void testProduce_operationalTableFields() throws QException + { + QInstance qInstance = buildQInstance(); + QContext.init(qInstance, new QSession()); + + QuickSearchQBitProducer producer = new QuickSearchQBitProducer() + .withConfig(buildValidConfig()); + + producer.produce(qInstance); + + QTableMetaData indexTable = qInstance.getTable("quickSearchIndex"); + assertThat(indexTable.getFields()).containsKey("id"); + assertThat(indexTable.getFields()).containsKey("tableName"); + assertThat(indexTable.getFields()).containsKey("enabled"); + assertThat(indexTable.getFields()).containsKey("basepullIntervalMinutes"); + assertThat(indexTable.getFields()).containsKey("basepullTimestampField"); + assertThat(indexTable.getFields()).containsKey("searchableFieldsJson"); + assertThat(indexTable.getFields()).containsKey("lastBasepullTime"); + assertThat(indexTable.getFields()).containsKey("lastFullReindexTime"); + assertThat(indexTable.getFields()).containsKey("recordCount"); + assertThat(indexTable.getFields()).containsKey("status"); + assertThat(indexTable.getFields()).containsKey("createDate"); + assertThat(indexTable.getFields()).containsKey("modifyDate"); + + QTableMetaData indexRunTable = qInstance.getTable("quickSearchIndexRun"); + assertThat(indexRunTable.getFields()).containsKey("id"); + assertThat(indexRunTable.getFields()).containsKey("quickSearchIndexId"); + assertThat(indexRunTable.getFields()).containsKey("runType"); + assertThat(indexRunTable.getFields()).containsKey("status"); + assertThat(indexRunTable.getFields()).containsKey("startTime"); + assertThat(indexRunTable.getFields()).containsKey("endTime"); + assertThat(indexRunTable.getFields()).containsKey("recordsProcessed"); + assertThat(indexRunTable.getFields()).containsKey("recordsIndexed"); + assertThat(indexRunTable.getFields()).containsKey("errorCount"); + assertThat(indexRunTable.getFields()).containsKey("errorMessage"); + assertThat(indexRunTable.getFields()).containsKey("createDate"); + assertThat(indexRunTable.getFields()).containsKey("modifyDate"); + } + + + + /*************************************************************************** + ** Build a base QInstance with a memory backend and source tables. + ***************************************************************************/ + private QInstance buildQInstance() + { + QInstance qInstance = new QInstance(); + + qInstance.addBackend(new QBackendMetaData() + .withName(BACKEND_NAME) + .withBackendType(MemoryBackendModule.class)); + + qInstance.setAuthentication(new QAuthenticationMetaData() + .withName("anonymous") + .withType(QAuthenticationType.FULLY_ANONYMOUS)); + + qInstance.addTable(new QTableMetaData() + .withName(SOURCE_TABLE) + .withBackendName(BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withField(new QFieldMetaData("lastName", QFieldType.STRING)) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))); + + return (qInstance); + } + + + + /*************************************************************************** + ** Add an order table to the QInstance. + ***************************************************************************/ + private void addOrderTable(QInstance qInstance) + { + qInstance.addTable(new QTableMetaData() + .withName("order") + .withBackendName(BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("orderNumber", QFieldType.STRING)) + .withField(new QFieldMetaData("customerName", QFieldType.STRING)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))); + } + + + + /*************************************************************************** + ** Build a valid QuickSearchQBitConfig for testing. + ***************************************************************************/ + private QuickSearchQBitConfig buildValidConfig() + { + return new QuickSearchQBitConfig() + .withBackendName(BACKEND_NAME) + .withOpensearchHost("localhost") + .withOpensearchPort(9200) + .withOpensearchIndexName("test-quick-search") + .withSearchableEntityClasses(List.of(PersonEntity.class)); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfigTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfigTest.java index e0afdb0..86eaa77 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfigTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/QuickSearchableTableConfigTest.java @@ -17,6 +17,7 @@ package com.kingsrook.qbits.quicksearch; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -28,22 +29,28 @@ class QuickSearchableTableConfigTest { /*************************************************************************** - ** Test all getters and fluent setters. + ** Test all fluent setters and getters including new map fields. ***************************************************************************/ @Test void testGettersAndSetters() { List fields = List.of("name", "description"); + Map weights = Map.of("name", 10, "description", 5); + Map includeLabels = Map.of("name", true, "description", false); QuickSearchableTableConfig config = new QuickSearchableTableConfig() .withTableName("orders") .withSearchableFields(fields) + .withFieldWeights(weights) + .withFieldIncludeLabels(includeLabels) .withBasepullIntervalMinutes(10) .withBasepullTimestampField("updatedAt") .withEnabledByDefault(false); assertThat(config.getTableName()).isEqualTo("orders"); assertThat(config.getSearchableFields()).isEqualTo(fields); + assertThat(config.getFieldWeights()).isEqualTo(weights); + assertThat(config.getFieldIncludeLabels()).isEqualTo(includeLabels); assertThat(config.getBasepullIntervalMinutes()).isEqualTo(10); assertThat(config.getBasepullTimestampField()).isEqualTo("updatedAt"); assertThat(config.getEnabledByDefault()).isFalse(); @@ -58,16 +65,22 @@ class QuickSearchableTableConfigTest void testStandardSetters() { List fields = List.of("field1", "field2"); + Map weights = Map.of("field1", 3); + Map includeLabels = Map.of("field1", true); QuickSearchableTableConfig config = new QuickSearchableTableConfig(); config.setTableName("customers"); config.setSearchableFields(fields); + config.setFieldWeights(weights); + config.setFieldIncludeLabels(includeLabels); config.setBasepullIntervalMinutes(15); config.setBasepullTimestampField("modifyDate"); config.setEnabledByDefault(true); assertThat(config.getTableName()).isEqualTo("customers"); assertThat(config.getSearchableFields()).isEqualTo(fields); + assertThat(config.getFieldWeights()).isEqualTo(weights); + assertThat(config.getFieldIncludeLabels()).isEqualTo(includeLabels); assertThat(config.getBasepullIntervalMinutes()).isEqualTo(15); assertThat(config.getBasepullTimestampField()).isEqualTo("modifyDate"); assertThat(config.getEnabledByDefault()).isTrue(); @@ -83,14 +96,27 @@ class QuickSearchableTableConfigTest { QuickSearchableTableConfig config = new QuickSearchableTableConfig(); - QuickSearchableTableConfig result = config - .withTableName("test") - .withSearchableFields(List.of("a")) - .withBasepullIntervalMinutes(5) - .withBasepullTimestampField("ts") - .withEnabledByDefault(true); + assertThat(config.withTableName("test")).isSameAs(config); + assertThat(config.withSearchableFields(List.of("a"))).isSameAs(config); + assertThat(config.withFieldWeights(Map.of("a", 1))).isSameAs(config); + assertThat(config.withFieldIncludeLabels(Map.of("a", true))).isSameAs(config); + assertThat(config.withBasepullIntervalMinutes(5)).isSameAs(config); + assertThat(config.withBasepullTimestampField("ts")).isSameAs(config); + assertThat(config.withEnabledByDefault(true)).isSameAs(config); + } - assertThat(result).isSameAs(config); + + + /*************************************************************************** + ** Test null defaults for new fields. + ***************************************************************************/ + @Test + void testNewFields_defaultToNull() + { + QuickSearchableTableConfig config = new QuickSearchableTableConfig(); + + assertThat(config.getFieldWeights()).isNull(); + assertThat(config.getFieldIncludeLabels()).isNull(); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchActionTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchActionTest.java index 8de82bb..99f8a0f 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchActionTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchActionTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,109 +13,439 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.actions; -import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.HitsMetadata; +import org.opensearch.client.opensearch.core.search.TotalHits; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /******************************************************************************* ** Tests for QuickSearchAction. + ** + ** Uses Mockito to mock QuickSearchOpenSearchClient and the OpenSearch + ** SearchResponse chain (SearchResponse → HitsMetadata → TotalHits / Hit). *******************************************************************************/ -class QuickSearchActionTest extends BaseQuickSearchTest +@SuppressWarnings("unchecked") +class QuickSearchActionTest { + private QuickSearchOpenSearchClient mockClient; + private List tableConfigs; + private QuickSearchAction action; - /*************************************************************************** - ** Test execute with empty search term returns empty results. - ***************************************************************************/ - @Test - void testExecute_emptySearchTerm_returnsEmptyResults() throws Exception + + + /******************************************************************************* + ** Set up a mock client, table configs, and context state before each test. + *******************************************************************************/ + @BeforeEach + void setUp() { - QuickSearchAction action = new QuickSearchAction(); - QuickSearchInput input = new QuickSearchInput() - .withSearchTerm(""); + mockClient = mock(QuickSearchOpenSearchClient.class); + + tableConfigs = List.of( + new QuickSearchableTableConfig() + .withTableName("orders") + .withPrimaryKeyField("id") + .withSearchableFields(List.of("orderNumber", "customerName"))); + + QuickSearchQBitContext.setClient(mockClient); + QuickSearchQBitContext.setDiscoveredTables(tableConfigs); + + action = new QuickSearchAction(); + } + + + + /******************************************************************************* + ** Clear context after each test to prevent state leakage. + *******************************************************************************/ + @AfterEach + void tearDown() + { + QuickSearchQBitContext.clear(); + } + + + + /******************************************************************************* + ** Build a mocked SearchResponse with the given hits and totalHits count. + *******************************************************************************/ + private SearchResponse buildMockResponse(List> hits, long totalHits) + { + TotalHits mockTotal = mock(TotalHits.class); + when(mockTotal.value()).thenReturn(totalHits); + + HitsMetadata mockHitsMeta = mock(HitsMetadata.class); + when(mockHitsMeta.total()).thenReturn(mockTotal); + when(mockHitsMeta.hits()).thenReturn(hits); + + SearchResponse mockResponse = mock(SearchResponse.class); + when(mockResponse.hits()).thenReturn(mockHitsMeta); + + return (mockResponse); + } + + + + /******************************************************************************* + ** Build a single mocked Hit with the given document, score, and highlights. + *******************************************************************************/ + private Hit buildMockHit(OpenSearchDocument doc, Double score, Map> highlight) + { + Hit hit = mock(Hit.class); + when(hit.source()).thenReturn(doc); + when(hit.score()).thenReturn(score); + when(hit.highlight()).thenReturn(highlight); + return (hit); + } + + + + /******************************************************************************* + ** Null searchTerm returns empty output with totalHits=0 and hasMore=false. + *******************************************************************************/ + @Test + void testNullSearchTerm_returnsEmptyOutput() throws QException + { + QuickSearchInput input = new QuickSearchInput().withSearchTerm(null); QuickSearchOutput output = action.execute(input); assertThat(output.getResults()).isEmpty(); - assertThat(output.getTotalCount()).isEqualTo(0); + assertThat(output.getTotalHits()).isEqualTo(0L); + assertThat(output.getHasMore()).isFalse(); } - /*************************************************************************** - ** Test execute with null search term returns empty results. - ***************************************************************************/ + /******************************************************************************* + ** Empty string searchTerm returns empty output. + *******************************************************************************/ @Test - void testExecute_nullSearchTerm_returnsEmptyResults() throws Exception + void testEmptySearchTerm_returnsEmptyOutput() throws QException { - QuickSearchAction action = new QuickSearchAction(); - QuickSearchInput input = new QuickSearchInput() - .withSearchTerm(null); + QuickSearchInput input = new QuickSearchInput().withSearchTerm(""); QuickSearchOutput output = action.execute(input); assertThat(output.getResults()).isEmpty(); - assertThat(output.getTotalCount()).isEqualTo(0); + assertThat(output.getTotalHits()).isEqualTo(0L); + assertThat(output.getHasMore()).isFalse(); } - /*************************************************************************** - ** Test execute with whitespace search term returns empty results. - ***************************************************************************/ + /******************************************************************************* + ** Whitespace-only searchTerm returns empty output. + *******************************************************************************/ @Test - void testExecute_whitespaceSearchTerm_returnsEmptyResults() throws Exception + void testBlankSearchTerm_returnsEmptyOutput() throws QException { - QuickSearchAction action = new QuickSearchAction(); - QuickSearchInput input = new QuickSearchInput() - .withSearchTerm(" "); + QuickSearchInput input = new QuickSearchInput().withSearchTerm(" "); QuickSearchOutput output = action.execute(input); assertThat(output.getResults()).isEmpty(); - assertThat(output.getTotalCount()).isEqualTo(0); + assertThat(output.getTotalHits()).isEqualTo(0L); + assertThat(output.getHasMore()).isFalse(); } - /*************************************************************************** - ** Test execute with valid search term throws exception when OpenSearch - ** is not running, with expected error message. - ***************************************************************************/ + /******************************************************************************* + ** Valid search maps each hit to a QuickSearchResult with all fields populated. + *******************************************************************************/ @Test - void testExecute_validSearchTerm_throwsWhenOpenSearchUnavailable() + void testValidSearch_mapsResultsCorrectly() throws QException { - QuickSearchAction action = new QuickSearchAction(); - QuickSearchInput input = new QuickSearchInput() - .withSearchTerm("test") - .withLimit(null); + OpenSearchDocument doc = new OpenSearchDocument() + .withSourceTable("orders") + .withRecordId("42") + .withRecordLabel("Order #42"); - assertThatThrownBy(() -> action.execute(input)) - .isInstanceOf(Exception.class) - .hasMessageContaining("Failed to search documents"); + Hit hit = buildMockHit(doc, 1.5, Map.of("searchableText", List.of("Order 42"))); + + SearchResponse response = buildMockResponse(List.of(hit), 1L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + QuickSearchInput input = new QuickSearchInput() + .withSearchTerm("42") + .withLimit(10) + .withOffset(0); + + QuickSearchOutput output = action.execute(input); + + assertThat(output.getResults()).hasSize(1); + QuickSearchResult result = output.getResults().get(0); + assertThat(result.getTableName()).isEqualTo("orders"); + assertThat(result.getRecordId()).isEqualTo("42"); + assertThat(result.getRecordLabel()).isEqualTo("Order #42"); + assertThat(result.getScore()).isEqualTo(1.5f); + assertThat(result.getHighlightSnippet()).isEqualTo("Order 42"); + assertThat(output.getTotalHits()).isEqualTo(1L); + assertThat(output.getHasMore()).isFalse(); } - /*************************************************************************** - ** Test execute with negative limit throws exception when OpenSearch - ** is not running, verifying limit handling occurs before the search. - ***************************************************************************/ + /******************************************************************************* + ** Multiple highlight fragments are joined with "...". + *******************************************************************************/ @Test - void testExecute_negativeLimit_throwsWhenOpenSearchUnavailable() + void testHighlightFragments_joinedWithEllipsis() throws QException { - QuickSearchAction action = new QuickSearchAction(); - QuickSearchInput input = new QuickSearchInput() - .withSearchTerm("test") - .withLimit(-1); + OpenSearchDocument doc = new OpenSearchDocument() + .withSourceTable("orders") + .withRecordId("1") + .withRecordLabel("Order 1"); - assertThatThrownBy(() -> action.execute(input)) - .isInstanceOf(Exception.class) - .hasMessageContaining("Failed to search documents"); + Hit hit = buildMockHit(doc, 1.0, Map.of("searchableText", List.of("fragment one", "fragment two"))); + + SearchResponse response = buildMockResponse(List.of(hit), 1L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("one")); + + assertThat(output.getResults().get(0).getHighlightSnippet()).isEqualTo("fragment one...fragment two"); + } + + + + /******************************************************************************* + ** When highlight map is null, highlightSnippet is null. + *******************************************************************************/ + @Test + void testNullHighlight_resultHasNullSnippet() throws QException + { + OpenSearchDocument doc = new OpenSearchDocument() + .withSourceTable("orders") + .withRecordId("5") + .withRecordLabel("Order 5"); + + Hit hit = buildMockHit(doc, 0.9, null); + + SearchResponse response = buildMockResponse(List.of(hit), 1L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + QuickSearchOutput output = action.execute(new QuickSearchInput().withSearchTerm("order")); + + assertThat(output.getResults().get(0).getHighlightSnippet()).isNull(); + } + + + + /******************************************************************************* + ** Null limit is defaulted to 25. + *******************************************************************************/ + @Test + void testNullLimit_defaultsTo25() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(null)); + + verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList()); + } + + + + /******************************************************************************* + ** Zero limit is defaulted to 25. + *******************************************************************************/ + @Test + void testZeroLimit_defaultsTo25() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(0)); + + verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList()); + } + + + + /******************************************************************************* + ** Negative limit is defaulted to 25. + *******************************************************************************/ + @Test + void testNegativeLimit_defaultsTo25() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput().withSearchTerm("test").withLimit(-5)); + + verify(mockClient).search(anyString(), any(), eq(25), anyInt(), anyList()); + } + + + + /******************************************************************************* + ** Null offset is defaulted to 0. + *******************************************************************************/ + @Test + void testNullOffset_defaultsTo0() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput().withSearchTerm("test").withOffset(null)); + + verify(mockClient).search(anyString(), any(), anyInt(), eq(0), anyList()); + } + + + + /******************************************************************************* + ** Negative offset is defaulted to 0. + *******************************************************************************/ + @Test + void testNegativeOffset_defaultsTo0() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput().withSearchTerm("test").withOffset(-1)); + + verify(mockClient).search(anyString(), any(), anyInt(), eq(0), anyList()); + } + + + + /******************************************************************************* + ** hasMore is true when (offset + results.size()) < totalHits. + *******************************************************************************/ + @Test + void testHasMore_trueWhenMoreResultsAvailable() throws QException + { + OpenSearchDocument doc = new OpenSearchDocument() + .withSourceTable("orders").withRecordId("1").withRecordLabel("Order 1"); + + List> hits = List.of(buildMockHit(doc, 1.0, null)); + + // 1 result, offset=0, totalHits=5 → hasMore = (0+1) < 5 = true + SearchResponse response = buildMockResponse(hits, 5L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + QuickSearchOutput output = action.execute(new QuickSearchInput() + .withSearchTerm("order") + .withOffset(0) + .withLimit(1)); + + assertThat(output.getHasMore()).isTrue(); + assertThat(output.getTotalHits()).isEqualTo(5L); + } + + + + /******************************************************************************* + ** hasMore is false when (offset + results.size()) >= totalHits. + *******************************************************************************/ + @Test + void testHasMore_falseWhenOnLastPage() throws QException + { + OpenSearchDocument doc1 = new OpenSearchDocument() + .withSourceTable("orders").withRecordId("4").withRecordLabel("Order 4"); + OpenSearchDocument doc2 = new OpenSearchDocument() + .withSourceTable("orders").withRecordId("5").withRecordLabel("Order 5"); + + List> hits = List.of( + buildMockHit(doc1, 1.0, null), + buildMockHit(doc2, 0.9, null)); + + // 2 results, offset=3, totalHits=5 → hasMore = (3+2) < 5 = false + SearchResponse response = buildMockResponse(hits, 5L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + QuickSearchOutput output = action.execute(new QuickSearchInput() + .withSearchTerm("order") + .withOffset(3) + .withLimit(2)); + + assertThat(output.getHasMore()).isFalse(); + } + + + + /******************************************************************************* + ** tableName filter is passed through to client.search(). + *******************************************************************************/ + @Test + void testTableNameFilter_passedToClient() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), anyString(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput() + .withSearchTerm("widget") + .withTableName("products")); + + verify(mockClient).search(anyString(), eq("products"), anyInt(), anyInt(), anyList()); + } + + + + /******************************************************************************* + ** null tableName is passed through to client.search() as null. + *******************************************************************************/ + @Test + void testNullTableName_passedAsNullToClient() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), isNull(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput() + .withSearchTerm("widget") + .withTableName(null)); + + verify(mockClient).search(anyString(), isNull(), anyInt(), anyInt(), anyList()); + } + + + + /******************************************************************************* + ** Search term is normalized (lowercased, trimmed) before being passed to client. + *******************************************************************************/ + @Test + void testSearchTerm_isNormalizedBeforeClientCall() throws QException + { + SearchResponse response = buildMockResponse(Collections.emptyList(), 0L); + when(mockClient.search(anyString(), any(), anyInt(), anyInt(), anyList())).thenReturn(response); + + action.execute(new QuickSearchInput().withSearchTerm(" HELLO WORLD ")); + + ArgumentCaptor termCaptor = ArgumentCaptor.forClass(String.class); + verify(mockClient).search(termCaptor.capture(), any(), anyInt(), anyInt(), anyList()); + + assertThat(termCaptor.getValue()).isEqualTo("hello world"); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchDTOsTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchDTOsTest.java index 5fd7e4f..d82b214 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchDTOsTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/actions/QuickSearchDTOsTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.actions; @@ -22,146 +23,119 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* - ** Tests for QuickSearch DTOs (Input, Output, Result). + ** Tests for QuickSearchInput, QuickSearchOutput, and QuickSearchResult DTOs. *******************************************************************************/ class QuickSearchDTOsTest { - /*************************************************************************** - ** Test QuickSearchInput getters and setters. - ***************************************************************************/ + /******************************************************************************* + ** Test that QuickSearchInput fluent setters return this and getters return values. + *******************************************************************************/ @Test - void testQuickSearchInput_gettersAndSetters() + void testQuickSearchInputFluentSetters() { QuickSearchInput input = new QuickSearchInput() - .withSearchTerm("test query") - .withTableName("orders") - .withLimit(25); + .withSearchTerm("hello") + .withTableName("myTable") + .withLimit(10) + .withOffset(20); - assertThat(input.getSearchTerm()).isEqualTo("test query"); - assertThat(input.getTableName()).isEqualTo("orders"); - assertThat(input.getLimit()).isEqualTo(25); + assertThat(input.getSearchTerm()).isEqualTo("hello"); + assertThat(input.getTableName()).isEqualTo("myTable"); + assertThat(input.getLimit()).isEqualTo(10); + assertThat(input.getOffset()).isEqualTo(20); } - - - /*************************************************************************** - ** Test QuickSearchInput standard setters. - ***************************************************************************/ + /******************************************************************************* + ** Test that QuickSearchInput with setter returns this (fluent chaining). + *******************************************************************************/ @Test - void testQuickSearchInput_standardSetters() - { - QuickSearchInput input = new QuickSearchInput(); - input.setSearchTerm("search"); - input.setTableName("customers"); - input.setLimit(100); - - assertThat(input.getSearchTerm()).isEqualTo("search"); - assertThat(input.getTableName()).isEqualTo("customers"); - assertThat(input.getLimit()).isEqualTo(100); - } - - - - /*************************************************************************** - ** Test QuickSearchOutput getters and setters. - ***************************************************************************/ - @Test - void testQuickSearchOutput_gettersAndSetters() - { - List results = List.of( - new QuickSearchResult().withRecordId("1"), - new QuickSearchResult().withRecordId("2") - ); - - QuickSearchOutput output = new QuickSearchOutput() - .withResults(results) - .withTotalCount(2); - - assertThat(output.getResults()).hasSize(2); - assertThat(output.getTotalCount()).isEqualTo(2); - } - - - - /*************************************************************************** - ** Test QuickSearchOutput standard setters. - ***************************************************************************/ - @Test - void testQuickSearchOutput_standardSetters() - { - List results = List.of(new QuickSearchResult()); - - QuickSearchOutput output = new QuickSearchOutput(); - output.setResults(results); - output.setTotalCount(1); - - assertThat(output.getResults()).hasSize(1); - assertThat(output.getTotalCount()).isEqualTo(1); - } - - - - /*************************************************************************** - ** Test QuickSearchResult getters and setters. - ***************************************************************************/ - @Test - void testQuickSearchResult_gettersAndSetters() - { - QuickSearchResult result = new QuickSearchResult() - .withTableName("orders") - .withRecordId("12345") - .withRecordLabel("Order #12345") - .withMatchedText("customer john doe shipping"); - - assertThat(result.getTableName()).isEqualTo("orders"); - assertThat(result.getRecordId()).isEqualTo("12345"); - assertThat(result.getRecordLabel()).isEqualTo("Order #12345"); - assertThat(result.getMatchedText()).isEqualTo("customer john doe shipping"); - } - - - - /*************************************************************************** - ** Test QuickSearchResult standard setters. - ***************************************************************************/ - @Test - void testQuickSearchResult_standardSetters() - { - QuickSearchResult result = new QuickSearchResult(); - result.setTableName("customers"); - result.setRecordId("999"); - result.setRecordLabel("Customer 999"); - result.setMatchedText("acme corp"); - - assertThat(result.getTableName()).isEqualTo("customers"); - assertThat(result.getRecordId()).isEqualTo("999"); - assertThat(result.getRecordLabel()).isEqualTo("Customer 999"); - assertThat(result.getMatchedText()).isEqualTo("acme corp"); - } - - - - /*************************************************************************** - ** Test fluent setters return this. - ***************************************************************************/ - @Test - void testFluentSetters_returnThis() + void testQuickSearchInputWithReturnsThis() { QuickSearchInput input = new QuickSearchInput(); assertThat(input.withSearchTerm("test")).isSameAs(input); assertThat(input.withTableName("table")).isSameAs(input); - assertThat(input.withLimit(10)).isSameAs(input); + assertThat(input.withLimit(5)).isSameAs(input); + assertThat(input.withOffset(0)).isSameAs(input); + } - QuickSearchOutput output = new QuickSearchOutput(); - assertThat(output.withResults(List.of())).isSameAs(output); - assertThat(output.withTotalCount(0)).isSameAs(output); + /******************************************************************************* + ** Test that QuickSearchInput defaults are null. + *******************************************************************************/ + @Test + void testQuickSearchInputDefaults() + { + QuickSearchInput input = new QuickSearchInput(); + assertThat(input.getSearchTerm()).isNull(); + assertThat(input.getTableName()).isNull(); + assertThat(input.getLimit()).isNull(); + assertThat(input.getOffset()).isNull(); + } + /******************************************************************************* + ** Test that QuickSearchResult fluent setters return this and getters return values. + *******************************************************************************/ + @Test + void testQuickSearchResultFluentSetters() + { + QuickSearchResult result = new QuickSearchResult() + .withTableName("orders") + .withRecordId("42") + .withRecordLabel("Order #42") + .withScore(0.95f) + .withHighlightSnippet("matched text here"); + + assertThat(result.getTableName()).isEqualTo("orders"); + assertThat(result.getRecordId()).isEqualTo("42"); + assertThat(result.getRecordLabel()).isEqualTo("Order #42"); + assertThat(result.getScore()).isEqualTo(0.95f); + assertThat(result.getHighlightSnippet()).isEqualTo("matched text here"); + } + + /******************************************************************************* + ** Test that QuickSearchResult withX methods return this. + *******************************************************************************/ + @Test + void testQuickSearchResultWithReturnsThis() + { QuickSearchResult result = new QuickSearchResult(); assertThat(result.withTableName("t")).isSameAs(result); assertThat(result.withRecordId("1")).isSameAs(result); - assertThat(result.withRecordLabel("l")).isSameAs(result); - assertThat(result.withMatchedText("m")).isSameAs(result); + assertThat(result.withRecordLabel("label")).isSameAs(result); + assertThat(result.withScore(1.0f)).isSameAs(result); + assertThat(result.withHighlightSnippet("snip")).isSameAs(result); + } + + /******************************************************************************* + ** Test that QuickSearchOutput fluent setters return this and getters return values. + *******************************************************************************/ + @Test + void testQuickSearchOutputFluentSetters() + { + QuickSearchResult r1 = new QuickSearchResult().withRecordId("1"); + QuickSearchResult r2 = new QuickSearchResult().withRecordId("2"); + List results = List.of(r1, r2); + + QuickSearchOutput output = new QuickSearchOutput() + .withResults(results) + .withTotalHits(100L) + .withHasMore(true); + + assertThat(output.getResults()).containsExactly(r1, r2); + assertThat(output.getTotalHits()).isEqualTo(100L); + assertThat(output.getHasMore()).isTrue(); + } + + /******************************************************************************* + ** Test that QuickSearchOutput withX methods return this. + *******************************************************************************/ + @Test + void testQuickSearchOutputWithReturnsThis() + { + QuickSearchOutput output = new QuickSearchOutput(); + assertThat(output.withResults(List.of())).isSameAs(output); + assertThat(output.withTotalHits(0L)).isSameAs(output); + assertThat(output.withHasMore(false)).isSameAs(output); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchDeleteCustomizerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchDeleteCustomizerTest.java deleted file mode 100644 index 78c38fb..0000000 --- a/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchDeleteCustomizerTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2024 Kingsrook, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kingsrook.qbits.quicksearch.customizers; - - -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; - - -/******************************************************************************* - ** Tests for QuickSearchDeleteCustomizer. - *******************************************************************************/ -class QuickSearchDeleteCustomizerTest extends BaseQuickSearchTest -{ - - /*************************************************************************** - ** Test postDelete with null records returns null. - ***************************************************************************/ - @Test - void testPostDelete_nullRecords_returnsNull() throws Exception - { - QuickSearchDeleteCustomizer customizer = new QuickSearchDeleteCustomizer(config, TEST_TABLE_NAME, "id"); - - DeleteInput deleteInput = new DeleteInput(); - deleteInput.setTableName(TEST_TABLE_NAME); - - List result = customizer.postDelete(deleteInput, null); - - assertThat(result).isNull(); - } - - - - /*************************************************************************** - ** Test postDelete with empty records returns empty list. - ***************************************************************************/ - @Test - void testPostDelete_emptyRecords_returnsEmptyList() throws Exception - { - QuickSearchDeleteCustomizer customizer = new QuickSearchDeleteCustomizer(config, TEST_TABLE_NAME, "id"); - - DeleteInput deleteInput = new DeleteInput(); - deleteInput.setTableName(TEST_TABLE_NAME); - - List result = customizer.postDelete(deleteInput, new ArrayList<>()); - - assertThat(result).isEmpty(); - } - - - - /*************************************************************************** - ** Test postDelete returns same records it received. - ***************************************************************************/ - @Test - void testPostDelete_withRecords_returnsSameRecords() throws Exception - { - QuickSearchDeleteCustomizer customizer = new QuickSearchDeleteCustomizer(config, TEST_TABLE_NAME, "id"); - - DeleteInput deleteInput = new DeleteInput(); - deleteInput.setTableName(TEST_TABLE_NAME); - - List records = List.of( - new QRecord().withValue("id", 1).withValue("name", "Test 1"), - new QRecord().withValue("id", 2).withValue("name", "Test 2") - ); - - // Note: This will fail to delete from OpenSearch since no OpenSearch is running, - // but the customizer should handle errors gracefully and return the records - List result = customizer.postDelete(deleteInput, records); - - assertThat(result).hasSize(2); - assertThat(result).isEqualTo(records); - } - - - -} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java new file mode 100644 index 0000000..6fbc80b --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostDeleteCustomizerTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.customizers; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.publisher.IndexEvent; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + + +/******************************************************************************* + ** Tests for QuickSearchPostDeleteCustomizer. + *******************************************************************************/ +class QuickSearchPostDeleteCustomizerTest +{ + private static final String TABLE_NAME = "orders"; + private static final String PK_FIELD = "id"; + + private IndexEventPublisher mockPublisher; + + + + /******************************************************************************* + ** Set up mock publisher in QuickSearchQBitContext and a QContext with a test + ** QInstance before each test. + *******************************************************************************/ + @BeforeEach + void setUp() + { + mockPublisher = mock(IndexEventPublisher.class); + QuickSearchQBitContext.setPublisher(mockPublisher); + QuickSearchQBitContext.setDiscoveredTables(List.of( + new QuickSearchableTableConfig().withTableName(TABLE_NAME).withPrimaryKeyField(PK_FIELD) + )); + + QInstance qInstance = new QInstance(); + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME) + .withPrimaryKeyField(PK_FIELD) + .withField(new QFieldMetaData(PK_FIELD, QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING))); + qInstance.addBackend(new com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData().withName("testBackend")); + qInstance.setAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS)); + qInstance.getTable(TABLE_NAME).withBackendName("testBackend"); + QContext.init(qInstance, new QSession()); + } + + + + /******************************************************************************* + ** Clear QuickSearchQBitContext and QContext after each test to prevent leakage. + *******************************************************************************/ + @AfterEach + void tearDown() + { + QuickSearchQBitContext.clear(); + QContext.clear(); + } + + + + /******************************************************************************* + ** Test that no-arg constructor completes without throwing. + *******************************************************************************/ + @Test + void testNoArgConstructor_doesNotThrow() + { + assertThatNoException().isThrownBy(QuickSearchPostDeleteCustomizer::new); + } + + + + /******************************************************************************* + ** Test that post-delete with valid records builds correct DELETE events and + ** calls publisher.publishDeleteEvents. Records have no QRecord attached + ** (DELETE events only carry tableName and recordId). + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Test + void testPostDelete_withRecords_publishesDeleteEvents() throws QException + { + QRecord record1 = new QRecord().withValue(PK_FIELD, 5); + QRecord record2 = new QRecord().withValue(PK_FIELD, 6); + + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TABLE_NAME); + + QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer(); + List result = customizer.postDelete(deleteInput, List.of(record1, record2)); + + assertThat(result).containsExactly(record1, record2); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mockPublisher).publishDeleteEvents(captor.capture()); + + List events = captor.getValue(); + assertThat(events).hasSize(2); + assertThat(events.get(0).getAction()).isEqualTo(IndexEventAction.DELETE); + assertThat(events.get(0).getTableName()).isEqualTo(TABLE_NAME); + assertThat(events.get(0).getRecordId()).isEqualTo("5"); + assertThat(events.get(0).getRecord()).isNull(); + assertThat(events.get(1).getAction()).isEqualTo(IndexEventAction.DELETE); + assertThat(events.get(1).getRecordId()).isEqualTo("6"); + } + + + + /******************************************************************************* + ** Test that post-delete with empty records list returns input without calling + ** publisher. + *******************************************************************************/ + @Test + void testPostDelete_withEmptyRecords_returnsInputWithoutPublishing() throws QException + { + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TABLE_NAME); + + QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer(); + List result = customizer.postDelete(deleteInput, List.of()); + + assertThat(result).isEmpty(); + verify(mockPublisher, never()).publishDeleteEvents(anyList()); + } + + + + /******************************************************************************* + ** Test that post-delete with null records returns null without calling publisher. + *******************************************************************************/ + @Test + void testPostDelete_withNullRecords_returnsNullWithoutPublishing() throws QException + { + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TABLE_NAME); + + QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer(); + List result = customizer.postDelete(deleteInput, null); + + assertThat(result).isNull(); + verify(mockPublisher, never()).publishDeleteEvents(anyList()); + } + + + + /******************************************************************************* + ** Test that when publisher is null, records are still returned (no exception). + *******************************************************************************/ + @Test + void testPostDelete_publisherNull_returnsRecordsWithoutThrowing() throws QException + { + QuickSearchQBitContext.setPublisher(null); + + QRecord record = new QRecord().withValue(PK_FIELD, 1); + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TABLE_NAME); + + QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer(); + List result = customizer.postDelete(deleteInput, List.of(record)); + + assertThat(result).containsExactly(record); + } + + + + /******************************************************************************* + ** Test that when publisher throws, a warning is logged and records are returned + ** (no exception propagated to caller). + *******************************************************************************/ + @Test + void testPostDelete_publisherThrows_returnsRecordsWithoutThrowing() throws QException + { + doThrow(new QException("publish failed")) + .when(mockPublisher).publishDeleteEvents(anyList()); + + QRecord record = new QRecord().withValue(PK_FIELD, 1); + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TABLE_NAME); + + QuickSearchPostDeleteCustomizer customizer = new QuickSearchPostDeleteCustomizer(); + assertThatNoException().isThrownBy(() -> customizer.postDelete(deleteInput, List.of(record))); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java new file mode 100644 index 0000000..9d1f472 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostInsertCustomizerTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.customizers; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.publisher.IndexEvent; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + + +/******************************************************************************* + ** Tests for QuickSearchPostInsertCustomizer. + *******************************************************************************/ +class QuickSearchPostInsertCustomizerTest +{ + private static final String TABLE_NAME = "orders"; + private static final String PK_FIELD = "id"; + + private IndexEventPublisher mockPublisher; + + + + /******************************************************************************* + ** Set up mock publisher in QuickSearchQBitContext and a QContext with a test + ** QInstance before each test. + *******************************************************************************/ + @BeforeEach + void setUp() + { + mockPublisher = mock(IndexEventPublisher.class); + QuickSearchQBitContext.setPublisher(mockPublisher); + QuickSearchQBitContext.setDiscoveredTables(List.of( + new QuickSearchableTableConfig().withTableName(TABLE_NAME).withPrimaryKeyField(PK_FIELD) + )); + + QInstance qInstance = new QInstance(); + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME) + .withPrimaryKeyField(PK_FIELD) + .withField(new QFieldMetaData(PK_FIELD, QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING))); + qInstance.addBackend(new com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData().withName("testBackend")); + qInstance.setAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS)); + qInstance.getTable(TABLE_NAME).withBackendName("testBackend"); + QContext.init(qInstance, new QSession()); + } + + + + /******************************************************************************* + ** Clear QuickSearchQBitContext and QContext after each test to prevent leakage. + *******************************************************************************/ + @AfterEach + void tearDown() + { + QuickSearchQBitContext.clear(); + QContext.clear(); + } + + + + /******************************************************************************* + ** Test that no-arg constructor completes without throwing. + *******************************************************************************/ + @Test + void testNoArgConstructor_doesNotThrow() + { + assertThatNoException().isThrownBy(QuickSearchPostInsertCustomizer::new); + } + + + + /******************************************************************************* + ** Test that post-insert with valid records builds correct INDEX events and + ** calls publisher.publishIndexEvents. + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Test + void testPostInsert_withRecords_publishesIndexEvents() throws QException + { + QRecord record1 = new QRecord().withValue(PK_FIELD, 1).withValue("name", "Alice"); + QRecord record2 = new QRecord().withValue(PK_FIELD, 2).withValue("name", "Bob"); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME); + + QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer(); + List result = customizer.postInsert(insertInput, List.of(record1, record2)); + + assertThat(result).containsExactly(record1, record2); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mockPublisher).publishIndexEvents(captor.capture()); + + List events = captor.getValue(); + assertThat(events).hasSize(2); + assertThat(events.get(0).getAction()).isEqualTo(IndexEventAction.INDEX); + assertThat(events.get(0).getTableName()).isEqualTo(TABLE_NAME); + assertThat(events.get(0).getRecordId()).isEqualTo("1"); + assertThat(events.get(0).getRecord()).isSameAs(record1); + assertThat(events.get(1).getAction()).isEqualTo(IndexEventAction.INDEX); + assertThat(events.get(1).getRecordId()).isEqualTo("2"); + assertThat(events.get(1).getRecord()).isSameAs(record2); + } + + + + /******************************************************************************* + ** Test that post-insert with empty records list returns input without calling + ** publisher. + *******************************************************************************/ + @Test + void testPostInsert_withEmptyRecords_returnsInputWithoutPublishing() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME); + + QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer(); + List result = customizer.postInsert(insertInput, List.of()); + + assertThat(result).isEmpty(); + verify(mockPublisher, never()).publishIndexEvents(anyList()); + } + + + + /******************************************************************************* + ** Test that post-insert with null records returns null without calling publisher. + *******************************************************************************/ + @Test + void testPostInsert_withNullRecords_returnsNullWithoutPublishing() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME); + + QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer(); + List result = customizer.postInsert(insertInput, null); + + assertThat(result).isNull(); + verify(mockPublisher, never()).publishIndexEvents(anyList()); + } + + + + /******************************************************************************* + ** Test that when publisher is null, records are still returned (no exception). + *******************************************************************************/ + @Test + void testPostInsert_publisherNull_returnsRecordsWithoutThrowing() throws QException + { + QuickSearchQBitContext.setPublisher(null); + + QRecord record = new QRecord().withValue(PK_FIELD, 1); + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME); + + QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer(); + List result = customizer.postInsert(insertInput, List.of(record)); + + assertThat(result).containsExactly(record); + } + + + + /******************************************************************************* + ** Test that when publisher throws, a warning is logged and records are returned + ** (no exception propagated to caller). + *******************************************************************************/ + @Test + void testPostInsert_publisherThrows_returnsRecordsWithoutThrowing() throws QException + { + doThrow(new QException("publish failed")) + .when(mockPublisher).publishIndexEvents(anyList()); + + QRecord record = new QRecord().withValue(PK_FIELD, 1); + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TABLE_NAME); + + QuickSearchPostInsertCustomizer customizer = new QuickSearchPostInsertCustomizer(); + assertThatNoException().isThrownBy(() -> customizer.postInsert(insertInput, List.of(record))); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java new file mode 100644 index 0000000..433bc71 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/customizers/QuickSearchPostUpdateCustomizerTest.java @@ -0,0 +1,223 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.customizers; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.publisher.IndexEvent; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction; +import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + + +/******************************************************************************* + ** Tests for QuickSearchPostUpdateCustomizer. + *******************************************************************************/ +class QuickSearchPostUpdateCustomizerTest +{ + private static final String TABLE_NAME = "orders"; + private static final String PK_FIELD = "id"; + + private IndexEventPublisher mockPublisher; + + + + /******************************************************************************* + ** Set up mock publisher in QuickSearchQBitContext and a QContext with a test + ** QInstance before each test. + *******************************************************************************/ + @BeforeEach + void setUp() + { + mockPublisher = mock(IndexEventPublisher.class); + QuickSearchQBitContext.setPublisher(mockPublisher); + QuickSearchQBitContext.setDiscoveredTables(List.of( + new QuickSearchableTableConfig().withTableName(TABLE_NAME).withPrimaryKeyField(PK_FIELD) + )); + + QInstance qInstance = new QInstance(); + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME) + .withPrimaryKeyField(PK_FIELD) + .withField(new QFieldMetaData(PK_FIELD, QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING))); + qInstance.addBackend(new com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData().withName("testBackend")); + qInstance.setAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS)); + qInstance.getTable(TABLE_NAME).withBackendName("testBackend"); + QContext.init(qInstance, new QSession()); + } + + + + /******************************************************************************* + ** Clear QuickSearchQBitContext and QContext after each test to prevent leakage. + *******************************************************************************/ + @AfterEach + void tearDown() + { + QuickSearchQBitContext.clear(); + QContext.clear(); + } + + + + /******************************************************************************* + ** Test that no-arg constructor completes without throwing. + *******************************************************************************/ + @Test + void testNoArgConstructor_doesNotThrow() + { + assertThatNoException().isThrownBy(QuickSearchPostUpdateCustomizer::new); + } + + + + /******************************************************************************* + ** Test that post-update with valid records builds correct INDEX events and + ** calls publisher.publishIndexEvents. + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Test + void testPostUpdate_withRecords_publishesIndexEvents() throws QException + { + QRecord record1 = new QRecord().withValue(PK_FIELD, 10).withValue("name", "Alice Updated"); + QRecord record2 = new QRecord().withValue(PK_FIELD, 20).withValue("name", "Bob Updated"); + + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TABLE_NAME); + + QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer(); + List result = customizer.postUpdate(updateInput, List.of(record1, record2), Optional.empty()); + + assertThat(result).containsExactly(record1, record2); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mockPublisher).publishIndexEvents(captor.capture()); + + List events = captor.getValue(); + assertThat(events).hasSize(2); + assertThat(events.get(0).getAction()).isEqualTo(IndexEventAction.INDEX); + assertThat(events.get(0).getTableName()).isEqualTo(TABLE_NAME); + assertThat(events.get(0).getRecordId()).isEqualTo("10"); + assertThat(events.get(0).getRecord()).isSameAs(record1); + assertThat(events.get(1).getAction()).isEqualTo(IndexEventAction.INDEX); + assertThat(events.get(1).getRecordId()).isEqualTo("20"); + assertThat(events.get(1).getRecord()).isSameAs(record2); + } + + + + /******************************************************************************* + ** Test that post-update with empty records list returns input without calling + ** publisher. + *******************************************************************************/ + @Test + void testPostUpdate_withEmptyRecords_returnsInputWithoutPublishing() throws QException + { + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TABLE_NAME); + + QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer(); + List result = customizer.postUpdate(updateInput, List.of(), Optional.empty()); + + assertThat(result).isEmpty(); + verify(mockPublisher, never()).publishIndexEvents(anyList()); + } + + + + /******************************************************************************* + ** Test that post-update with null records returns null without calling publisher. + *******************************************************************************/ + @Test + void testPostUpdate_withNullRecords_returnsNullWithoutPublishing() throws QException + { + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TABLE_NAME); + + QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer(); + List result = customizer.postUpdate(updateInput, null, Optional.empty()); + + assertThat(result).isNull(); + verify(mockPublisher, never()).publishIndexEvents(anyList()); + } + + + + /******************************************************************************* + ** Test that when publisher is null, records are still returned (no exception). + *******************************************************************************/ + @Test + void testPostUpdate_publisherNull_returnsRecordsWithoutThrowing() throws QException + { + QuickSearchQBitContext.setPublisher(null); + + QRecord record = new QRecord().withValue(PK_FIELD, 1); + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TABLE_NAME); + + QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer(); + List result = customizer.postUpdate(updateInput, List.of(record), Optional.empty()); + + assertThat(result).containsExactly(record); + } + + + + /******************************************************************************* + ** Test that when publisher throws, a warning is logged and records are returned + ** (no exception propagated to caller). + *******************************************************************************/ + @Test + void testPostUpdate_publisherThrows_returnsRecordsWithoutThrowing() throws QException + { + doThrow(new QException("publish failed")) + .when(mockPublisher).publishIndexEvents(anyList()); + + QRecord record = new QRecord().withValue(PK_FIELD, 1); + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TABLE_NAME); + + QuickSearchPostUpdateCustomizer customizer = new QuickSearchPostUpdateCustomizer(); + assertThatNoException().isThrownBy(() -> customizer.postUpdate(updateInput, List.of(record), Optional.empty())); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRunTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRunTest.java index 31412a0..8a0c4b5 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRunTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexRunTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.model; @@ -27,9 +28,9 @@ import static org.assertj.core.api.Assertions.assertThat; class QuickSearchIndexRunTest { - /*************************************************************************** - ** Test TABLE_NAME constant. - ***************************************************************************/ + /******************************************************************************* + ** Test that TABLE_NAME constant has expected value. + *******************************************************************************/ @Test void testTableNameConstant() { @@ -38,95 +39,92 @@ class QuickSearchIndexRunTest - /*************************************************************************** - ** Test all getters and fluent setters. - ***************************************************************************/ + /******************************************************************************* + ** Test that fluent setters return this (fluent chaining). + *******************************************************************************/ @Test - void testGettersAndSetters() + void testFluentSettersReturnThis() { - Instant start = Instant.now(); - Instant end = start.plusSeconds(60); - - QuickSearchIndexRun run = new QuickSearchIndexRun() - .withId(1) - .withQuickSearchIndexId(100) - .withRunType("FULL") - .withStatus("COMPLETE") - .withStartTime(start) - .withEndTime(end) - .withRecordsProcessed(1000) - .withRecordsIndexed(950) - .withErrorCount(5) - .withErrorMessage("Some warnings"); - - assertThat(run.getId()).isEqualTo(1); - assertThat(run.getQuickSearchIndexId()).isEqualTo(100); - assertThat(run.getRunType()).isEqualTo("FULL"); - assertThat(run.getStatus()).isEqualTo("COMPLETE"); - assertThat(run.getStartTime()).isEqualTo(start); - assertThat(run.getEndTime()).isEqualTo(end); - assertThat(run.getRecordsProcessed()).isEqualTo(1000); - assertThat(run.getRecordsIndexed()).isEqualTo(950); - assertThat(run.getErrorCount()).isEqualTo(5); - assertThat(run.getErrorMessage()).isEqualTo("Some warnings"); - } - - - - /*************************************************************************** - ** Test standard setters. - ***************************************************************************/ - @Test - void testStandardSetters() - { - Instant start = Instant.now(); - Instant end = start.plusSeconds(30); - - QuickSearchIndexRun run = new QuickSearchIndexRun(); - run.setId(2); - run.setQuickSearchIndexId(200); - run.setRunType("BASEPULL"); - run.setStatus("RUNNING"); - run.setStartTime(start); - run.setEndTime(end); - run.setRecordsProcessed(500); - run.setRecordsIndexed(500); - run.setErrorCount(0); - run.setErrorMessage(null); - - assertThat(run.getId()).isEqualTo(2); - assertThat(run.getQuickSearchIndexId()).isEqualTo(200); - assertThat(run.getRunType()).isEqualTo("BASEPULL"); - assertThat(run.getStatus()).isEqualTo("RUNNING"); - assertThat(run.getStartTime()).isEqualTo(start); - assertThat(run.getEndTime()).isEqualTo(end); - assertThat(run.getRecordsProcessed()).isEqualTo(500); - assertThat(run.getRecordsIndexed()).isEqualTo(500); - assertThat(run.getErrorCount()).isEqualTo(0); - assertThat(run.getErrorMessage()).isNull(); - } - - - - /*************************************************************************** - ** Test fluent setters return this. - ***************************************************************************/ - @Test - void testFluentSetters_returnThis() - { - QuickSearchIndexRun run = new QuickSearchIndexRun(); + QuickSearchIndexRun entity = new QuickSearchIndexRun(); Instant now = Instant.now(); - assertThat(run.withId(1)).isSameAs(run); - assertThat(run.withQuickSearchIndexId(1)).isSameAs(run); - assertThat(run.withRunType("FULL")).isSameAs(run); - assertThat(run.withStatus("RUNNING")).isSameAs(run); - assertThat(run.withStartTime(now)).isSameAs(run); - assertThat(run.withEndTime(now)).isSameAs(run); - assertThat(run.withRecordsProcessed(0)).isSameAs(run); - assertThat(run.withRecordsIndexed(0)).isSameAs(run); - assertThat(run.withErrorCount(0)).isSameAs(run); - assertThat(run.withErrorMessage(null)).isSameAs(run); + assertThat(entity.withId(1)).isSameAs(entity); + assertThat(entity.withQuickSearchIndexId(10)).isSameAs(entity); + assertThat(entity.withRunType("basepull")).isSameAs(entity); + assertThat(entity.withStatus("running")).isSameAs(entity); + assertThat(entity.withStartTime(now)).isSameAs(entity); + assertThat(entity.withEndTime(now)).isSameAs(entity); + assertThat(entity.withRecordsProcessed(500)).isSameAs(entity); + assertThat(entity.withRecordsIndexed(490)).isSameAs(entity); + assertThat(entity.withErrorCount(10)).isSameAs(entity); + assertThat(entity.withErrorMessage("something failed")).isSameAs(entity); + assertThat(entity.withCreateDate(now)).isSameAs(entity); + assertThat(entity.withModifyDate(now)).isSameAs(entity); + } + + + + /******************************************************************************* + ** Test that getters return values set via fluent setters. + *******************************************************************************/ + @Test + void testGettersReturnSetValues() + { + Instant startTime = Instant.parse("2024-03-01T09:00:00Z"); + Instant endTime = Instant.parse("2024-03-01T09:05:00Z"); + Instant createDate = Instant.parse("2024-03-01T09:00:00Z"); + Instant modifyDate = Instant.parse("2024-03-01T09:05:30Z"); + + QuickSearchIndexRun entity = new QuickSearchIndexRun() + .withId(7) + .withQuickSearchIndexId(3) + .withRunType("fullReindex") + .withStatus("completed") + .withStartTime(startTime) + .withEndTime(endTime) + .withRecordsProcessed(1000) + .withRecordsIndexed(998) + .withErrorCount(2) + .withErrorMessage("2 records failed validation") + .withCreateDate(createDate) + .withModifyDate(modifyDate); + + assertThat(entity.getId()).isEqualTo(7); + assertThat(entity.getQuickSearchIndexId()).isEqualTo(3); + assertThat(entity.getRunType()).isEqualTo("fullReindex"); + assertThat(entity.getStatus()).isEqualTo("completed"); + assertThat(entity.getStartTime()).isEqualTo(startTime); + assertThat(entity.getEndTime()).isEqualTo(endTime); + assertThat(entity.getRecordsProcessed()).isEqualTo(1000); + assertThat(entity.getRecordsIndexed()).isEqualTo(998); + assertThat(entity.getErrorCount()).isEqualTo(2); + assertThat(entity.getErrorMessage()).isEqualTo("2 records failed validation"); + assertThat(entity.getCreateDate()).isEqualTo(createDate); + assertThat(entity.getModifyDate()).isEqualTo(modifyDate); + } + + + + /******************************************************************************* + ** Test that all fields default to null on a new instance. + *******************************************************************************/ + @Test + void testDefaultsAreNull() + { + QuickSearchIndexRun entity = new QuickSearchIndexRun(); + + assertThat(entity.getId()).isNull(); + assertThat(entity.getQuickSearchIndexId()).isNull(); + assertThat(entity.getRunType()).isNull(); + assertThat(entity.getStatus()).isNull(); + assertThat(entity.getStartTime()).isNull(); + assertThat(entity.getEndTime()).isNull(); + assertThat(entity.getRecordsProcessed()).isNull(); + assertThat(entity.getRecordsIndexed()).isNull(); + assertThat(entity.getErrorCount()).isNull(); + assertThat(entity.getErrorMessage()).isNull(); + assertThat(entity.getCreateDate()).isNull(); + assertThat(entity.getModifyDate()).isNull(); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexTest.java index e355b4f..515803c 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/model/QuickSearchIndexTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.model; @@ -27,9 +28,9 @@ import static org.assertj.core.api.Assertions.assertThat; class QuickSearchIndexTest { - /*************************************************************************** - ** Test TABLE_NAME constant. - ***************************************************************************/ + /******************************************************************************* + ** Test that TABLE_NAME constant has expected value. + *******************************************************************************/ @Test void testTableNameConstant() { @@ -38,98 +39,92 @@ class QuickSearchIndexTest - /*************************************************************************** - ** Test all getters and fluent setters. - ***************************************************************************/ + /******************************************************************************* + ** Test that fluent setters return this (fluent chaining). + *******************************************************************************/ @Test - void testGettersAndSetters() + void testFluentSettersReturnThis() { + QuickSearchIndex entity = new QuickSearchIndex(); Instant now = Instant.now(); - QuickSearchIndex index = new QuickSearchIndex() - .withId(1) - .withTableName("orders") - .withIsEnabled(true) - .withBasepullIntervalMinutes(5) - .withBasepullTimestampField("modifyDate") + assertThat(entity.withId(1)).isSameAs(entity); + assertThat(entity.withTableName("myTable")).isSameAs(entity); + assertThat(entity.withEnabled(true)).isSameAs(entity); + assertThat(entity.withBasepullIntervalMinutes(60)).isSameAs(entity); + assertThat(entity.withBasepullTimestampField("modifyDate")).isSameAs(entity); + assertThat(entity.withSearchableFieldsJson("[]")).isSameAs(entity); + assertThat(entity.withLastBasepullTime(now)).isSameAs(entity); + assertThat(entity.withLastFullReindexTime(now)).isSameAs(entity); + assertThat(entity.withRecordCount(100)).isSameAs(entity); + assertThat(entity.withStatus("active")).isSameAs(entity); + assertThat(entity.withCreateDate(now)).isSameAs(entity); + assertThat(entity.withModifyDate(now)).isSameAs(entity); + } + + + + /******************************************************************************* + ** Test that getters return values set via fluent setters. + *******************************************************************************/ + @Test + void testGettersReturnSetValues() + { + Instant basepullTime = Instant.parse("2024-01-15T10:00:00Z"); + Instant reindexTime = Instant.parse("2024-01-16T12:00:00Z"); + Instant createDate = Instant.parse("2024-01-01T00:00:00Z"); + Instant modifyDate = Instant.parse("2024-01-17T08:30:00Z"); + + QuickSearchIndex entity = new QuickSearchIndex() + .withId(42) + .withTableName("orderTable") + .withEnabled(true) + .withBasepullIntervalMinutes(30) + .withBasepullTimestampField("lastModified") .withSearchableFieldsJson("[\"name\",\"description\"]") - .withLastFullIndexTime(now) - .withLastBasepullTime(now) - .withIndexedRecordCount(100) - .withCreateDate(now) - .withModifyDate(now); + .withLastBasepullTime(basepullTime) + .withLastFullReindexTime(reindexTime) + .withRecordCount(9999) + .withStatus("active") + .withCreateDate(createDate) + .withModifyDate(modifyDate); - assertThat(index.getId()).isEqualTo(1); - assertThat(index.getTableName()).isEqualTo("orders"); - assertThat(index.getIsEnabled()).isTrue(); - assertThat(index.getBasepullIntervalMinutes()).isEqualTo(5); - assertThat(index.getBasepullTimestampField()).isEqualTo("modifyDate"); - assertThat(index.getSearchableFieldsJson()).isEqualTo("[\"name\",\"description\"]"); - assertThat(index.getLastFullIndexTime()).isEqualTo(now); - assertThat(index.getLastBasepullTime()).isEqualTo(now); - assertThat(index.getIndexedRecordCount()).isEqualTo(100); - assertThat(index.getCreateDate()).isEqualTo(now); - assertThat(index.getModifyDate()).isEqualTo(now); + assertThat(entity.getId()).isEqualTo(42); + assertThat(entity.getTableName()).isEqualTo("orderTable"); + assertThat(entity.getEnabled()).isTrue(); + assertThat(entity.getBasepullIntervalMinutes()).isEqualTo(30); + assertThat(entity.getBasepullTimestampField()).isEqualTo("lastModified"); + assertThat(entity.getSearchableFieldsJson()).isEqualTo("[\"name\",\"description\"]"); + assertThat(entity.getLastBasepullTime()).isEqualTo(basepullTime); + assertThat(entity.getLastFullReindexTime()).isEqualTo(reindexTime); + assertThat(entity.getRecordCount()).isEqualTo(9999); + assertThat(entity.getStatus()).isEqualTo("active"); + assertThat(entity.getCreateDate()).isEqualTo(createDate); + assertThat(entity.getModifyDate()).isEqualTo(modifyDate); } - /*************************************************************************** - ** Test standard setters. - ***************************************************************************/ + /******************************************************************************* + ** Test that all fields default to null on a new instance. + *******************************************************************************/ @Test - void testStandardSetters() + void testDefaultsAreNull() { - Instant now = Instant.now(); + QuickSearchIndex entity = new QuickSearchIndex(); - QuickSearchIndex index = new QuickSearchIndex(); - index.setId(2); - index.setTableName("customers"); - index.setIsEnabled(false); - index.setBasepullIntervalMinutes(10); - index.setBasepullTimestampField("updatedAt"); - index.setSearchableFieldsJson("[]"); - index.setLastFullIndexTime(now); - index.setLastBasepullTime(now); - index.setIndexedRecordCount(50); - index.setCreateDate(now); - index.setModifyDate(now); - - assertThat(index.getId()).isEqualTo(2); - assertThat(index.getTableName()).isEqualTo("customers"); - assertThat(index.getIsEnabled()).isFalse(); - assertThat(index.getBasepullIntervalMinutes()).isEqualTo(10); - assertThat(index.getBasepullTimestampField()).isEqualTo("updatedAt"); - assertThat(index.getSearchableFieldsJson()).isEqualTo("[]"); - assertThat(index.getLastFullIndexTime()).isEqualTo(now); - assertThat(index.getLastBasepullTime()).isEqualTo(now); - assertThat(index.getIndexedRecordCount()).isEqualTo(50); - assertThat(index.getCreateDate()).isEqualTo(now); - assertThat(index.getModifyDate()).isEqualTo(now); - } - - - - /*************************************************************************** - ** Test fluent setters return this. - ***************************************************************************/ - @Test - void testFluentSetters_returnThis() - { - QuickSearchIndex index = new QuickSearchIndex(); - Instant now = Instant.now(); - - assertThat(index.withId(1)).isSameAs(index); - assertThat(index.withTableName("t")).isSameAs(index); - assertThat(index.withIsEnabled(true)).isSameAs(index); - assertThat(index.withBasepullIntervalMinutes(5)).isSameAs(index); - assertThat(index.withBasepullTimestampField("f")).isSameAs(index); - assertThat(index.withSearchableFieldsJson("[]")).isSameAs(index); - assertThat(index.withLastFullIndexTime(now)).isSameAs(index); - assertThat(index.withLastBasepullTime(now)).isSameAs(index); - assertThat(index.withIndexedRecordCount(0)).isSameAs(index); - assertThat(index.withCreateDate(now)).isSameAs(index); - assertThat(index.withModifyDate(now)).isSameAs(index); + assertThat(entity.getId()).isNull(); + assertThat(entity.getTableName()).isNull(); + assertThat(entity.getEnabled()).isNull(); + assertThat(entity.getBasepullIntervalMinutes()).isNull(); + assertThat(entity.getBasepullTimestampField()).isNull(); + assertThat(entity.getSearchableFieldsJson()).isNull(); + assertThat(entity.getLastBasepullTime()).isNull(); + assertThat(entity.getLastFullReindexTime()).isNull(); + assertThat(entity.getRecordCount()).isNull(); + assertThat(entity.getStatus()).isNull(); + assertThat(entity.getCreateDate()).isNull(); + assertThat(entity.getModifyDate()).isNull(); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResultTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResultTest.java new file mode 100644 index 0000000..503636b --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/BulkIndexResultTest.java @@ -0,0 +1,137 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.opensearch; + + +import java.util.List; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Tests for BulkIndexResult. + *******************************************************************************/ +class BulkIndexResultTest +{ + + /******************************************************************************* + ** Test default initial values. + *******************************************************************************/ + @Test + void testDefaults() + { + BulkIndexResult result = new BulkIndexResult(); + assertThat(result.getSuccessCount()).isEqualTo(0); + assertThat(result.getFailureCount()).isEqualTo(0); + assertThat(result.getErrors()).isEmpty(); + } + + /******************************************************************************* + ** Test addSuccess() increments successCount. + *******************************************************************************/ + @Test + void testAddSuccess() + { + BulkIndexResult result = new BulkIndexResult(); + result.addSuccess(); + result.addSuccess(); + result.addSuccess(); + + assertThat(result.getSuccessCount()).isEqualTo(3); + assertThat(result.getFailureCount()).isEqualTo(0); + } + + /******************************************************************************* + ** Test addFailure() increments failureCount and appends error message. + *******************************************************************************/ + @Test + void testAddFailure() + { + BulkIndexResult result = new BulkIndexResult(); + result.addFailure("Connection timeout"); + result.addFailure("Document too large"); + + assertThat(result.getFailureCount()).isEqualTo(2); + assertThat(result.getErrors()).containsExactly("Connection timeout", "Document too large"); + assertThat(result.getSuccessCount()).isEqualTo(0); + } + + /******************************************************************************* + ** Test isFullySuccessful() returns true when no failures. + *******************************************************************************/ + @Test + void testIsFullySuccessfulNoFailures() + { + BulkIndexResult result = new BulkIndexResult(); + result.addSuccess(); + result.addSuccess(); + + assertThat(result.isFullySuccessful()).isTrue(); + } + + /******************************************************************************* + ** Test isFullySuccessful() returns false when failures exist. + *******************************************************************************/ + @Test + void testIsFullySuccessfulWithFailures() + { + BulkIndexResult result = new BulkIndexResult(); + result.addSuccess(); + result.addFailure("Oops"); + + assertThat(result.isFullySuccessful()).isFalse(); + } + + /******************************************************************************* + ** Test isFullySuccessful() on empty (no ops yet) result. + *******************************************************************************/ + @Test + void testIsFullySuccessfulEmpty() + { + BulkIndexResult result = new BulkIndexResult(); + assertThat(result.isFullySuccessful()).isTrue(); + } + + /******************************************************************************* + ** Test fluent setters return this. + *******************************************************************************/ + @Test + void testFluentSettersReturnThis() + { + BulkIndexResult result = new BulkIndexResult(); + assertThat(result.withSuccessCount(5)).isSameAs(result); + assertThat(result.withFailureCount(2)).isSameAs(result); + assertThat(result.withErrors(List.of("err1"))).isSameAs(result); + } + + /******************************************************************************* + ** Test fluent setters set values correctly. + *******************************************************************************/ + @Test + void testFluentSettersValues() + { + BulkIndexResult result = new BulkIndexResult() + .withSuccessCount(10) + .withFailureCount(3) + .withErrors(List.of("e1", "e2", "e3")); + + assertThat(result.getSuccessCount()).isEqualTo(10); + assertThat(result.getFailureCount()).isEqualTo(3); + assertThat(result.getErrors()).containsExactly("e1", "e2", "e3"); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocumentTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocumentTest.java index 807d9c7..3590df4 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocumentTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/OpenSearchDocumentTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,10 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.opensearch; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -27,76 +30,70 @@ import static org.assertj.core.api.Assertions.assertThat; class OpenSearchDocumentTest { - /*************************************************************************** - ** Test document ID generation. - ***************************************************************************/ + /******************************************************************************* + ** Test that fluent setters return this and getters return values. + *******************************************************************************/ @Test - void testGetDocumentId_returnsCompositeKey() + void testFluentSetters() + { + Instant now = Instant.now(); + Map fieldValues = new HashMap<>(); + fieldValues.put("name", "Alice"); + + OpenSearchDocument doc = new OpenSearchDocument() + .withSourceTable("users") + .withRecordId("7") + .withRecordLabel("Alice") + .withSearchableText("Alice Smith admin user") + .withIndexedAt(now) + .withFieldValues(fieldValues); + + assertThat(doc.getSourceTable()).isEqualTo("users"); + assertThat(doc.getRecordId()).isEqualTo("7"); + assertThat(doc.getRecordLabel()).isEqualTo("Alice"); + assertThat(doc.getSearchableText()).isEqualTo("Alice Smith admin user"); + assertThat(doc.getIndexedAt()).isEqualTo(now); + assertThat(doc.getFieldValues()).containsEntry("name", "Alice"); + } + + /******************************************************************************* + ** Test that withX methods return this. + *******************************************************************************/ + @Test + void testWithReturnsThis() + { + OpenSearchDocument doc = new OpenSearchDocument(); + assertThat(doc.withSourceTable("t")).isSameAs(doc); + assertThat(doc.withRecordId("1")).isSameAs(doc); + assertThat(doc.withRecordLabel("l")).isSameAs(doc); + assertThat(doc.withSearchableText("text")).isSameAs(doc); + assertThat(doc.withIndexedAt(Instant.now())).isSameAs(doc); + assertThat(doc.withFieldValues(Map.of())).isSameAs(doc); + } + + /******************************************************************************* + ** Test getDocumentId() returns "sourceTable:recordId". + *******************************************************************************/ + @Test + void testGetDocumentId() { OpenSearchDocument doc = new OpenSearchDocument() .withSourceTable("orders") - .withRecordId("12345"); + .withRecordId("42"); - assertThat(doc.getDocumentId()).isEqualTo("orders:12345"); + assertThat(doc.getDocumentId()).isEqualTo("orders:42"); } - - - /*************************************************************************** - ** Test document ID with null source table. - ***************************************************************************/ + /******************************************************************************* + ** Test getDocumentId() when sourceTable is null. + *******************************************************************************/ @Test - void testGetDocumentId_nullSourceTable_handlesGracefully() + void testGetDocumentIdWithNullSourceTable() { OpenSearchDocument doc = new OpenSearchDocument() - .withSourceTable(null) - .withRecordId("12345"); + .withRecordId("10"); - assertThat(doc.getDocumentId()).isEqualTo("null:12345"); - } - - - - /*************************************************************************** - ** Test all getters and setters. - ***************************************************************************/ - @Test - void testGettersAndSetters() - { - Instant now = Instant.now(); - - OpenSearchDocument doc = new OpenSearchDocument() - .withSourceTable("customers") - .withRecordId("999") - .withRecordLabel("Customer #999") - .withSearchableText("john doe acme corp") - .withIndexedAt(now); - - assertThat(doc.getSourceTable()).isEqualTo("customers"); - assertThat(doc.getRecordId()).isEqualTo("999"); - assertThat(doc.getRecordLabel()).isEqualTo("Customer #999"); - assertThat(doc.getSearchableText()).isEqualTo("john doe acme corp"); - assertThat(doc.getIndexedAt()).isEqualTo(now); - } - - - - /*************************************************************************** - ** Test fluent setters return this. - ***************************************************************************/ - @Test - void testFluentSetters_returnThis() - { - OpenSearchDocument doc = new OpenSearchDocument(); - - OpenSearchDocument result = doc - .withSourceTable("test") - .withRecordId("1") - .withRecordLabel("label") - .withSearchableText("text") - .withIndexedAt(Instant.now()); - - assertThat(result).isSameAs(doc); + assertThat(doc.getDocumentId()).isEqualTo("null:10"); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClientTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClientTest.java index 03dd20e..2567f59 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClientTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/opensearch/QuickSearchOpenSearchClientTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.opensearch; -import java.time.Instant; import java.util.List; -import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -27,245 +27,238 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* ** Tests for QuickSearchOpenSearchClient. + ** + ** All tests use localhost:19876 where nothing is listening, so they exercise + ** constructor success, error-handling paths, and lifecycle behaviour without + ** requiring a live OpenSearch instance. *******************************************************************************/ class QuickSearchOpenSearchClientTest { + private static final String TEST_HOST = "localhost"; + private static final int TEST_PORT = 19876; + private static final String TEST_INDEX = "test-index"; - /*************************************************************************** - ** Test constructor with valid config creates client. - ***************************************************************************/ - @Test - void testConstructor_validConfig_createsClient() throws Exception + + + /******************************************************************************* + ** Helper to build a minimal plain-HTTP config. + *******************************************************************************/ + private QuickSearchQBitConfig plainConfig() { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); + return (new QuickSearchQBitConfig() + .withOpensearchHost(TEST_HOST) + .withOpensearchPort(TEST_PORT) + .withOpensearchIndexName(TEST_INDEX) + .withUseSsl(false)); + } - assertThat(client).isNotNull(); + + + /******************************************************************************* + ** Constructor succeeds for plain HTTP (no credentials). + *******************************************************************************/ + @Test + void testConstructor_plainHttp_succeeds() throws QException + { + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); client.close(); } - /*************************************************************************** - ** Test constructor with SSL enabled. - ***************************************************************************/ + /******************************************************************************* + ** Constructor succeeds when useSsl is true. + *******************************************************************************/ @Test - void testConstructor_withSsl_createsClient() throws Exception + void testConstructor_withSsl_succeeds() throws QException { - QuickSearchQBitConfig config = createTestConfig() - .withUseSsl(true) - .withOpensearchPort(443); - + QuickSearchQBitConfig config = plainConfig().withUseSsl(true); QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThat(client).isNotNull(); client.close(); } - /*************************************************************************** - ** Test constructor with credentials. - ***************************************************************************/ + /******************************************************************************* + ** Constructor succeeds when both username and password are provided. + *******************************************************************************/ @Test - void testConstructor_withCredentials_createsClient() throws Exception + void testConstructor_withCredentials_succeeds() throws QException { - QuickSearchQBitConfig config = createTestConfig() - .withOpensearchUsername("testuser") - .withOpensearchPassword("testpass"); - + QuickSearchQBitConfig config = plainConfig() + .withOpensearchUsername("admin") + .withOpensearchPassword("secret"); QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThat(client).isNotNull(); client.close(); } - /*************************************************************************** - ** Test ensureIndexExists throws when server unavailable. - ***************************************************************************/ + /******************************************************************************* + ** close() does not throw any exception. + *******************************************************************************/ @Test - void testEnsureIndexExists_serverUnavailable_throws() throws Exception + void testClose_doesNotThrow() throws QException { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThatThrownBy(() -> client.ensureIndexExists()) - .isInstanceOf(QException.class) - .hasMessageContaining("Failed to ensure index exists"); - + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); client.close(); } - /*************************************************************************** - ** Test indexDocument throws when server unavailable. - ***************************************************************************/ + /******************************************************************************* + ** close() is idempotent: calling it twice must not throw. + *******************************************************************************/ @Test - void testIndexDocument_serverUnavailable_throws() throws Exception + void testClose_idempotent() throws QException { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); + client.close(); + client.close(); + } - OpenSearchDocument document = new OpenSearchDocument() - .withSourceTable("test") + + + /******************************************************************************* + ** indexDocuments with a null list returns an empty, fully-successful result. + *******************************************************************************/ + @Test + void testIndexDocuments_nullList_returnsEmptyResult() throws QException + { + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); + + try + { + BulkIndexResult result = client.indexDocuments(null, 100); + + assertThat(result).isNotNull(); + assertThat(result.getSuccessCount()).isEqualTo(0); + assertThat(result.getFailureCount()).isEqualTo(0); + assertThat(result.isFullySuccessful()).isTrue(); + } + finally + { + client.close(); + } + } + + + + /******************************************************************************* + ** indexDocuments with an empty list returns an empty, fully-successful result. + *******************************************************************************/ + @Test + void testIndexDocuments_emptyList_returnsEmptyResult() throws QException + { + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); + + try + { + BulkIndexResult result = client.indexDocuments(List.of(), 100); + + assertThat(result).isNotNull(); + assertThat(result.getSuccessCount()).isEqualTo(0); + assertThat(result.getFailureCount()).isEqualTo(0); + assertThat(result.isFullySuccessful()).isTrue(); + } + finally + { + client.close(); + } + } + + + + /******************************************************************************* + ** ensureIndexExists throws QException when the server is unavailable. + *******************************************************************************/ + @Test + void testEnsureIndexExists_serverUnavailable_throwsQException() throws QException + { + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); + + try + { + assertThatThrownBy(client::ensureIndexExists) + .isInstanceOf(QException.class) + .hasMessageContaining(TEST_INDEX); + } + finally + { + client.close(); + } + } + + + + /******************************************************************************* + ** indexDocument throws QException when the server is unavailable. + *******************************************************************************/ + @Test + void testIndexDocument_serverUnavailable_throwsQException() throws QException + { + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); + + OpenSearchDocument doc = new OpenSearchDocument() + .withSourceTable("orders") .withRecordId("1") - .withSearchableText("test content") - .withIndexedAt(Instant.now()); + .withRecordLabel("Order #1") + .withSearchableText("test order"); - assertThatThrownBy(() -> client.indexDocument(document)) - .isInstanceOf(Exception.class); - - client.close(); + try + { + assertThatThrownBy(() -> client.indexDocument(doc)) + .isInstanceOf(QException.class) + .hasMessageContaining("orders:1"); + } + finally + { + client.close(); + } } - /*************************************************************************** - ** Test indexDocuments with empty list does nothing. - ***************************************************************************/ + /******************************************************************************* + ** search throws QException when the server is unavailable. + *******************************************************************************/ @Test - void testIndexDocuments_emptyList_doesNothing() throws Exception + void testSearch_serverUnavailable_throwsQException() throws QException { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); - // Should not throw even though server is unavailable - client.indexDocuments(List.of()); - - client.close(); + try + { + assertThatThrownBy(() -> client.search("hello", null, 10, 0, List.of())) + .isInstanceOf(QException.class) + .hasMessageContaining("hello"); + } + finally + { + client.close(); + } } - /*************************************************************************** - ** Test indexDocuments throws when server unavailable. - ***************************************************************************/ + /******************************************************************************* + ** deleteDocument throws QException when the server is unavailable. + *******************************************************************************/ @Test - void testIndexDocuments_serverUnavailable_throws() throws Exception + void testDeleteDocument_serverUnavailable_throwsQException() throws QException { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); + QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(plainConfig()); - List documents = List.of( - new OpenSearchDocument() - .withSourceTable("test") - .withRecordId("1") - .withSearchableText("test") - .withIndexedAt(Instant.now()) - ); - - assertThatThrownBy(() -> client.indexDocuments(documents)) - .isInstanceOf(Exception.class); - - client.close(); - } - - - - /*************************************************************************** - ** Test deleteDocument throws when server unavailable. - ***************************************************************************/ - @Test - void testDeleteDocument_serverUnavailable_throws() throws Exception - { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThatThrownBy(() -> client.deleteDocument("test", "1")) - .isInstanceOf(QException.class) - .hasMessageContaining("Failed to delete document"); - - client.close(); - } - - - - /*************************************************************************** - ** Test deleteDocumentsForTable throws when server unavailable. - ***************************************************************************/ - @Test - void testDeleteDocumentsForTable_serverUnavailable_throws() throws Exception - { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThatThrownBy(() -> client.deleteDocumentsForTable("test")) - .isInstanceOf(QException.class) - .hasMessageContaining("Failed to delete documents for table"); - - client.close(); - } - - - - /*************************************************************************** - ** Test search with no table filter throws when server unavailable. - ***************************************************************************/ - @Test - void testSearch_noTableFilter_serverUnavailable_throws() throws Exception - { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThatThrownBy(() -> client.search("test query", 10)) - .isInstanceOf(QException.class) - .hasMessageContaining("Failed to search documents"); - - client.close(); - } - - - - /*************************************************************************** - ** Test search with table filter throws when server unavailable. - ***************************************************************************/ - @Test - void testSearch_withTableFilter_serverUnavailable_throws() throws Exception - { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThatThrownBy(() -> client.search("test query", "orders", 10)) - .isInstanceOf(QException.class) - .hasMessageContaining("Failed to search documents"); - - client.close(); - } - - - - /*************************************************************************** - ** Test search with null limit uses default. - ***************************************************************************/ - @Test - void testSearch_nullLimit_serverUnavailable_throws() throws Exception - { - QuickSearchQBitConfig config = createTestConfig(); - QuickSearchOpenSearchClient client = new QuickSearchOpenSearchClient(config); - - assertThatThrownBy(() -> client.search("test query", null, null)) - .isInstanceOf(QException.class) - .hasMessageContaining("Failed to search documents"); - - client.close(); - } - - - - /*************************************************************************** - ** Helper to create test configuration. - ***************************************************************************/ - private QuickSearchQBitConfig createTestConfig() - { - return new QuickSearchQBitConfig() - .withBackendName("memory") - .withOpensearchHost("localhost") - .withOpensearchPort(9200) - .withOpensearchIndexName("test-quick-search") - .withUseSsl(false); + try + { + assertThatThrownBy(() -> client.deleteDocument("orders", "1")) + .isInstanceOf(QException.class) + .hasMessageContaining("orders:1"); + } + finally + { + client.close(); + } } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java new file mode 100644 index 0000000..d51c6b5 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java @@ -0,0 +1,419 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.kingsrook.qbits.quicksearch.processes; + + +import java.time.Instant; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; +import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +/******************************************************************************* + ** Tests for BasepullIndexStep. + ** + ** Uses the in-memory QInstance and mock OpenSearch client from BaseQuickSearchTest. + *******************************************************************************/ +class BasepullIndexStepTest extends BaseQuickSearchTest +{ + + /******************************************************************************* + ** Insert records into the testEntity source table. + *******************************************************************************/ + private void insertTestEntities(int count) throws QException + { + List records = new java.util.ArrayList<>(); + for(int i = 1; i <= count; i++) + { + records.add(new QRecord() + .withValue("name", "Entity " + i) + .withValue("description", "Description " + i)); + } + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TEST_ENTITY_TABLE); + insertInput.setRecords(records); + + new InsertAction().execute(insertInput); + } + + + + /******************************************************************************* + ** Insert a QuickSearchIndex row with the given parameters. + *******************************************************************************/ + private QRecord insertIndexRow(String tableName, Boolean enabled, Instant lastBasepullTime, Integer intervalMinutes) throws QException + { + QRecord record = new QRecord() + .withValue("tableName", tableName) + .withValue("enabled", enabled) + .withValue("basepullIntervalMinutes", intervalMinutes) + .withValue("basepullTimestampField", "modifyDate") + .withValue("lastBasepullTime", lastBasepullTime) + .withValue("status", "ACTIVE"); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(QuickSearchIndex.TABLE_NAME); + insertInput.setRecords(List.of(record)); + + InsertInput input = new InsertInput(); + input.setTableName(QuickSearchIndex.TABLE_NAME); + input.setRecords(List.of(record)); + + return (new InsertAction().execute(input).getRecords().get(0)); + } + + + + /******************************************************************************* + ** Query all quickSearchIndexRun rows. + *******************************************************************************/ + private List queryAllRunRecords() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndexRun.TABLE_NAME); + queryInput.setFilter(new QQueryFilter()); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (queryOutput.getRecords()); + } + + + + /******************************************************************************* + ** Query all quickSearchIndex rows. + *******************************************************************************/ + private List queryAllIndexRows() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndex.TABLE_NAME); + queryInput.setFilter(new QQueryFilter()); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (queryOutput.getRecords()); + } + + + + /******************************************************************************* + ** Build a BulkIndexResult representing all successes for N documents. + *******************************************************************************/ + private BulkIndexResult allSuccessResult(int count) + { + BulkIndexResult result = new BulkIndexResult(); + for(int i = 0; i < count; i++) + { + result.addSuccess(); + } + return (result); + } + + + + /******************************************************************************* + ** Test: no discovered tables, no quickSearchIndex rows - step completes + ** without calling the OpenSearch client. + *******************************************************************************/ + @Test + void testNoEnabledIndexes_noClientCalls() throws QException + { + //////////////////////////////////////////////////// + // Override: no discovered tables, so no index // + // rows will be lazy-created or processed. // + //////////////////////////////////////////////////// + QuickSearchQBitContext.setDiscoveredTables(List.of()); + + QuickSearchOpenSearchClient mockClient = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + + BasepullIndexStep step = new BasepullIndexStep(); + step.run(new RunBackendStepInput(), new RunBackendStepOutput()); + + verify(mockClient, never()).indexDocuments(any(), anyInt()); + } + + + + /******************************************************************************* + ** Test: one enabled index with lastBasepullTime=null triggers a full basepull. + ** Verifies indexDocuments is called and a COMPLETED run record is created. + *******************************************************************************/ + @Test + void testOneEnabledIndex_dueForBasepull_indexesAndCompletesRun() throws QException + { + //////////////////////////////////////////////////// + // Insert some source records // + //////////////////////////////////////////////////// + insertTestEntities(3); + + //////////////////////////////////////////////////// + // Insert a QuickSearchIndex row (never run). // + // Keep discoveredTables set so tableConfig is // + // available during document building. // + //////////////////////////////////////////////////// + insertIndexRow(TEST_ENTITY_TABLE, true, null, 5); + + //////////////////////////////////////////////////// + // Configure mock client to return success // + //////////////////////////////////////////////////// + QuickSearchOpenSearchClient mockClient = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(allSuccessResult(3)); + + //////////////////////////////////////////////////// + // Run the step - discoveredTables is still set // + // from baseSetUp so ensureIndexRowExists will // + // just find the existing row (no-op). // + //////////////////////////////////////////////////// + BasepullIndexStep step = new BasepullIndexStep(); + step.run(new RunBackendStepInput(), new RunBackendStepOutput()); + + //////////////////////////////////////////////////// + // Verify client was called // + //////////////////////////////////////////////////// + verify(mockClient).indexDocuments(any(), anyInt()); + + //////////////////////////////////////////////////// + // Verify a run record was created with COMPLETED // + //////////////////////////////////////////////////// + List runs = queryAllRunRecords(); + assertThat(runs).hasSize(1); + assertThat(runs.get(0).getValueString("status")).isEqualTo("COMPLETED"); + assertThat(runs.get(0).getValueString("runType")).isEqualTo("BASEPULL"); + assertThat(runs.get(0).getValueInteger("recordsProcessed")).isEqualTo(3); + } + + + + /******************************************************************************* + ** Test: index with a very recent lastBasepullTime and 60-minute interval is + ** skipped (not yet due). + *******************************************************************************/ + @Test + void testIndexNotYetDue_skipped() throws QException + { + //////////////////////////////////////////////////// + // Insert source records // + //////////////////////////////////////////////////// + insertTestEntities(2); + + //////////////////////////////////////////////////// + // lastBasepullTime = now, interval = 60 min. // + // ensureIndexRowExists will find the existing // + // row so no duplicate is created. // + //////////////////////////////////////////////////// + insertIndexRow(TEST_ENTITY_TABLE, true, Instant.now(), 60); + + QuickSearchOpenSearchClient mockClient = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + + BasepullIndexStep step = new BasepullIndexStep(); + step.run(new RunBackendStepInput(), new RunBackendStepOutput()); + + //////////////////////////////////////////////////// + // Client should never be called // + //////////////////////////////////////////////////// + verify(mockClient, never()).indexDocuments(any(), anyInt()); + + //////////////////////////////////////////////////// + // No run records created // + //////////////////////////////////////////////////// + assertThat(queryAllRunRecords()).isEmpty(); + } + + + + /******************************************************************************* + ** Test: index is due but no source records match the filter. + ** Verifies a run record is created with 0 recordsProcessed. + *******************************************************************************/ + @Test + void testDueForBasepull_noMatchingRecords_zeroCountRun() throws QException + { + //////////////////////////////////////////////////// + // No source records // + //////////////////////////////////////////////////// + + //////////////////////////////////////////////////// + // Index row is due (null lastBasepullTime). // + // discoveredTables stays set from baseSetUp. // + //////////////////////////////////////////////////// + insertIndexRow(TEST_ENTITY_TABLE, true, null, 5); + + QuickSearchOpenSearchClient mockClient = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + + BasepullIndexStep step = new BasepullIndexStep(); + step.run(new RunBackendStepInput(), new RunBackendStepOutput()); + + //////////////////////////////////////////////////// + // Run record should exist with 0 processed // + //////////////////////////////////////////////////// + List runs = queryAllRunRecords(); + assertThat(runs).hasSize(1); + assertThat(runs.get(0).getValueInteger("recordsProcessed")).isEqualTo(0); + assertThat(runs.get(0).getValueString("status")).isEqualTo("COMPLETED"); + + //////////////////////////////////////////////////// + // Client never called (no documents to index) // + //////////////////////////////////////////////////// + verify(mockClient, never()).indexDocuments(any(), anyInt()); + } + + + + /******************************************************************************* + ** Test: discovered tables are present but no quickSearchIndex rows exist. + ** Running the step should lazy-create the index rows. + *******************************************************************************/ + @Test + void testLazyIndexRowCreation_createsRowsForDiscoveredTables() throws QException + { + //////////////////////////////////////////////////// + // Sanity: no index rows yet // + //////////////////////////////////////////////////// + assertThat(queryAllIndexRows()).isEmpty(); + + //////////////////////////////////////////////////// + // BaseQuickSearchTest sets up one discovered // + // table (testEntity); run step with no pre- // + // existing index rows. // + //////////////////////////////////////////////////// + QuickSearchOpenSearchClient mockClient = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(allSuccessResult(0)); + + BasepullIndexStep step = new BasepullIndexStep(); + step.run(new RunBackendStepInput(), new RunBackendStepOutput()); + + //////////////////////////////////////////////////// + // Verify one index row was created // + //////////////////////////////////////////////////// + List indexRows = queryAllIndexRows(); + assertThat(indexRows).hasSize(1); + assertThat(indexRows.get(0).getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE); + } + + + + /******************************************************************************* + ** Test: when the OpenSearch client throws an exception, the run record is + ** updated with FAILED status. + *******************************************************************************/ + @Test + void testClientThrows_runRecordMarkedFailed() throws QException + { + //////////////////////////////////////////////////// + // Insert source records so the step attempts to // + // index something // + //////////////////////////////////////////////////// + insertTestEntities(2); + + //////////////////////////////////////////////////// + // Index row due for basepull // + //////////////////////////////////////////////////// + insertIndexRow(TEST_ENTITY_TABLE, true, null, 5); + + //////////////////////////////////////////////////// + // Mock client throws on indexDocuments. // + // Keep discoveredTables so tableConfig is found // + // and documents are built - triggering the call. // + //////////////////////////////////////////////////// + QuickSearchOpenSearchClient mockClient = (QuickSearchOpenSearchClient) QuickSearchQBitContext.getClient(); + when(mockClient.indexDocuments(any(), anyInt())) + .thenThrow(new QException("OpenSearch unavailable")); + + BasepullIndexStep step = new BasepullIndexStep(); + + //////////////////////////////////////////////////// + // Step should not throw (per-table errors are // + // caught and logged) // + //////////////////////////////////////////////////// + step.run(new RunBackendStepInput(), new RunBackendStepOutput()); + + //////////////////////////////////////////////////// + // Run record should be FAILED // + //////////////////////////////////////////////////// + List runs = queryAllRunRecords(); + assertThat(runs).hasSize(1); + assertThat(runs.get(0).getValueString("status")).isEqualTo("FAILED"); + } + + + + /******************************************************************************* + ** Test isDueForBasepull directly. + *******************************************************************************/ + @Test + void testIsDueForBasepull_nullLastTime_returnsTrue() + { + BasepullIndexStep step = new BasepullIndexStep(); + QuickSearchIndex index = new QuickSearchIndex() + .withLastBasepullTime(null) + .withBasepullIntervalMinutes(60); + + assertThat(step.isDueForBasepull(index)).isTrue(); + } + + + + /******************************************************************************* + ** Test isDueForBasepull when last run was just now - not due. + *******************************************************************************/ + @Test + void testIsDueForBasepull_recentLastTime_returnsFalse() + { + BasepullIndexStep step = new BasepullIndexStep(); + QuickSearchIndex index = new QuickSearchIndex() + .withLastBasepullTime(Instant.now()) + .withBasepullIntervalMinutes(60); + + assertThat(step.isDueForBasepull(index)).isFalse(); + } + + + + /******************************************************************************* + ** Test isDueForBasepull when last run was more than the interval ago. + *******************************************************************************/ + @Test + void testIsDueForBasepull_pastDueLastTime_returnsTrue() + { + BasepullIndexStep step = new BasepullIndexStep(); + QuickSearchIndex index = new QuickSearchIndex() + .withLastBasepullTime(Instant.now().minus(90, java.time.temporal.ChronoUnit.MINUTES)) + .withBasepullIntervalMinutes(60); + + assertThat(step.isDueForBasepull(index)).isTrue(); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStepTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStepTest.java new file mode 100644 index 0000000..4c7a538 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/processes/FullReindexStepTest.java @@ -0,0 +1,360 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitContext; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex; +import com.kingsrook.qbits.quicksearch.model.QuickSearchIndexRun; +import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +/******************************************************************************* + ** Tests for FullReindexStep. + ** + ** Verifies that: + ** - All tables are reindexed when no tableName input is provided + ** - Only the specified table is reindexed when tableName input is provided + ** - An empty table results in a run record with 0 records processed + ** - QuickSearchIndex rows are lazily created if they do not yet exist + ** - A failed indexDocuments() call results in a FAILED run record + ** - lastFullReindexTime is updated on the QuickSearchIndex row after reindex + *******************************************************************************/ +class FullReindexStepTest extends BaseQuickSearchTest +{ + + /******************************************************************************* + ** Helper: insert source records into the testEntity table. + *******************************************************************************/ + private void insertTestEntities(int count) throws QException + { + List records = new java.util.ArrayList<>(); + for(int i = 1; i <= count; i++) + { + records.add(new QRecord() + .withValue("name", "Entity " + i) + .withValue("description", "Description " + i)); + } + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TEST_ENTITY_TABLE); + insertInput.setRecords(records); + + new InsertAction().execute(insertInput); + } + + + + /******************************************************************************* + ** Helper: build a RunBackendStepInput with no special values. + *******************************************************************************/ + private RunBackendStepInput buildEmptyInput() + { + return (new RunBackendStepInput()); + } + + + + /******************************************************************************* + ** Helper: build a RunBackendStepInput with a tableName filter. + *******************************************************************************/ + private RunBackendStepInput buildInputWithTableName(String tableName) + { + RunBackendStepInput input = new RunBackendStepInput(); + input.addValue("tableName", tableName); + return (input); + } + + + + /******************************************************************************* + ** Helper: query all records from quickSearchIndexRun. + *******************************************************************************/ + private List queryAllRunRecords() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndexRun.TABLE_NAME); + queryInput.setFilter(new QQueryFilter()); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (queryOutput.getRecords()); + } + + + + /******************************************************************************* + ** Helper: query all records from quickSearchIndex. + *******************************************************************************/ + private List queryAllIndexRecords() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndex.TABLE_NAME); + queryInput.setFilter(new QQueryFilter()); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (queryOutput.getRecords()); + } + + + + /******************************************************************************* + ** Helper: query a quickSearchIndex record by tableName. + *******************************************************************************/ + private QRecord queryIndexByTableName(String tableName) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(QuickSearchIndex.TABLE_NAME); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + List records = queryOutput.getRecords(); + return (records.isEmpty() ? null : records.get(0)); + } + + + + /******************************************************************************* + ** Test: reindex all tables when no tableName input is given. + ** + ** With one discovered table (testEntity) and source records present, + ** deleteDocumentsForTable() and indexDocuments() should each be called once. + *******************************************************************************/ + @Test + void testReindexAllTables_noTableNameInput_reindexesAllDiscoveredTables() throws QException + { + insertTestEntities(3); + + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(3)); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + step.run(buildEmptyInput(), new RunBackendStepOutput()); + + verify(mockClient, times(1)).deleteDocumentsForTable(TEST_ENTITY_TABLE); + verify(mockClient, times(1)).indexDocuments(any(), anyInt()); + } + + + + /******************************************************************************* + ** Test: reindex only the specified table when tableName input is provided. + ** + ** Registers two discovered tables, passes tableName=testEntity. + ** Only testEntity should be reindexed; the other table should be skipped. + *******************************************************************************/ + @Test + void testReindexSingleTable_tableNameInputProvided_onlyThatTableReindexed() throws QException + { + insertTestEntities(2); + + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(2)); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + step.run(buildInputWithTableName(TEST_ENTITY_TABLE), new RunBackendStepOutput()); + + verify(mockClient, times(1)).deleteDocumentsForTable(TEST_ENTITY_TABLE); + verify(mockClient, never()).deleteDocumentsForTable("someOtherTable"); + } + + + + /******************************************************************************* + ** Test: empty source table produces a run record with 0 records processed. + *******************************************************************************/ + @Test + void testEmptyTable_noSourceRecords_runRecordCreatedWithZeroProcessed() throws QException + { + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult()); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + step.run(buildEmptyInput(), new RunBackendStepOutput()); + + List runRecords = queryAllRunRecords(); + assertThat(runRecords).hasSize(1); + + QRecord runRecord = runRecords.get(0); + assertThat(runRecord.getValueString("status")).isEqualTo("SUCCESS"); + assertThat(runRecord.getValueInteger("recordsProcessed")).isEqualTo(0); + assertThat(runRecord.getValueInteger("recordsIndexed")).isEqualTo(0); + + verify(mockClient, times(1)).deleteDocumentsForTable(TEST_ENTITY_TABLE); + verify(mockClient, never()).indexDocuments(any(), anyInt()); + } + + + + /******************************************************************************* + ** Test: QuickSearchIndex row is lazily created if it does not exist yet. + ** + ** No pre-existing index rows. After run(), a quickSearchIndex row for + ** testEntity should be present. + *******************************************************************************/ + @Test + void testLazyIndexRowCreation_noExistingIndexRow_createdBeforeReindex() throws QException + { + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult()); + QuickSearchQBitContext.setClient(mockClient); + + //////////////////////////////////////////////////// + // Verify no rows exist before run // + //////////////////////////////////////////////////// + assertThat(queryAllIndexRecords()).isEmpty(); + + FullReindexStep step = new FullReindexStep(); + step.run(buildEmptyInput(), new RunBackendStepOutput()); + + //////////////////////////////////////////////////// + // Verify index row was created // + //////////////////////////////////////////////////// + List indexRecords = queryAllIndexRecords(); + assertThat(indexRecords).hasSize(1); + assertThat(indexRecords.get(0).getValueString("tableName")).isEqualTo(TEST_ENTITY_TABLE); + } + + + + /******************************************************************************* + ** Test: when indexDocuments() throws, the run record is marked FAILED. + *******************************************************************************/ + @Test + void testErrorDuringIndexing_mockClientThrows_failedRunRecord() throws QException + { + insertTestEntities(1); + + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())) + .thenThrow(new QException("OpenSearch connection refused")); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + assertThatThrownBy(() -> step.run(buildEmptyInput(), new RunBackendStepOutput())) + .isInstanceOf(QException.class) + .hasMessageContaining("Full reindex failed"); + + List runRecords = queryAllRunRecords(); + assertThat(runRecords).hasSize(1); + assertThat(runRecords.get(0).getValueString("status")).isEqualTo("FAILED"); + assertThat(runRecords.get(0).getValueString("errorMessage")).contains("OpenSearch connection refused"); + } + + + + /******************************************************************************* + ** Test: lastFullReindexTime is updated on the QuickSearchIndex row after reindex. + *******************************************************************************/ + @Test + void testUpdatesLastFullReindexTime_afterReindex_fieldIsSet() throws QException + { + insertTestEntities(2); + + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(2)); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + step.run(buildEmptyInput(), new RunBackendStepOutput()); + + QRecord indexRecord = queryIndexByTableName(TEST_ENTITY_TABLE); + assertThat(indexRecord).isNotNull(); + assertThat(indexRecord.getValue("lastFullReindexTime")).isNotNull(); + assertThat(indexRecord.getValueInteger("recordCount")).isEqualTo(2); + } + + + + /******************************************************************************* + ** Test: run record has correct runType = "FULL_REINDEX". + *******************************************************************************/ + @Test + void testRunRecord_runTypeIsFullReindex() throws QException + { + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + when(mockClient.indexDocuments(any(), anyInt())).thenReturn(new BulkIndexResult()); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + step.run(buildEmptyInput(), new RunBackendStepOutput()); + + List runRecords = queryAllRunRecords(); + assertThat(runRecords).hasSize(1); + assertThat(runRecords.get(0).getValueString("runType")).isEqualTo("FULL_REINDEX"); + } + + + + /******************************************************************************* + ** Test: documents are built and passed to indexDocuments with source records. + ** + ** Verifies that the correct number of documents are submitted to the client. + *******************************************************************************/ + @Test + void testIndexDocuments_documentsBuiltFromSourceRecords() throws QException + { + insertTestEntities(5); + + QuickSearchOpenSearchClient mockClient = mock(QuickSearchOpenSearchClient.class); + ArgumentCaptor> docsCaptor = ArgumentCaptor.forClass(List.class); + when(mockClient.indexDocuments(docsCaptor.capture(), anyInt())).thenReturn(new BulkIndexResult().withSuccessCount(5)); + QuickSearchQBitContext.setClient(mockClient); + + FullReindexStep step = new FullReindexStep(); + step.run(buildEmptyInput(), new RunBackendStepOutput()); + + assertThat(docsCaptor.getValue()).hasSize(5); + + List runRecords = queryAllRunRecords(); + assertThat(runRecords).hasSize(1); + assertThat(runRecords.get(0).getValueInteger("recordsProcessed")).isEqualTo(5); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtilsTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtilsTest.java index b68317a..5dcd2c2 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtilsTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/processes/IndexingUtilsTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,139 +13,256 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.kingsrook.qbits.quicksearch.processes; +import java.util.HashMap; import java.util.List; -import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; -import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import java.util.Map; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* - ** Tests for IndexingUtils. + ** Tests for IndexingUtils text normalization and document building. *******************************************************************************/ -class IndexingUtilsTest extends BaseQuickSearchTest +class IndexingUtilsTest { - /*************************************************************************** - ** Test normalizeSearchText with normal input. - ***************************************************************************/ + /******************************************************************************* + ** Test that normal input is lowercased and trimmed. + *******************************************************************************/ @Test - void testNormalizeSearchText_normalInput_returnsLowercase() + void testNormalizeSearchText_normalInput_lowercaseTrimmed() { - String result = IndexingUtils.normalizeSearchText("Hello World"); - assertThat(result).isEqualTo("hello world"); + assertThat(IndexingUtils.normalizeSearchText("Hello World")).isEqualTo("hello world"); } - - /*************************************************************************** - ** Test normalizeSearchText with null input returns empty string. - ***************************************************************************/ + /******************************************************************************* + ** Test that null input returns empty string. + *******************************************************************************/ @Test - void testNormalizeSearchText_nullInput_returnsEmptyString() + void testNormalizeSearchText_null_returnsEmpty() { - String result = IndexingUtils.normalizeSearchText(null); + assertThat(IndexingUtils.normalizeSearchText(null)).isEqualTo(""); + } + + + /******************************************************************************* + ** Test that empty string returns empty string. + *******************************************************************************/ + @Test + void testNormalizeSearchText_empty_returnsEmpty() + { + assertThat(IndexingUtils.normalizeSearchText("")).isEqualTo(""); + } + + + /******************************************************************************* + ** Test that blank string (whitespace only) returns empty string. + *******************************************************************************/ + @Test + void testNormalizeSearchText_blank_returnsEmpty() + { + assertThat(IndexingUtils.normalizeSearchText(" ")).isEqualTo(""); + } + + + /******************************************************************************* + ** Test that multiple internal whitespace is collapsed to single space. + *******************************************************************************/ + @Test + void testNormalizeSearchText_multipleSpaces_collapsed() + { + assertThat(IndexingUtils.normalizeSearchText(" hello world ")).isEqualTo("hello world"); + } + + + /******************************************************************************* + ** Test that mixed case input is fully lowercased. + *******************************************************************************/ + @Test + void testNormalizeSearchText_mixedCase_lowercased() + { + assertThat(IndexingUtils.normalizeSearchText("HeLLo WoRLd")).isEqualTo("hello world"); + } + + + /******************************************************************************* + ** Test that field values are concatenated with space separator. + *******************************************************************************/ + @Test + void testBuildSearchableText_normalFields_concatenated() + { + QRecord record = new QRecord(); + record.setValue("name", "John"); + record.setValue("description", "Developer"); + + String result = IndexingUtils.buildSearchableText(record, List.of("name", "description"), Map.of()); + + assertThat(result).isEqualTo("John Developer"); + } + + + /******************************************************************************* + ** Test that null field values are skipped. + *******************************************************************************/ + @Test + void testBuildSearchableText_nullValue_skipped() + { + QRecord record = new QRecord(); + + String result = IndexingUtils.buildSearchableText(record, List.of("name"), Map.of()); + assertThat(result).isEqualTo(""); } - - /*************************************************************************** - ** Test normalizeSearchText with blank input returns empty string. - ***************************************************************************/ + /******************************************************************************* + ** Test that fields with includeLabel=true are prefixed with fieldName. + *******************************************************************************/ @Test - void testNormalizeSearchText_blankInput_returnsEmptyString() + void testBuildSearchableText_includeLabel_prefixed() { - String result = IndexingUtils.normalizeSearchText(" "); + QRecord record = new QRecord(); + record.setValue("name", "John"); + + String result = IndexingUtils.buildSearchableText(record, List.of("name"), Map.of("name", Boolean.TRUE)); + + assertThat(result).isEqualTo("name: John"); + } + + + /******************************************************************************* + ** Test that empty fields list returns empty string. + *******************************************************************************/ + @Test + void testBuildSearchableText_emptyFields_returnsEmpty() + { + QRecord record = new QRecord(); + record.setValue("name", "John"); + + String result = IndexingUtils.buildSearchableText(record, List.of(), Map.of()); + assertThat(result).isEqualTo(""); } - - /*************************************************************************** - ** Test normalizeSearchText trims leading/trailing whitespace. - ***************************************************************************/ + /******************************************************************************* + ** Test that null fields list returns empty string. + *******************************************************************************/ @Test - void testNormalizeSearchText_extraWhitespace_trims() + void testBuildSearchableText_nullFields_returnsEmpty() { - String result = IndexingUtils.normalizeSearchText(" Hello World "); - assertThat(result).isEqualTo("hello world"); + QRecord record = new QRecord(); + record.setValue("name", "John"); + + String result = IndexingUtils.buildSearchableText(record, null, Map.of()); + + assertThat(result).isEqualTo(""); } - - /*************************************************************************** - ** Test buildDocument with valid record. - ***************************************************************************/ + /******************************************************************************* + ** Test that buildDocument populates all fields correctly. + *******************************************************************************/ @Test - void testBuildDocument_validRecord_buildsDocument() + void testBuildDocument_validRecord_allFieldsPopulated() { - QTableMetaData table = createTestEntityTable(); - QRecord record = new QRecord() - .withValue("id", 123) - .withValue("name", "Test Item") - .withValue("description", "A test description") - .withValue("status", "ACTIVE"); + QRecord record = new QRecord(); + record.setValue("id", 42); + record.setValue("name", "John"); + record.setValue("description", "Developer"); + record.setRecordLabel("John Doe"); - List searchableFields = List.of("name", "description", "status"); + OpenSearchDocument doc = IndexingUtils.buildDocument( + record, + "customers", + "id", + List.of("name", "description"), + Map.of(), + Map.of() + ); - OpenSearchDocument doc = IndexingUtils.buildDocument(record, table, searchableFields); - - assertThat(doc.getSourceTable()).isEqualTo(TEST_TABLE_NAME); - assertThat(doc.getRecordId()).isEqualTo("123"); - assertThat(doc.getDocumentId()).isEqualTo(TEST_TABLE_NAME + ":123"); - assertThat(doc.getSearchableText()).contains("Test Item"); - assertThat(doc.getSearchableText()).contains("A test description"); - assertThat(doc.getSearchableText()).contains("ACTIVE"); + assertThat(doc.getSourceTable()).isEqualTo("customers"); + assertThat(doc.getRecordId()).isEqualTo("42"); + assertThat(doc.getRecordLabel()).isEqualTo("John Doe"); + assertThat(doc.getSearchableText()).isEqualTo("John Developer"); + assertThat(doc.getFieldValues()).containsKey("name"); + assertThat(doc.getFieldValues()).containsKey("description"); + assertThat(doc.getIndexedAt()).isNotNull(); } - - /*************************************************************************** - ** Test buildDocument with null field values skips those fields. - ***************************************************************************/ + /******************************************************************************* + ** Test that buildDocument with null primary key value yields null recordId. + *******************************************************************************/ @Test - void testBuildDocument_nullFieldValues_skipsNulls() + void testBuildDocument_nullPrimaryKey_recordIdIsNull() { - QTableMetaData table = createTestEntityTable(); - QRecord record = new QRecord() - .withValue("id", 456) - .withValue("name", "Test") - .withValue("description", null) - .withValue("status", null); + QRecord record = new QRecord(); + record.setValue("name", "John"); - List searchableFields = List.of("name", "description", "status"); + OpenSearchDocument doc = IndexingUtils.buildDocument( + record, + "customers", + "id", + List.of("name"), + Map.of(), + Map.of() + ); - OpenSearchDocument doc = IndexingUtils.buildDocument(record, table, searchableFields); - - assertThat(doc.getSearchableText()).isEqualTo("Test"); + assertThat(doc.getRecordId()).isNull(); } - - /*************************************************************************** - ** Test buildDocument with empty searchable fields list returns empty text. - ***************************************************************************/ + /******************************************************************************* + ** Test that null field values are not included in fieldValues map. + *******************************************************************************/ @Test - void testBuildDocument_emptySearchableFields_returnsEmptyText() + void testBuildDocument_nullFieldValue_notInFieldValues() { - QTableMetaData table = createTestEntityTable(); - QRecord record = new QRecord() - .withValue("id", 789) - .withValue("name", "All Fields") - .withValue("description", "Should include all") - .withValue("status", "PENDING"); + QRecord record = new QRecord(); + record.setValue("id", 1); + record.setValue("name", "John"); - List searchableFields = List.of(); + OpenSearchDocument doc = IndexingUtils.buildDocument( + record, + "customers", + "id", + List.of("name", "description"), + Map.of(), + Map.of() + ); - OpenSearchDocument doc = IndexingUtils.buildDocument(record, table, searchableFields); + assertThat(doc.getFieldValues()).containsKey("name"); + assertThat(doc.getFieldValues()).doesNotContainKey("description"); + } - assertThat(doc.getSearchableText()).isEmpty(); + + /******************************************************************************* + ** Test that buildDocument with empty searchable fields yields empty searchableText. + *******************************************************************************/ + @Test + void testBuildDocument_emptySearchableFields_emptySearchableText() + { + QRecord record = new QRecord(); + record.setValue("id", 1); + + OpenSearchDocument doc = IndexingUtils.buildDocument( + record, + "customers", + "id", + List.of(), + Map.of(), + Map.of() + ); + + assertThat(doc.getSearchableText()).isEqualTo(""); } } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventTest.java new file mode 100644 index 0000000..673a953 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/publisher/IndexEventTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.publisher; + + +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Tests for IndexEvent and IndexEventAction. + *******************************************************************************/ +class IndexEventTest +{ + + /******************************************************************************* + ** Test that IndexEventAction enum has INDEX and DELETE values. + *******************************************************************************/ + @Test + void testIndexEventActionEnumValues() + { + assertThat(IndexEventAction.INDEX).isNotNull(); + assertThat(IndexEventAction.DELETE).isNotNull(); + assertThat(IndexEventAction.values()).hasSize(2); + } + + /******************************************************************************* + ** Test that IndexEvent fluent setters return this and getters return values. + *******************************************************************************/ + @Test + void testIndexEventFluentSetters() + { + QRecord record = new QRecord(); + IndexEvent event = new IndexEvent() + .withTableName("customers") + .withRecordId("99") + .withAction(IndexEventAction.INDEX) + .withRecord(record); + + assertThat(event.getTableName()).isEqualTo("customers"); + assertThat(event.getRecordId()).isEqualTo("99"); + assertThat(event.getAction()).isEqualTo(IndexEventAction.INDEX); + assertThat(event.getRecord()).isSameAs(record); + } + + /******************************************************************************* + ** Test that IndexEvent withX methods return this. + *******************************************************************************/ + @Test + void testIndexEventWithReturnsThis() + { + IndexEvent event = new IndexEvent(); + assertThat(event.withTableName("t")).isSameAs(event); + assertThat(event.withRecordId("1")).isSameAs(event); + assertThat(event.withAction(IndexEventAction.DELETE)).isSameAs(event); + assertThat(event.withRecord(null)).isSameAs(event); + } + + /******************************************************************************* + ** Test that IndexEvent record field can be null. + *******************************************************************************/ + @Test + void testIndexEventRecordNullable() + { + IndexEvent event = new IndexEvent() + .withTableName("orders") + .withRecordId("1") + .withAction(IndexEventAction.DELETE); + + assertThat(event.getRecord()).isNull(); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/publisher/SynchronousIndexEventPublisherTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/publisher/SynchronousIndexEventPublisherTest.java new file mode 100644 index 0000000..8f7cc51 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/publisher/SynchronousIndexEventPublisherTest.java @@ -0,0 +1,290 @@ +/* + * Copyright 2024 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.publisher; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig; +import com.kingsrook.qbits.quicksearch.opensearch.BulkIndexResult; +import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument; +import com.kingsrook.qbits.quicksearch.opensearch.QuickSearchOpenSearchClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + + +/******************************************************************************* + ** Tests for SynchronousIndexEventPublisher. + *******************************************************************************/ +class SynchronousIndexEventPublisherTest +{ + private QuickSearchOpenSearchClient mockClient; + private SynchronousIndexEventPublisher publisher; + + private static final Integer BATCH_SIZE = 100; + + + + /******************************************************************************* + ** Set up a publisher with a mocked client and a single table config. + *******************************************************************************/ + @BeforeEach + void setUp() + { + mockClient = mock(QuickSearchOpenSearchClient.class); + + QuickSearchableTableConfig ordersConfig = new QuickSearchableTableConfig() + .withTableName("orders") + .withPrimaryKeyField("id") + .withSearchableFields(List.of("orderNumber", "customerName")) + .withFieldWeights(Map.of("orderNumber", 2, "customerName", 1)) + .withFieldIncludeLabels(Map.of("orderNumber", false, "customerName", false)); + + publisher = new SynchronousIndexEventPublisher(mockClient, List.of(ordersConfig), BATCH_SIZE); + } + + + + /******************************************************************************* + ** Test publishIndexEvents with INDEX events calls client.indexDocuments with + ** correctly built documents. + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Test + void testPublishIndexEvents_withIndexEvents_callsClientWithDocuments() throws QException + { + QRecord record1 = new QRecord() + .withValue("id", 1) + .withValue("orderNumber", "ORD-001") + .withValue("customerName", "Alice"); + + QRecord record2 = new QRecord() + .withValue("id", 2) + .withValue("orderNumber", "ORD-002") + .withValue("customerName", "Bob"); + + List events = List.of( + new IndexEvent() + .withTableName("orders") + .withRecordId("1") + .withAction(IndexEventAction.INDEX) + .withRecord(record1), + new IndexEvent() + .withTableName("orders") + .withRecordId("2") + .withAction(IndexEventAction.INDEX) + .withRecord(record2)); + + when(mockClient.indexDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult()); + + publisher.publishIndexEvents(events); + + ArgumentCaptor> docsCaptor = ArgumentCaptor.forClass(List.class); + ArgumentCaptor batchCaptor = ArgumentCaptor.forClass(Integer.class); + verify(mockClient).indexDocuments(docsCaptor.capture(), batchCaptor.capture()); + + List capturedDocs = docsCaptor.getValue(); + assertThat(capturedDocs).hasSize(2); + assertThat(capturedDocs.get(0).getSourceTable()).isEqualTo("orders"); + assertThat(capturedDocs.get(0).getRecordId()).isEqualTo("1"); + assertThat(capturedDocs.get(0).getSearchableText()).contains("ORD-001"); + assertThat(capturedDocs.get(1).getRecordId()).isEqualTo("2"); + assertThat(batchCaptor.getValue()).isEqualTo(BATCH_SIZE); + } + + + + /******************************************************************************* + ** Test publishIndexEvents with empty list makes no client calls. + *******************************************************************************/ + @Test + void testPublishIndexEvents_emptyList_noClientCalls() throws QException + { + publisher.publishIndexEvents(Collections.emptyList()); + verifyNoInteractions(mockClient); + } + + + + /******************************************************************************* + ** Test publishIndexEvents with null list makes no client calls. + *******************************************************************************/ + @Test + void testPublishIndexEvents_nullList_noClientCalls() throws QException + { + publisher.publishIndexEvents(null); + verifyNoInteractions(mockClient); + } + + + + /******************************************************************************* + ** Test publishIndexEvents skips events for unknown tables. + *******************************************************************************/ + @Test + void testPublishIndexEvents_unknownTable_skipped() throws QException + { + IndexEvent event = new IndexEvent() + .withTableName("unknownTable") + .withRecordId("1") + .withAction(IndexEventAction.INDEX) + .withRecord(new QRecord().withValue("id", 1)); + + publisher.publishIndexEvents(List.of(event)); + + verify(mockClient, never()).indexDocuments(anyList(), anyInt()); + } + + + + /******************************************************************************* + ** Test publishDeleteEvents calls client.deleteDocument for each event. + *******************************************************************************/ + @Test + void testPublishDeleteEvents_callsClientForEachEvent() throws QException + { + List events = List.of( + new IndexEvent() + .withTableName("orders") + .withRecordId("10") + .withAction(IndexEventAction.DELETE), + new IndexEvent() + .withTableName("orders") + .withRecordId("20") + .withAction(IndexEventAction.DELETE)); + + publisher.publishDeleteEvents(events); + + verify(mockClient).deleteDocument("orders", "10"); + verify(mockClient).deleteDocument("orders", "20"); + } + + + + /******************************************************************************* + ** Test publishDeleteEvents with empty list makes no client calls. + *******************************************************************************/ + @Test + void testPublishDeleteEvents_emptyList_noClientCalls() throws QException + { + publisher.publishDeleteEvents(Collections.emptyList()); + verifyNoInteractions(mockClient); + } + + + + /******************************************************************************* + ** Test publishDeleteEvents when client throws propagates QException. + *******************************************************************************/ + @Test + void testPublishDeleteEvents_clientThrows_propagatesException() throws QException + { + doThrow(new QException("delete failed")) + .when(mockClient).deleteDocument("orders", "1"); + + IndexEvent event = new IndexEvent() + .withTableName("orders") + .withRecordId("1") + .withAction(IndexEventAction.DELETE); + + assertThatThrownBy(() -> publisher.publishDeleteEvents(List.of(event))) + .isInstanceOf(QException.class) + .hasMessageContaining("delete failed"); + } + + + + /******************************************************************************* + ** Test close delegates to client.close(). + *******************************************************************************/ + @Test + void testClose_delegatesToClient() throws QException + { + publisher.close(); + verify(mockClient).close(); + } + + + + /******************************************************************************* + ** Test publishIndexEvents uses default primaryKeyField "id" when config + ** does not specify one. + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Test + void testPublishIndexEvents_defaultPrimaryKeyField() throws QException + { + QuickSearchableTableConfig configWithoutPk = new QuickSearchableTableConfig() + .withTableName("products") + .withSearchableFields(List.of("name")) + .withFieldWeights(Map.of("name", 1)) + .withFieldIncludeLabels(Map.of("name", false)); + + SynchronousIndexEventPublisher pub = new SynchronousIndexEventPublisher( + mockClient, List.of(configWithoutPk), BATCH_SIZE); + + QRecord record = new QRecord() + .withValue("id", 42) + .withValue("name", "Widget"); + + IndexEvent event = new IndexEvent() + .withTableName("products") + .withRecordId("42") + .withAction(IndexEventAction.INDEX) + .withRecord(record); + + when(mockClient.indexDocuments(anyList(), anyInt())).thenReturn(new BulkIndexResult()); + + pub.publishIndexEvents(List.of(event)); + + ArgumentCaptor> docsCaptor = ArgumentCaptor.forClass(List.class); + verify(mockClient).indexDocuments(docsCaptor.capture(), anyInt()); + + List docs = docsCaptor.getValue(); + assertThat(docs).hasSize(1); + assertThat(docs.get(0).getRecordId()).isEqualTo("42"); + assertThat(docs.get(0).getSourceTable()).isEqualTo("products"); + } + + + + /******************************************************************************* + ** Test publishDeleteEvents with null list makes no client calls. + *******************************************************************************/ + @Test + void testPublishDeleteEvents_nullList_noClientCalls() throws QException + { + publisher.publishDeleteEvents(null); + verifyNoInteractions(mockClient); + } + +}