From 24653de55b447ae3b70ef82f18ee51c646e1636e Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 8 Oct 2026 07:23:44 -0500 Subject: [PATCH 1/2] build(#29): adopt qbit-quick-search 1.0.0-RC.2 Pin qbit-quick-search to 1.0.0-RC.2 and move DemoQBits to the 1.0 API. The producer registers its own QBitMetaData, so the manual addQBit is removed. Quick search now connects through opensearchUrl, runs its basepull on demoScheduler, keeps OpenSearch credentials as ${env.X} references, and reads its startup mode from QQQ_ALL_OPENSEARCH_STARTUP_MODE: FAIL_FAST by default, DEGRADED in the full-profile tests, where no OpenSearch listens. Grant the full-profile admin role quickSearchAdmin.hasAccess and the operational table permissions that 1.0 requires, and document the upgrade path for volumes created with 0.x. Closes #29 --- .../postgres/06-quick-search-permissions.sql | 24 ++++ qqq-all-app/README.md | 4 +- .../java/com/kingsrook/qqq/all/DemoQBits.java | 37 ++++-- .../kingsrook/qqq/all/FullProfileTest.java | 2 + .../com/kingsrook/qqq/all/FullQBitsTest.java | 111 +++++++++++++++++- .../kingsrook/qqq/all/bom/StackBomTest.java | 1 + qqq-all-bom/pom.xml | 5 +- 7 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 infra/postgres/06-quick-search-permissions.sql diff --git a/infra/postgres/06-quick-search-permissions.sql b/infra/postgres/06-quick-search-permissions.sql new file mode 100644 index 0000000..e2beb5c --- /dev/null +++ b/infra/postgres/06-quick-search-permissions.sql @@ -0,0 +1,24 @@ +/* Give the full-profile demo admin the quick-search admin app, its processes, and + its operational tables. qbit-quick-search 1.0 protects the app and processes + with quickSearchAdmin.hasAccess and the tables with read/write permissions. + The viewer role keeps searching customers through customer.read. */ +INSERT INTO permission (name, object_type, object_label) +VALUES + ('quickSearchAdmin.hasAccess', 'app', 'Quick Search Admin'), + ('quickSearchIndex.read', 'table', 'quickSearchIndex'), + ('quickSearchIndex.write', 'table', 'quickSearchIndex'), + ('quickSearchIndexRun.read', 'table', 'quickSearchIndexRun'), + ('quickSearchIndexRun.write', 'table', 'quickSearchIndexRun'), + ('quickSearchFailedEvent.read', 'table', 'quickSearchFailedEvent'), + ('quickSearchFailedEvent.write', 'table', 'quickSearchFailedEvent') +ON CONFLICT (name) DO NOTHING; + +INSERT INTO role_permission_int (role_id, permission_id) +SELECT role.id, permission.id +FROM role CROSS JOIN permission +WHERE role.name = 'admin' + AND permission.name IN ('quickSearchAdmin.hasAccess', + 'quickSearchIndex.read', 'quickSearchIndex.write', + 'quickSearchIndexRun.read', 'quickSearchIndexRun.write', + 'quickSearchFailedEvent.read', 'quickSearchFailedEvent.write') +ON CONFLICT (role_id, permission_id) DO NOTHING; diff --git a/qqq-all-app/README.md b/qqq-all-app/README.md index 5315e1d..b9f47fa 100644 --- a/qqq-all-app/README.md +++ b/qqq-all-app/README.md @@ -31,7 +31,7 @@ Configure the following environment variables before starting the same jar: | ESB | `QQQ_ALL_ARTEMIS_URL`, `QQQ_ALL_ARTEMIS_USER`, `QQQ_ALL_ARTEMIS_PASSWORD`, `QQQ_ALL_ARTEMIS_MANAGEMENT_URL`, `QQQ_ALL_RABBITMQ_URL`, `QQQ_ALL_RABBITMQ_USER`, `QQQ_ALL_RABBITMQ_PASSWORD`, `QQQ_ALL_RABBITMQ_MANAGEMENT_URL` | — | | Mailpit SMTP | `QQQ_ALL_SMTP_HOST` | `QQQ_ALL_SMTP_PORT` (1025) | | Keycloak OIDC | `QQQ_ALL_OIDC_BASE_URL`, `QQQ_ALL_OIDC_CLIENT_ID`, `QQQ_ALL_OIDC_CLIENT_SECRET` | `QQQ_ALL_OIDC_EXTERNAL_BASE_URL` (base URL), `QQQ_ALL_OIDC_SCOPES` (`openid email profile`) | -| OpenSearch quick search | `QQQ_ALL_OPENSEARCH_HOST` | `QQQ_ALL_OPENSEARCH_PORT` (9200), `QQQ_ALL_OPENSEARCH_INDEX` (`qqq-all-customers`), `QQQ_ALL_OPENSEARCH_USER`, `QQQ_ALL_OPENSEARCH_PASSWORD`, `QQQ_ALL_OPENSEARCH_SSL` (false) | +| OpenSearch quick search | `QQQ_ALL_OPENSEARCH_HOST` | `QQQ_ALL_OPENSEARCH_PORT` (9200), `QQQ_ALL_OPENSEARCH_INDEX` (`qqq-all-customers`), `QQQ_ALL_OPENSEARCH_USER`, `QQQ_ALL_OPENSEARCH_PASSWORD`, `QQQ_ALL_OPENSEARCH_SSL` (false), `QQQ_ALL_OPENSEARCH_STARTUP_MODE` (`FAIL_FAST`) | The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Quick search is configured for customer `name` and `email`; run `quickSearchFullReindex` after OpenSearch and the customer data are ready. The SFTP integration adds its import source and staging tables plus `SFTPImportFileSyncProcess`; `externalImportFile` remains the direct SFTP file sample. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. Webhook tables and delivery processes require explicit permissions: the viewer role receives read access, while the admin role can edit destinations and run delivery. The Compose profile provisions demo SQL tables, OAuth session and redirect-state tables, qbit permission tables and roles, and the MinIO bucket on first startup; `infra/postgres/03-webhook-permissions.sql` adds webhook grants after the role seed. @@ -43,6 +43,8 @@ COMPOSE_PROFILES=full docker compose exec -T postgres sh -c 'psql -v ON_ERROR_ST Then sign in as Admin and open the ESB overview. The Viewer role must remain unable to open it. +Quick search uses `qbit-quick-search` 1.0. It indexes customer writes after they commit, records failed real-time updates in `quickSearchFailedEvent`, and runs its basepull catch-up on `demoScheduler`. Startup fails when OpenSearch is unreachable; set `QQQ_ALL_OPENSEARCH_STARTUP_MODE=DEGRADED` to boot without it and retry on first use. OpenSearch credentials are passed to the qbit as `${env.QQQ_ALL_OPENSEARCH_USER}` and `${env.QQQ_ALL_OPENSEARCH_PASSWORD}` references and require `QQQ_ALL_OPENSEARCH_SSL=true`; the qbit refuses them over plain HTTP. Its admin app, processes, and tables require `quickSearchAdmin.hasAccess` and table read/write permissions, which `infra/postgres/06-quick-search-permissions.sql` grants to the admin role. A volume created with quick-search 0.x has an older `quick_search_index` table and index mapping: apply `06-quick-search-permissions.sql` with the same `psql` command as above, drop the `quick_search_*` tables (or recreate the volume) so startup can create the 1.0 columns, then run `quickSearchFullReindex` once to rebuild the index behind its alias. + On startup, the full demo creates its remaining PostgreSQL qbit tables from the registered metadata and checks every expected column and type. The hand-maintained domain, OAuth, and role tables stay under `infra/postgres`. If a persisted qbit table is missing a required column or has a different type, startup fails with a migration/reset message instead of silently accepting it. The full-profile RabbitMQ completion-event smoke requires QQQ's process publishing and permission-scoped ESB message endpoint; the broker management smoke below can run independently. After starting the full Compose profile, run the opt-in broker management smoke test with the same broker credentials. For the supplied demo environment: diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java index 938a12f..055e638 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java @@ -11,6 +11,7 @@ import com.kingsrook.qbits.customizabletableviews.CustomizableTableViewsQBitConf import com.kingsrook.qbits.customizabletableviews.CustomizableTableViewsQBitProducer; import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer; +import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode; import com.kingsrook.qbits.quicksearch.SearchableFieldConfig; import com.kingsrook.qbits.sftpdataintegration.SFTPDataIntegrationQBitConfig; import com.kingsrook.qbits.sftpdataintegration.SFTPDataIntegrationQBitProducer; @@ -34,7 +35,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.qbits.ProvidedOrSuppliedTableConfig; -import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; @@ -148,19 +148,26 @@ final class DemoQBits static void addFull(QInstance instance, FullProfileSettings settings) throws QException { + String scheme = Boolean.parseBoolean(settings.getOrDefault("OPENSEARCH_SSL", "false")) ? "https" : "http"; QuickSearchQBitConfig searchConfig = new QuickSearchQBitConfig() .withBackendName("postgres") - .withOpensearchHost(settings.get("OPENSEARCH_HOST")) - .withOpensearchPort(settings.port("OPENSEARCH_PORT", "9200")) + .withOpensearchUrl(scheme + "://" + settings.get("OPENSEARCH_HOST") + ":" + settings.port("OPENSEARCH_PORT", "9200")) .withOpensearchIndexName(settings.getOrDefault("OPENSEARCH_INDEX", "qqq-all-customers")) - .withOpensearchUsername(settings.get("OPENSEARCH_USER")) - .withOpensearchPassword(settings.get("OPENSEARCH_PASSWORD")) - .withUseSsl(Boolean.parseBoolean(settings.getOrDefault("OPENSEARCH_SSL", "false"))) + .withSchedulerName("demoScheduler") + .withStartupMode(quickSearchStartupMode(settings)) .withSearchableTable("customer", List.of(new SearchableFieldConfig("name"), new SearchableFieldConfig("email"))); + // Secrets stay in the environment: the qbit resolves these references only when it connects. + if(settings.getOrDefault("OPENSEARCH_USER", null) != null) + { + searchConfig.withOpensearchUsername("${env.QQQ_ALL_OPENSEARCH_USER}"); + } + if(settings.getOrDefault("OPENSEARCH_PASSWORD", null) != null) + { + searchConfig.withOpensearchPassword("${env.QQQ_ALL_OPENSEARCH_PASSWORD}"); + } + // The producer registers its own QBitMetaData (com.kingsrook.qbits:quick-search). new QuickSearchQBitProducer().withConfig(searchConfig).produce(instance); - instance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits") - .withArtifactId("quick-search").withVersion("0.2.1-SNAPSHOT").withConfig(searchConfig)); new SavedBulkLoadProfileMetaDataProvider().defineAll(instance, "postgres", null); new SavedReportsMetaDataProvider().defineAll(instance, "postgres", "sftp", null); @@ -181,4 +188,18 @@ final class DemoQBits return table; })).produce(instance, "demo-import"); } + + /** FAIL_FAST (the qbit default) stops startup when OpenSearch is unreachable; DEGRADED boots and retries on first use. */ + private static QuickSearchStartupMode quickSearchStartupMode(FullProfileSettings settings) + { + String mode = settings.getOrDefault("OPENSEARCH_STARTUP_MODE", QuickSearchStartupMode.FAIL_FAST.name()); + try + { + return QuickSearchStartupMode.valueOf(mode); + } + catch(IllegalArgumentException e) + { + throw new IllegalArgumentException("QQQ_ALL_OPENSEARCH_STARTUP_MODE must be FAIL_FAST or DEGRADED"); + } + } } diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java index ce2fe57..86e466a 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java @@ -122,6 +122,8 @@ class FullProfileTest values.put("QQQ_ALL_OIDC_CLIENT_SECRET", "fixture"); values.put("QQQ_ALL_OPENSEARCH_HOST", "127.0.0.1"); values.put("QQQ_ALL_OPENSEARCH_PORT", "1"); + // Nothing listens on port 1, so quick search must boot degraded instead of failing fast. + values.put("QQQ_ALL_OPENSEARCH_STARTUP_MODE", "DEGRADED"); return values; } } diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java index 8b14d3b..e3b5afa 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java @@ -7,18 +7,25 @@ package com.kingsrook.qqq.all; import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; +import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode; +import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -27,6 +34,7 @@ import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -75,13 +83,106 @@ class FullQBitsTest assertEquals(PermissionLevel.HAS_ACCESS_PERMISSION, instance.getProcess(processName).getPermissionRules().getLevel(), processName); } - QuickSearchQBitConfig search = (QuickSearchQBitConfig) instance.getQBits().values().stream() - .filter(qbit -> "quick-search".equals(qbit.getArtifactId())).findFirst().orElseThrow().getConfig(); + QuickSearchQBitConfig search = quickSearch(instance); assertEquals("customer", search.getSearchableTables().getFirst().getTableName()); assertEquals(Set.of("name", "email"), search.getSearchableTables().getFirst().getFields().stream() .map(field -> field.getFieldName()).collect(Collectors.toSet())); } + @Test + void quickSearchRegistersItselfAndRunsOnTheDemoScheduler() throws Exception + { + QInstance instance = new FullProfileApplication(dataDirectory, + FullProfileSettings.from(FullProfileTest.environment())).defineQInstance(); + new QInstanceValidator().validate(instance); + + List registered = instance.getQBits().values().stream() + .filter(qbit -> "quick-search".equals(qbit.getArtifactId())).toList(); + assertEquals(1, registered.size()); + assertEquals("com.kingsrook.qbits", registered.getFirst().getGroupId()); + assertNotNull(registered.getFirst().getVersion()); + + QuickSearchQBitConfig search = quickSearch(instance); + assertEquals("http://127.0.0.1:1", search.getOpensearchUrl()); + assertEquals(QuickSearchStartupMode.DEGRADED, search.getStartupMode()); + assertNull(search.getOpensearchUsername()); + assertNull(search.getOpensearchPassword()); + assertEquals("demoScheduler", search.getSchedulerName()); + assertNotNull(instance.getScheduler("demoScheduler")); + assertEquals("demoScheduler", instance.getProcess("quickSearchBasepullIndex").getSchedule().getSchedulerName()); + assertNotNull(instance.getTable("quickSearchFailedEvent")); + assertTrue(instance.getRecordChangeListeners().stream() + .anyMatch(listener -> QuickSearchRecordChangeListener.class.getName().equals(listener.getName()))); + } + + @Test + void quickSearchKeepsOpenSearchCredentialsAsEnvironmentReferences() throws Exception + { + Map environment = new HashMap<>(FullProfileTest.environment()); + environment.put("QQQ_ALL_OPENSEARCH_SSL", "true"); + environment.put("QQQ_ALL_OPENSEARCH_USER", "fixture-user"); + environment.put("QQQ_ALL_OPENSEARCH_PASSWORD", "fixture-password"); + QuickSearchQBitConfig search = quickSearch(new FullProfileApplication(dataDirectory, + FullProfileSettings.from(environment)).defineQInstance()); + + assertEquals("https://127.0.0.1:1", search.getOpensearchUrl()); + assertEquals("${env.QQQ_ALL_OPENSEARCH_USER}", search.getOpensearchUsername()); + assertEquals("${env.QQQ_ALL_OPENSEARCH_PASSWORD}", search.getOpensearchPassword()); + } + + @Test + void quickSearchFailsFastWithoutOpenSearchUnlessDegraded() + { + Map environment = new HashMap<>(FullProfileTest.environment()); + environment.remove("QQQ_ALL_OPENSEARCH_STARTUP_MODE"); + assertThrows(QException.class, () -> new FullProfileApplication(dataDirectory, + FullProfileSettings.from(environment)).defineQInstance()); + + environment.put("QQQ_ALL_OPENSEARCH_STARTUP_MODE", "SOMETIMES"); + assertThrows(IllegalArgumentException.class, () -> new FullProfileApplication(dataDirectory, + FullProfileSettings.from(environment)).defineQInstance()); + } + + @Test + void onlyQuickSearchAdminsCanRunTheReindexAndEditItsTables() throws Exception + { + QInstance instance = new FullProfileApplication(dataDirectory, + FullProfileSettings.from(FullProfileTest.environment())).defineQInstance(); + QContext.init(instance, new QSession().withPermissions("customer.read")); + try + { + assertThrows(QPermissionDeniedException.class, + () -> PermissionsHelper.checkProcessPermissionThrowing(new RunProcessInput(), "quickSearchFullReindex")); + assertFalse(PermissionsHelper.hasTablePermission(new QueryInput("quickSearchIndex"), + "quickSearchIndex", TablePermissionSubType.READ)); + } + finally + { + QContext.clear(); + } + + // The grants infra/postgres/06-quick-search-permissions.sql gives the admin role. + QContext.init(instance, new QSession().withPermissions("quickSearchAdmin.hasAccess", + "quickSearchIndex.read", "quickSearchIndex.write", "quickSearchIndexRun.read", "quickSearchIndexRun.write", + "quickSearchFailedEvent.read", "quickSearchFailedEvent.write")); + try + { + for(String processName : Set.of("quickSearchFullReindex", "quickSearchReconcileIndex", "quickSearchBasepullIndex")) + { + PermissionsHelper.checkProcessPermissionThrowing(new RunProcessInput(), processName); + } + for(String tableName : Set.of("quickSearchIndex", "quickSearchIndexRun", "quickSearchFailedEvent")) + { + assertTrue(PermissionsHelper.hasTablePermission(new QueryInput(tableName), + tableName, TablePermissionSubType.EDIT), tableName); + } + } + finally + { + QContext.clear(); + } + } + @Test void viewerPermissionsCannotChangeWebhookDestinationOrSendEvent() throws Exception { @@ -125,4 +226,10 @@ class FullQBitsTest QContext.clear(); } } + + private static QuickSearchQBitConfig quickSearch(QInstance instance) + { + return (QuickSearchQBitConfig) instance.getQBits().values().stream() + .filter(qbit -> "quick-search".equals(qbit.getArtifactId())).findFirst().orElseThrow().getConfig(); + } } diff --git a/qqq-all-bom-test/src/test/java/com/kingsrook/qqq/all/bom/StackBomTest.java b/qqq-all-bom-test/src/test/java/com/kingsrook/qqq/all/bom/StackBomTest.java index 0e0124d..08e783d 100644 --- a/qqq-all-bom-test/src/test/java/com/kingsrook/qqq/all/bom/StackBomTest.java +++ b/qqq-all-bom-test/src/test/java/com/kingsrook/qqq/all/bom/StackBomTest.java @@ -98,6 +98,7 @@ class StackBomTest assertEquals(List.of(), missing, "Artifacts the stack BOM does not manage"); assertEquals("3.9.0", managedVersions.get("com.rabbitmq.jms:rabbitmq-jms")); assertEquals("1.0.0-RC.8", managedVersions.get("com.kingsrook.qqq:qqq-frontend-next")); + assertEquals("1.0.0-RC.2", managedVersions.get("com.kingsrook.qbits:qbit-quick-search")); } diff --git a/qqq-all-bom/pom.xml b/qqq-all-bom/pom.xml index 76f7a07..c3e8c29 100644 --- a/qqq-all-bom/pom.xml +++ b/qqq-all-bom/pom.xml @@ -20,8 +20,9 @@ 4.1.0-SNAPSHOT 1.0.0-RC.8 - - 0.2.1-SNAPSHOT + + + 1.0.0-RC.2 0.33.0-SNAPSHOT 0.5.0-SNAPSHOT 0.3.0-SNAPSHOT -- 2.54.0 From eccc916db21218d8b562e252f9007e01223a76d4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 9 Oct 2026 18:31:16 -0500 Subject: [PATCH 2/2] fix(#29): tighten quick-search wiring, tests, and docs Build the OpenSearch URL with java.net.URI so IPv6 hosts get brackets, and parse QQQ_ALL_OPENSEARCH_STARTUP_MODE case-insensitively, naming the bad value and keeping the cause when it is invalid. The tests now check the fail-fast and invalid-mode messages, cover an IPv6 host, and read the admin grants from 06-quick-search-permissions.sql, so the test and the grant file cannot drift. The README no longer claims a basepull catch-up for customers: the demo customer table has no modifyDate, so after the first basepull only the real-time listener, failed-event replay, and on-demand reconcile or full reindex keep the index current. Refs #29 --- qqq-all-app/README.md | 2 +- .../java/com/kingsrook/qqq/all/DemoQBits.java | 27 ++++++++--- .../com/kingsrook/qqq/all/FullQBitsTest.java | 45 ++++++++++++++++--- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/qqq-all-app/README.md b/qqq-all-app/README.md index b9f47fa..a60b0bf 100644 --- a/qqq-all-app/README.md +++ b/qqq-all-app/README.md @@ -43,7 +43,7 @@ COMPOSE_PROFILES=full docker compose exec -T postgres sh -c 'psql -v ON_ERROR_ST Then sign in as Admin and open the ESB overview. The Viewer role must remain unable to open it. -Quick search uses `qbit-quick-search` 1.0. It indexes customer writes after they commit, records failed real-time updates in `quickSearchFailedEvent`, and runs its basepull catch-up on `demoScheduler`. Startup fails when OpenSearch is unreachable; set `QQQ_ALL_OPENSEARCH_STARTUP_MODE=DEGRADED` to boot without it and retry on first use. OpenSearch credentials are passed to the qbit as `${env.QQQ_ALL_OPENSEARCH_USER}` and `${env.QQQ_ALL_OPENSEARCH_PASSWORD}` references and require `QQQ_ALL_OPENSEARCH_SSL=true`; the qbit refuses them over plain HTTP. Its admin app, processes, and tables require `quickSearchAdmin.hasAccess` and table read/write permissions, which `infra/postgres/06-quick-search-permissions.sql` grants to the admin role. A volume created with quick-search 0.x has an older `quick_search_index` table and index mapping: apply `06-quick-search-permissions.sql` with the same `psql` command as above, drop the `quick_search_*` tables (or recreate the volume) so startup can create the 1.0 columns, then run `quickSearchFullReindex` once to rebuild the index behind its alias. +Quick search uses `qbit-quick-search` 1.0. It indexes customer writes after they commit and records failed real-time updates in `quickSearchFailedEvent`; the basepull scheduled on `demoScheduler` replays those every five minutes. Incremental basepull needs a timestamp field (`modifyDate` by default), which the demo `customer` table does not have, so for customers only the first basepull indexes every row. After that, a change that real-time indexing missed reaches the index only when you run `quickSearchReconcileIndex` or `quickSearchFullReindex`; the demo configures no reconcile cron. Startup fails when OpenSearch is unreachable; set `QQQ_ALL_OPENSEARCH_STARTUP_MODE=DEGRADED` to boot without it and retry on first use. OpenSearch credentials are passed to the qbit as `${env.QQQ_ALL_OPENSEARCH_USER}` and `${env.QQQ_ALL_OPENSEARCH_PASSWORD}` references and require `QQQ_ALL_OPENSEARCH_SSL=true`; the qbit refuses them over plain HTTP. Its admin app, processes, and tables require `quickSearchAdmin.hasAccess` and table read/write permissions, which `infra/postgres/06-quick-search-permissions.sql` grants to the admin role. A volume created with quick-search 0.x has an older `quick_search_index` table and index mapping: apply `06-quick-search-permissions.sql` with the same `psql` command as above, drop the `quick_search_*` tables (or recreate the volume) so startup can create the 1.0 columns, then run `quickSearchFullReindex` once to rebuild the index behind its alias. On startup, the full demo creates its remaining PostgreSQL qbit tables from the registered metadata and checks every expected column and type. The hand-maintained domain, OAuth, and role tables stay under `infra/postgres`. If a persisted qbit table is missing a required column or has a different type, startup fails with a migration/reset message instead of silently accepting it. The full-profile RabbitMQ completion-event smoke requires QQQ's process publishing and permission-scoped ESB message endpoint; the broker management smoke below can run independently. diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java index 055e638..43596db 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBits.java @@ -6,7 +6,10 @@ */ package com.kingsrook.qqq.all; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; +import java.util.Locale; import com.kingsrook.qbits.customizabletableviews.CustomizableTableViewsQBitConfig; import com.kingsrook.qbits.customizabletableviews.CustomizableTableViewsQBitProducer; import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; @@ -148,10 +151,9 @@ final class DemoQBits static void addFull(QInstance instance, FullProfileSettings settings) throws QException { - String scheme = Boolean.parseBoolean(settings.getOrDefault("OPENSEARCH_SSL", "false")) ? "https" : "http"; QuickSearchQBitConfig searchConfig = new QuickSearchQBitConfig() .withBackendName("postgres") - .withOpensearchUrl(scheme + "://" + settings.get("OPENSEARCH_HOST") + ":" + settings.port("OPENSEARCH_PORT", "9200")) + .withOpensearchUrl(opensearchUrl(settings)) .withOpensearchIndexName(settings.getOrDefault("OPENSEARCH_INDEX", "qqq-all-customers")) .withSchedulerName("demoScheduler") .withStartupMode(quickSearchStartupMode(settings)) @@ -189,17 +191,32 @@ final class DemoQBits })).produce(instance, "demo-import"); } + /** URI adds the brackets an IPv6 literal host needs. */ + static String opensearchUrl(FullProfileSettings settings) + { + String scheme = Boolean.parseBoolean(settings.getOrDefault("OPENSEARCH_SSL", "false")) ? "https" : "http"; + String host = settings.get("OPENSEARCH_HOST"); + try + { + return new URI(scheme, null, host, settings.port("OPENSEARCH_PORT", "9200"), null, null, null).toString(); + } + catch(URISyntaxException e) + { + throw new IllegalArgumentException("Invalid host in QQQ_ALL_OPENSEARCH_HOST [" + host + "]", e); + } + } + /** FAIL_FAST (the qbit default) stops startup when OpenSearch is unreachable; DEGRADED boots and retries on first use. */ - private static QuickSearchStartupMode quickSearchStartupMode(FullProfileSettings settings) + static QuickSearchStartupMode quickSearchStartupMode(FullProfileSettings settings) { String mode = settings.getOrDefault("OPENSEARCH_STARTUP_MODE", QuickSearchStartupMode.FAIL_FAST.name()); try { - return QuickSearchStartupMode.valueOf(mode); + return QuickSearchStartupMode.valueOf(mode.trim().toUpperCase(Locale.ROOT)); } catch(IllegalArgumentException e) { - throw new IllegalArgumentException("QQQ_ALL_OPENSEARCH_STARTUP_MODE must be FAIL_FAST or DEGRADED"); + throw new IllegalArgumentException("QQQ_ALL_OPENSEARCH_STARTUP_MODE must be FAIL_FAST or DEGRADED; found [" + mode + "]", e); } } } diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java index e3b5afa..8545953 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java @@ -6,11 +6,13 @@ */ package com.kingsrook.qqq.all; +import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode; @@ -40,6 +42,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class FullQBitsTest { + private static final Path QUICK_SEARCH_GRANTS = Path.of("..", "infra", "postgres", "06-quick-search-permissions.sql"); + private static final Pattern PERMISSION_NAME = Pattern.compile("'(quickSearch[A-Za-z]*[.](?:hasAccess|read|write))'"); + @TempDir Path dataDirectory; @Test @@ -131,16 +136,35 @@ class FullQBitsTest } @Test - void quickSearchFailsFastWithoutOpenSearchUnlessDegraded() + void quickSearchBracketsAnIpv6OpenSearchHost() throws Exception + { + Map environment = new HashMap<>(FullProfileTest.environment()); + environment.put("QQQ_ALL_OPENSEARCH_HOST", "::1"); + QuickSearchQBitConfig search = quickSearch(new FullProfileApplication(dataDirectory, + FullProfileSettings.from(environment)).defineQInstance()); + + assertEquals("http://[::1]:1", search.getOpensearchUrl()); + } + + @Test + void quickSearchFailsFastWithoutOpenSearchUnlessDegraded() throws Exception { Map environment = new HashMap<>(FullProfileTest.environment()); environment.remove("QQQ_ALL_OPENSEARCH_STARTUP_MODE"); - assertThrows(QException.class, () -> new FullProfileApplication(dataDirectory, + QException failFast = assertThrows(QException.class, () -> new FullProfileApplication(dataDirectory, FullProfileSettings.from(environment)).defineQInstance()); + assertTrue(failFast.getMessage().contains("[qqq-all-customers]"), failFast.getMessage()); + assertTrue(failFast.getMessage().contains("startupMode=DEGRADED"), failFast.getMessage()); + + environment.put("QQQ_ALL_OPENSEARCH_STARTUP_MODE", " degraded "); + assertEquals(QuickSearchStartupMode.DEGRADED, quickSearch(new FullProfileApplication(dataDirectory, + FullProfileSettings.from(environment)).defineQInstance()).getStartupMode()); environment.put("QQQ_ALL_OPENSEARCH_STARTUP_MODE", "SOMETIMES"); - assertThrows(IllegalArgumentException.class, () -> new FullProfileApplication(dataDirectory, + IllegalArgumentException invalid = assertThrows(IllegalArgumentException.class, () -> new FullProfileApplication(dataDirectory, FullProfileSettings.from(environment)).defineQInstance()); + assertEquals("QQQ_ALL_OPENSEARCH_STARTUP_MODE must be FAIL_FAST or DEGRADED; found [SOMETIMES]", invalid.getMessage()); + assertNotNull(invalid.getCause()); } @Test @@ -161,10 +185,12 @@ class FullQBitsTest QContext.clear(); } - // The grants infra/postgres/06-quick-search-permissions.sql gives the admin role. - QContext.init(instance, new QSession().withPermissions("quickSearchAdmin.hasAccess", - "quickSearchIndex.read", "quickSearchIndex.write", "quickSearchIndexRun.read", "quickSearchIndexRun.write", - "quickSearchFailedEvent.read", "quickSearchFailedEvent.write")); + // Exactly the permissions the grant file creates and gives the admin role. + String grants = Files.readString(QUICK_SEARCH_GRANTS); + Integer roleGrantStart = grants.indexOf("INSERT INTO role_permission_int"); + Set created = permissionNames(grants.substring(0, roleGrantStart)); + assertEquals(created, permissionNames(grants.substring(roleGrantStart))); + QContext.init(instance, new QSession().withPermissions(created.toArray(String[]::new))); try { for(String processName : Set.of("quickSearchFullReindex", "quickSearchReconcileIndex", "quickSearchBasepullIndex")) @@ -227,6 +253,11 @@ class FullQBitsTest } } + private static Set permissionNames(String sql) + { + return PERMISSION_NAME.matcher(sql).results().map(match -> match.group(1)).collect(Collectors.toSet()); + } + private static QuickSearchQBitConfig quickSearch(QInstance instance) { return (QuickSearchQBitConfig) instance.getQBits().values().stream() -- 2.54.0