diff --git a/CHANGELOG.md b/CHANGELOG.md index b78b611..eefe7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ Requires QQQ 4.1. Closes every blocker of the 2026-10-04 production-readiness au - An update was dropped without a `quickSearchFailedEvent` row when the listener's re-read failed or the optional AWS SDK was missing (`LinkageError`); both are now recorded for basepull, and a `DEGRADED` start without the AWS SDK no longer aborts host startup (#13). - Search pages overlapped under record locks, per-table mode ignored `offset`, and a large `maxSearchLimit` could exceed the 10,000 result window (#16). - A failing document count was swallowed and cleared `documentCount`; it is now logged at warn and the previous count is kept (#17). +- A real-time insert was indexed with raw values (a foreign key as its id, the record label as "2 - email") until the next basepull. Inserts are now re-read in the write's transaction with display values and possible-value labels, as updates are; a failed re-read is recorded in `quickSearchFailedEvent` for basepull and never fails the insert (#28). ## [0.2.0] - 2026-03-30 - Config-driven table indexing (`SearchableTableConfig`). diff --git a/docs/MIGRATION-1.0.md b/docs/MIGRATION-1.0.md index 2eb6971..cfe60d8 100644 --- a/docs/MIGRATION-1.0.md +++ b/docs/MIGRATION-1.0.md @@ -20,6 +20,7 @@ - Real-time indexing happens after commit. Code that searched for a record inside the same transaction that inserted it will not find it until commit. - Real-time failures no longer vanish: they land in `quickSearchFailedEvent` and are replayed by the next basepull. Review that table after an OpenSearch outage. +- Inserted and updated records are re-read inside the write's transaction, with display values and possible-value labels, before they are indexed. That is one extra query per write batch on an indexed table, and it runs the host's possible-value providers. If the re-read fails, the write still succeeds and the records are replayed by basepull. - A table without the change-detection field (`modifyDate` by default) is indexed on the first basepull and thereafter only by reconcile. - `limit` is capped (default 100), `offset + limit` at 10,000, and search terms must be 2 to 100 characters. - Query semantics changed from OR over n-grams to AND over whole terms with prefix matching: "blue widget" now requires both words. diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java b/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java index a357f09..865c95e 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java @@ -51,14 +51,20 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; ** Real-time indexing hook. Registered once per QInstance; applies to the ** tables this QBit indexes. ** - ** Events are built while the write's transaction is open (so updates can be - ** re-read in full), but published only after that transaction commits, via + ** Events are built while the write's transaction is open, but published only + ** after that transaction commits, via ** QBackendTransaction.addAfterCommitCallback. A write without a transaction ** is already committed, so it publishes immediately. A rolled-back write never ** reaches the index. ** - ** This listener never fails the host's write. When the re-read of updated - ** records or publishing fails (including a LinkageError from a missing + ** Inserted and updated records are re-read inside that transaction, with + ** display values and possible-value labels, so a real-time document matches + ** the one basepull, reconcile and full reindex build. The event's own records + ** are not enough: for an update they are the sparse input, and for an insert + ** they carry only raw values. + ** + ** This listener never fails the host's write. When the re-read of inserted or + ** updated records or publishing fails (including a LinkageError from a missing ** optional library while the client is built), each affected (table, record, ** action) is written to quickSearchFailedEvent for the scheduled basepull to ** replay. A failed re-read is recorded after the commit too, so a rolled-back @@ -115,8 +121,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte switch(event.getType()) { - case INSERT -> indexEvents.addAll(toIndexEvents(tableName, primaryKeyField, event.getRecords())); - case UPDATE -> + case INSERT, UPDATE -> { try { @@ -127,10 +132,11 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte /////////////////////////////////////////////////////////////////////// // log now: with a transaction, nothing else is logged until commit, // // and a rollback (a transaction the failed re-read aborted, too) // - // would leave no trace. Then record the updated keys after commit, // + // would leave no trace. Then record the written keys after commit, // // so basepull re-reads and indexes them. // /////////////////////////////////////////////////////////////////////// - LOG.warn("Could not re-read updated records for indexing; recording them for replay once the write commits", e, + LOG.warn("Could not re-read " + (event.getType() == RecordChangeType.INSERT ? "inserted" : "updated") + + " records for indexing; recording them for replay once the write commits", e, logPair("tableName", tableName), logPair("recordCount", event.getRecords().size())); List failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords()); runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e)); @@ -444,8 +450,9 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte /******************************************************************************* - ** Updated records are "as the backend returned them", which for RDBMS is the - ** sparse input. Re-read the full rows inside the same transaction, with + ** The event's records are "as the backend returned them": for an update on + ** RDBMS that is the sparse input, and for an insert the raw values with no + ** display values. Re-read the full rows inside the same transaction, with ** display values so possible-value labels are indexed (which can run host ** possible-value providers). *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/IndexDriftIntegrationTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/IndexDriftIntegrationTest.java index e97e89f..6aae72b 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/IndexDriftIntegrationTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/IndexDriftIntegrationTest.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; @@ -64,7 +65,8 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* ** Integration tests, against a real OpenSearch, that real-time indexing keeps - ** the index in step with the source table through updates and deletes, and + ** the index in step with the source table through inserts (with display + ** values), updates and deletes, and ** that the reconcile process repairs an index that has drifted. ** ** Each test uses its own record IDs and search terms, so tests are independent. @@ -75,6 +77,7 @@ class IndexDriftIntegrationTest { private static final String BACKEND_NAME = "driftMemoryBackend"; private static final String TABLE_NAME = "driftCustomer"; + private static final String REGION_TABLE = "driftRegion"; private static final String INDEX_NAME = "quick_search_drift_test"; @Container @@ -98,6 +101,9 @@ class IndexDriftIntegrationTest @QuickSearchField private String city; + + @QuickSearchField + private Integer regionId; } @@ -124,7 +130,19 @@ class IndexDriftIntegrationTest .withPrimaryKeyField("id") .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) .withField(new QFieldMetaData("name", QFieldType.STRING)) - .withField(new QFieldMetaData("city", QFieldType.STRING))); + .withField(new QFieldMetaData("city", QFieldType.STRING)) + .withField(new QFieldMetaData("regionId", QFieldType.INTEGER).withPossibleValueSourceName(REGION_TABLE)) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("name", "regionId")); + qInstance.addTable(new QTableMetaData() + .withName(REGION_TABLE) + .withBackendName(BACKEND_NAME) + .withPrimaryKeyField("id") + .withRecordLabelFormat("%s") + .withRecordLabelFields("name") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING))); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(REGION_TABLE)); QuickSearchQBitConfig config = new QuickSearchQBitConfig() .withBackendName(BACKEND_NAME) @@ -154,6 +172,33 @@ class IndexDriftIntegrationTest + /******************************************************************************* + ** Test: a just-inserted record's document carries display values: the + ** possible-value label of a foreign key is searchable and is in the record + ** label, with no basepull in between (#28). + *******************************************************************************/ + @Test + void testInsert_documentCarriesDisplayValuesAndPossibleValueLabel() throws Exception + { + InsertInput regionInsert = new InsertInput(); + regionInsert.setTableName(REGION_TABLE); + regionInsert.setRecords(List.of(new QRecord().withValue("id", 7).withValue("name", "Northwind Territory"))); + new InsertAction().execute(regionInsert); + + insert(new QRecord().withValue("id", 31).withValue("name", "Rhoda").withValue("city", "Labelburg").withValue("regionId", 7)); + + client().refreshIndex(); + List results = new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm("northwind").withTableName(TABLE_NAME)).getResults(); + assertThat(results).as("the region's label, not its key, is indexed").extracting(QuickSearchResult::getRecordId).containsExactly("31"); + assertThat(results.get(0).getRecordLabel()).isEqualTo("Rhoda - Northwind Territory"); + + Map fieldValues = client().search("labelburg", List.of(TABLE_NAME), 10, 0, QuickSearchQBitContext.getDiscoveredTables()) + .hits().hits().get(0).source().getFieldValues(); + assertThat(fieldValues).containsEntry("regionId", "Northwind Territory"); + } + + + /******************************************************************************* ** Test: a partial update (one field) keeps the other fields searchable. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListenerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListenerTest.java index 76f6a28..2b18079 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListenerTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListenerTest.java @@ -37,7 +37,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; import com.kingsrook.qbits.quicksearch.MissingLibraryTransportCustomizer; import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; @@ -74,6 +78,8 @@ import static org.mockito.Mockito.when; *******************************************************************************/ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest { + private static final String COMPANY_TABLE = "company"; + private IndexEventPublisher publisher; private QuickSearchRuntime runtime; @@ -100,6 +106,25 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest config.setRuntime(runtime); QInstance qInstance = QContext.getQInstance(); + + /////////////////////////////////////////////////////////////////// + // a foreign key to a company table, through a table PVS, and a // + // record label over it, so display values can be asserted // + /////////////////////////////////////////////////////////////////// + qInstance.addTable(new QTableMetaData() + .withName(COMPANY_TABLE) + .withBackendName(TEST_BACKEND_NAME) + .withPrimaryKeyField("id") + .withRecordLabelFormat("%s") + .withRecordLabelFields("name") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING))); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(COMPANY_TABLE)); + qInstance.getTable(TEST_ENTITY_TABLE) + .withField(new QFieldMetaData("companyId", QFieldType.INTEGER).withPossibleValueSourceName(COMPANY_TABLE)) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("name", "companyId"); + qInstance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("quick-search").withVersion("test").withConfig(config)); qInstance.withRecordChangeListener(new QCodeReference(QuickSearchRecordChangeListener.class)); } @@ -145,6 +170,79 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest + @Test + void testInsert_indexesDisplayValuesAndPossibleValueLabels() throws QException + { + InsertInput companyInsert = new InsertInput(); + companyInsert.setTableName(COMPANY_TABLE); + companyInsert.setRecords(List.of(new QRecord().withValue("name", "Acme Corp"))); + Integer companyId = new InsertAction().execute(companyInsert).getRecords().get(0).getValueInteger("id"); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TEST_ENTITY_TABLE); + insertInput.setRecords(List.of(new QRecord().withValue("name", "widget").withValue("description", "d").withValue("companyId", companyId))); + new InsertAction().execute(insertInput); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(publisher, times(1)).publishIndexEvents(captor.capture()); + QRecord indexed = captor.getValue().get(0).getRecord(); + assertThat(indexed.getValueInteger("companyId")).isEqualTo(companyId); + assertThat(indexed.getDisplayValue("companyId")).as("the PVS label, not the raw key").isEqualTo("Acme Corp"); + assertThat(indexed.getRecordLabel()).isEqualTo("widget - Acme Corp"); + } + + + + @Test + void testInsert_reReadFails_recordsFailedEventsAfterCommit() throws QException + { + QBackendTransaction transaction = new QBackendTransaction(); + try(MockedConstruction ignored = failingQueries(new QException("possible-value backend unavailable"))) + { + assertThatCode(() -> new QuickSearchRecordChangeListener().onRecordsChanged(changeEvent(RecordChangeType.INSERT, transaction, 7, 8))).doesNotThrowAnyException(); + } + + assertThat(queryFailedEvents()).isEmpty(); + transaction.commit(); + + List failed = queryFailedEvents(); + assertThat(failed).extracting(r -> r.getValueString("recordId")).containsExactlyInAnyOrder("7", "8"); + assertThat(failed).allSatisfy(r -> + { + assertThat(r.getValueString("action")).isEqualTo("INDEX"); + assertThat(r.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING); + assertThat(r.getValueString("errorMessage")).contains("possible-value backend unavailable"); + }); + verify(publisher, never()).publishIndexEvents(anyList()); + } + + + + @Test + void testInsert_reReadFailsThenRollback_logsCauseAndRecordsNothing() throws QException + { + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QuickSearchRecordChangeListener.class); + QBackendTransaction transaction = new QBackendTransaction(); + try(MockedConstruction ignored = failingQueries(new NoClassDefFoundError("com/example/pvs/Provider"))) + { + new QuickSearchRecordChangeListener().onRecordsChanged(changeEvent(RecordChangeType.INSERT, transaction, 7)); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(QuickSearchRecordChangeListener.class); + } + transaction.rollback(); + + assertThat(queryFailedEvents()).isEmpty(); + assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m -> + { + assertThat(m.getLevel()).isEqualTo(Level.WARN); + assertThat(m.getMessage()).contains("Could not re-read inserted records").contains(TEST_ENTITY_TABLE).contains("com/example/pvs/Provider"); + }); + } + + + @Test void testInsert_withTransaction_publishesOnlyAfterCommit() throws QException { @@ -470,13 +568,20 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest private RecordChangeEvent updateEvent(QBackendTransaction transaction, Integer... ids) + { + return (changeEvent(RecordChangeType.UPDATE, transaction, ids)); + } + + + + private RecordChangeEvent changeEvent(RecordChangeType type, QBackendTransaction transaction, Integer... ids) { List records = new ArrayList<>(); for(Integer id : ids) { records.add(new QRecord().withValue("id", id).withValue("description", "changed")); } - return (new RecordChangeEvent().withTableName(TEST_ENTITY_TABLE).withType(RecordChangeType.UPDATE).withRecords(records).withTransaction(transaction)); + return (new RecordChangeEvent().withTableName(TEST_ENTITY_TABLE).withType(type).withRecords(records).withTransaction(transaction)); }