fix(#28): index display values for real-time inserts

A real-time INSERT was indexed from the event's records, which hold raw
values: a foreign key was stored as its id and the record label was built
from raw values, until the next basepull corrected the document.

The listener now re-reads inserted records the way it re-reads updated
ones: one query per event, inside the write's transaction, with display
values and possible-value translation. A failed re-read records the
inserted keys as failed events after commit and never fails the write,
as for updates (#13).

Closes #28

src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java

@@ -51,18 +51,19 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** Real-time indexing hook. Registered once per QInstance; applies to the
  ** tables this QBit indexes.
  **
- ** Events are built while the write's transaction is open (so updates can be
- ** re-read in full), but published only after that transaction commits, via
+ ** Events are built while the write's transaction is open (so inserted and
+ ** updated records can be re-read in full, with display values), but
+ ** published only after that transaction commits, via
  ** QBackendTransaction.addAfterCommitCallback. A write without a transaction
  ** is already committed, so it publishes immediately. A rolled-back write never
  ** reaches the index.
  **
- ** This listener never fails the host's write. When the re-read of updated
- ** records or publishing fails (including a LinkageError from a missing
- ** optional library while the client is built), each affected (table, record,
- ** action) is written to quickSearchFailedEvent for the scheduled basepull to
- ** replay. A failed re-read is recorded after the commit too, so a rolled-back
- ** write leaves no row.
+ ** This listener never fails the host's write. When the re-read of inserted
+ ** or updated records or publishing fails (including a LinkageError from a
+ ** missing optional library while the client is built), each affected (table,
+ ** record, action) is written to quickSearchFailedEvent for the scheduled
+ ** basepull to replay. A failed re-read is recorded after the commit too, so a
+ ** rolled-back write leaves no row.
  **
  ** While a full reindex rebuilds into a new physical index, the alias still
  ** points at the old one, so a published delete never reaches the new index.
@@ -115,8 +116,7 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
       switch(event.getType())
       {
-         case INSERT -> indexEvents.addAll(toIndexEvents(tableName, primaryKeyField, event.getRecords()));
-         case UPDATE ->
+         case INSERT, UPDATE ->
          {
             try
             {
@@ -127,11 +127,11 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
                ///////////////////////////////////////////////////////////////////////
                // log now: with a transaction, nothing else is logged until commit, //
                // and a rollback (a transaction the failed re-read aborted, too)    //
-               // would leave no trace. Then record the updated keys after commit,  //
+               // would leave no trace. Then record the written keys after commit,  //
                // so basepull re-reads and indexes them.                            //
                ///////////////////////////////////////////////////////////////////////
-               LOG.warn("Could not re-read updated records for indexing; recording them for replay once the write commits", e,
-                  logPair("tableName", tableName), logPair("recordCount", event.getRecords().size()));
+               LOG.warn("Could not re-read inserted or updated records for indexing; recording them for replay once the write commits", e,
+                  logPair("tableName", tableName), logPair("changeType", event.getType()), logPair("recordCount", event.getRecords().size()));
                List<IndexEvent> failedEvents = toIndexEvents(tableName, primaryKeyField, event.getRecords());
                runAfterCommit(event, () -> recordFailures(runtime, failedEvents, e));
                return;
@@ -444,8 +444,9 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
 
 
    /*******************************************************************************
-    ** Updated records are "as the backend returned them", which for RDBMS is the
-    ** sparse input. Re-read the full rows inside the same transaction, with
+    ** Changed records are "as the backend returned them": for an update on
+    ** RDBMS that is the sparse input, and an insert has raw values only. Re-read
+    ** the full rows inside the same transaction, in one query per event, with
     ** display values so possible-value labels are indexed (which can run host
     ** possible-value providers).
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/quicksearch/IndexDriftIntegrationTest.java

@@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
@@ -64,8 +65,8 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 /*******************************************************************************
  ** Integration tests, against a real OpenSearch, that real-time indexing keeps
- ** the index in step with the source table through updates and deletes, and
- ** that the reconcile process repairs an index that has drifted.
+ ** the index in step with the source table through inserts, updates and
+ ** deletes, and that the reconcile process repairs an index that has drifted.
  **
  ** Each test uses its own record IDs and search terms, so tests are independent.
  *******************************************************************************/
@@ -75,6 +76,7 @@ class IndexDriftIntegrationTest
 {
    private static final String BACKEND_NAME = "driftMemoryBackend";
    private static final String TABLE_NAME   = "driftCustomer";
+   private static final String COMPANY_NAME = "driftCompany";
    private static final String INDEX_NAME   = "quick_search_drift_test";
 
    @Container
@@ -98,6 +100,9 @@ class IndexDriftIntegrationTest
 
       @QuickSearchField
       private String city;
+
+      @QuickSearchField
+      private Integer companyId;
    }
 
 
@@ -118,13 +123,23 @@ class IndexDriftIntegrationTest
       qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
          .withName("anonymous")
          .withType(QAuthenticationType.FULLY_ANONYMOUS));
+      qInstance.addTable(new QTableMetaData()
+         .withName(COMPANY_NAME)
+         .withBackendName(BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withRecordLabelFormatAndFields("%s", "name")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("name", QFieldType.STRING)));
+      qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(COMPANY_NAME));
       qInstance.addTable(new QTableMetaData()
          .withName(TABLE_NAME)
          .withBackendName(BACKEND_NAME)
          .withPrimaryKeyField("id")
+         .withRecordLabelFormatAndFields("%s (%s)", "name", "companyId")
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("name", QFieldType.STRING))
-         .withField(new QFieldMetaData("city", QFieldType.STRING)));
+         .withField(new QFieldMetaData("city", QFieldType.STRING))
+         .withField(new QFieldMetaData("companyId", QFieldType.INTEGER).withPossibleValueSourceName(COMPANY_NAME)));
 
       QuickSearchQBitConfig config = new QuickSearchQBitConfig()
          .withBackendName(BACKEND_NAME)
@@ -154,6 +169,27 @@ class IndexDriftIntegrationTest
 
 
 
+   /*******************************************************************************
+    ** Test: a just-inserted record is indexed with its possible-value labels, in
+    ** its searchable text and its record label, not with the raw foreign key.
+    *******************************************************************************/
+   @Test
+   void testInsert_indexesPossibleValueLabels() throws Exception
+   {
+      InsertInput companyInsert = new InsertInput();
+      companyInsert.setTableName(COMPANY_NAME);
+      companyInsert.setRecords(List.of(new QRecord().withValue("id", 1).withValue("name", "Rocketworks")));
+      new InsertAction().execute(companyInsert);
+
+      insert(new QRecord().withValue("id", 31).withValue("name", "Cyrus").withValue("city", "Labelton").withValue("companyId", 1));
+
+      List<QuickSearchResult> results = search("rocketworks");
+      assertThat(results).extracting(QuickSearchResult::getRecordId).containsExactly("31");
+      assertThat(results.get(0).getRecordLabel()).isEqualTo("Cyrus (Rocketworks)");
+   }
+
+
+
    /*******************************************************************************
     ** Test: a partial update (one field) keeps the other fields searchable.
     *******************************************************************************/
@@ -268,15 +304,24 @@ class IndexDriftIntegrationTest
     *******************************************************************************/
    private List<String> searchRecordIds(String term) throws QException
    {
-      client().refreshIndex();
-      return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME))
-         .getResults().stream()
+      return (search(term).stream()
          .map(QuickSearchResult::getRecordId)
          .toList());
    }
 
 
 
+   /*******************************************************************************
+    ** Refresh the index, then return the search results matching the term.
+    *******************************************************************************/
+   private List<QuickSearchResult> search(String term) throws QException
+   {
+      client().refreshIndex();
+      return (new QuickSearchAction().execute(new QuickSearchInput().withSearchTerm(term).withTableName(TABLE_NAME)).getResults());
+   }
+
+
+
    /*******************************************************************************
     ** Refresh the index, then return the record IDs of the documents matching
     ** the term, straight from OpenSearch (no record-security post-filter).

src/test/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListenerTest.java

@@ -37,6 +37,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest;
 import com.kingsrook.qbits.quicksearch.MissingLibraryTransportCustomizer;
@@ -46,7 +52,9 @@ import com.kingsrook.qbits.quicksearch.QuickSearchRuntime;
 import com.kingsrook.qbits.quicksearch.QuickSearchableTableConfig;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchFailedEvent;
 import com.kingsrook.qbits.quicksearch.model.QuickSearchIndex;
+import com.kingsrook.qbits.quicksearch.opensearch.OpenSearchDocument;
 import com.kingsrook.qbits.quicksearch.processes.AbstractIndexingStep;
+import com.kingsrook.qbits.quicksearch.processes.IndexingUtils;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEvent;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventAction;
 import com.kingsrook.qbits.quicksearch.publisher.IndexEventPublisher;
@@ -171,6 +179,59 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
 
 
 
+   @Test
+   void testInsert_indexesDisplayValuesAndPossibleValueLabels() throws QException
+   {
+      ///////////////////////////////////////////////////////////////////////
+      // the inserted record holds the raw categoryId (1); its document    //
+      // must carry the possible-value label, as an updated record's does //
+      ///////////////////////////////////////////////////////////////////////
+      addCategoryPossibleValueField();
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TEST_ENTITY_TABLE);
+      insertInput.setRecords(List.of(new QRecord().withValue("name", "widget").withValue("description", "d").withValue("categoryId", 1)));
+      new InsertAction().execute(insertInput);
+
+      ArgumentCaptor<List<IndexEvent>> captor = ArgumentCaptor.forClass(List.class);
+      verify(publisher, times(1)).publishIndexEvents(captor.capture());
+      OpenSearchDocument document = IndexingUtils.buildDocument(captor.getValue().get(0).getRecord(), new QuickSearchableTableConfig()
+         .withTableName(TEST_ENTITY_TABLE)
+         .withPrimaryKeyField("id")
+         .withSearchableFields(List.of("name", "categoryId")));
+
+      assertThat(document.getFieldValues()).containsEntry("categoryId", "Gadgets");
+      assertThat(document.getSearchableText()).isEqualTo("widget Gadgets");
+      assertThat(document.getRecordLabel()).isEqualTo("widget - Gadgets");
+   }
+
+
+
+   @Test
+   void testInsert_reReadFails_recordsFailedEventsAfterCommit() throws QException
+   {
+      QBackendTransaction transaction = new QBackendTransaction();
+      try(MockedConstruction<QueryAction> ignored = failingQueries(new QException("possible-value backend unavailable")))
+      {
+         assertThatCode(() -> new QuickSearchRecordChangeListener().onRecordsChanged(changeEvent(RecordChangeType.INSERT, transaction, 7, 8))).doesNotThrowAnyException();
+      }
+
+      assertThat(queryFailedEvents()).isEmpty();
+      transaction.commit();
+
+      List<QRecord> failed = queryFailedEvents();
+      assertThat(failed).extracting(r -> r.getValueString("recordId")).containsExactlyInAnyOrder("7", "8");
+      assertThat(failed).allSatisfy(r ->
+      {
+         assertThat(r.getValueString("action")).isEqualTo("INDEX");
+         assertThat(r.getValueString("status")).isEqualTo(QuickSearchFailedEvent.STATUS_PENDING);
+         assertThat(r.getValueString("errorMessage")).contains("possible-value backend unavailable");
+      });
+      verify(publisher, never()).publishIndexEvents(anyList());
+   }
+
+
+
    @Test
    void testUpdate_reReadsFullRecord() throws QException
    {
@@ -354,7 +415,7 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
       assertThat(collectingLogger.getCollectedMessages()).anySatisfy(m ->
       {
          assertThat(m.getLevel()).isEqualTo(Level.WARN);
-         assertThat(m.getMessage()).contains("Could not re-read updated records").contains(TEST_ENTITY_TABLE).contains("\"recordCount\":2").doesNotContain("changed");
+         assertThat(m.getMessage()).contains("Could not re-read inserted or updated records").contains(TEST_ENTITY_TABLE).contains("\"recordCount\":2").doesNotContain("changed");
          assertThat(m.getMessage()).contains("possible-value backend unavailable");
       });
    }
@@ -408,7 +469,7 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
          .anySatisfy(m ->
          {
             assertThat(m.getLevel()).isEqualTo(Level.WARN);
-            assertThat(m.getMessage()).contains("Could not re-read updated records").contains(TEST_ENTITY_TABLE).doesNotContain("changed");
+            assertThat(m.getMessage()).contains("Could not re-read inserted or updated records").contains(TEST_ENTITY_TABLE).doesNotContain("changed");
          })
          .anySatisfy(m ->
          {
@@ -470,13 +531,39 @@ class QuickSearchRecordChangeListenerTest extends BaseQuickSearchTest
 
 
    private RecordChangeEvent updateEvent(QBackendTransaction transaction, Integer... ids)
+   {
+      return (changeEvent(RecordChangeType.UPDATE, transaction, ids));
+   }
+
+
+
+   private RecordChangeEvent changeEvent(RecordChangeType type, QBackendTransaction transaction, Integer... ids)
    {
       List<QRecord> records = new ArrayList<>();
       for(Integer id : ids)
       {
          records.add(new QRecord().withValue("id", id).withValue("description", "changed"));
       }
-      return (new RecordChangeEvent().withTableName(TEST_ENTITY_TABLE).withType(RecordChangeType.UPDATE).withRecords(records).withTransaction(transaction));
+      return (new RecordChangeEvent().withTableName(TEST_ENTITY_TABLE).withType(type).withRecords(records).withTransaction(transaction));
+   }
+
+
+
+   /*******************************************************************************
+    ** Give the test table a categoryId field backed by an enum possible-value
+    ** source (1 is "Gadgets"), and a record label that includes it.
+    *******************************************************************************/
+   private void addCategoryPossibleValueField()
+   {
+      QInstance qInstance = QContext.getQInstance();
+      qInstance.addPossibleValueSource(new QPossibleValueSource()
+         .withName("testCategory")
+         .withType(QPossibleValueSourceType.ENUM)
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
+         .withEnumValues(List.of(new QPossibleValue<>(1, "Gadgets"))));
+      qInstance.getTable(TEST_ENTITY_TABLE)
+         .withField(new QFieldMetaData("categoryId", QFieldType.INTEGER).withPossibleValueSourceName("testCategory"))
+         .withRecordLabelFormatAndFields("%s - %s", "name", "categoryId");
    }