CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project are documented here. The format follows Keep
 Requires QQQ 4.1.0-RC.8 or later (central health checks, help bundles and the spec-managed UI API extension contract).
 
 ### Fixed
+- Indexing bookkeeping is no longer record-audited: the `quick_search_index` runtime fields written by basepull, full reindex and real-time indexing (status, last basepull time, document count, last run, real-time error count) and the settings row's priming outcome (`lastPrimedAt`, summary) are written with `omitDmlAudit`. On a production host these wrote about 470 "Record was Edited during process: Update search index" audit rows an hour. Admin edits to an index, its fields or the settings are still audited.
 - Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write.
 
 ### Added

src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java

@@ -412,8 +412,12 @@ public class QuickSearchConfigPrimer
       }
       else
       {
+         ///////////////////////////////////////////////////////////////////////
+         // Only the priming outcome (lastPrimedAt, summary) changes here, on  //
+         // every startup: bookkeeping, not a settings change, so not audited. //
+         ///////////////////////////////////////////////////////////////////////
          values.withValue("id", rows.get(0).getValue("id"));
-         updateAll(settingsTable, List.of(values));
+         updateAll(settingsTable, List.of(values), true);
       }
    }
 
@@ -477,6 +481,17 @@ public class QuickSearchConfigPrimer
     ** Update rows, failing on any record error.
     *******************************************************************************/
    public static void updateAll(String tableName, List<QRecord> records) throws QException
+   {
+      updateAll(tableName, records, false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update rows, failing on any record error; omitAudit skips the DML audit
+    ** for runtime bookkeeping writes.
+    *******************************************************************************/
+   public static void updateAll(String tableName, List<QRecord> records, boolean omitAudit) throws QException
    {
       if(records.isEmpty())
       {
@@ -485,6 +500,7 @@ public class QuickSearchConfigPrimer
       UpdateInput updateInput = new UpdateInput();
       updateInput.setTableName(tableName);
       updateInput.setRecords(records);
+      updateInput.setOmitDmlAudit(omitAudit);
       for(QRecord record : CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords()))
       {
          if(CollectionUtils.nullSafeHasContents(record.getErrors()))

src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java

@@ -334,6 +334,8 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte
             updateInput.setTableName(indexTable);
             updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id"))
                .withValue("realTimeErrorCount", Math.addExact(previous == null ? 0 : previous, entry.getValue()))));
+            // A runtime counter, not a configuration change: not record-audited.
+            updateInput.setOmitDmlAudit(true);
             List<QRecord> updated = QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)).getRecords();
             if(CollectionUtils.nullSafeIsEmpty(updated) || CollectionUtils.nullSafeHasContents(updated.get(0).getErrors()))
             {

src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java

@@ -493,7 +493,10 @@ public abstract class AbstractIndexingStep implements BackendStep
 
 
    /*******************************************************************************
-    ** Update fields on a quick_search_index row.
+    ** Update runtime state on a quick_search_index row (status, last run, last
+    ** basepull time, document count). Indexing bookkeeping, not a configuration
+    ** change: it is not record-audited, or every basepull tick would write an
+    ** audit row. Admin edits to the row go through the table and stay audited.
     *******************************************************************************/
    protected void updateIndexRow(Integer indexId, Map<String, ? extends Serializable> values) throws QException
    {
@@ -506,6 +509,7 @@ public abstract class AbstractIndexingStep implements BackendStep
       UpdateInput updateInput = new UpdateInput();
       updateInput.setTableName(getConfig().getQuickSearchIndexTableName());
       updateInput.setRecords(List.of(record));
+      updateInput.setOmitDmlAudit(true);
       QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput));
    }
 

src/test/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimerTest.java

@@ -21,7 +21,9 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -77,6 +79,38 @@ class QuickSearchConfigPrimerTest extends BaseDataDrivenTest
 
 
 
+   /*******************************************************************************
+    ** Re-priming only records the priming outcome on the settings row: that is
+    ** not audited. An admin settings edit still is.
+    *******************************************************************************/
+   @Test
+   void testRepriming_settingsBookkeepingIsNotAudited() throws QException
+   {
+      new AuditsMetaDataProvider().defineAll(QContext.getQInstance(), TEST_BACKEND_NAME, null);
+      new QuickSearchConfigPrimer(runtime).prime();
+      long afterFirst = settingsEditAudits();
+      new QuickSearchConfigPrimer(runtime).prime();
+      assertThat(settingsEditAudits()).as("re-priming is not audited").isEqualTo(afterFirst);
+
+      new UpdateAction().execute(new UpdateInput().withTableName(QuickSearchSettings.TABLE_NAME)
+         .withRecord(new QRecord().withValue("id", QuickSearchSettings.SETTINGS_ID).withValue("indexingPaused", true)));
+      assertThat(settingsEditAudits()).as("an admin settings edit is audited").isEqualTo(afterFirst + 1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Audit rows recording an edit of the settings row.
+    *******************************************************************************/
+   private long settingsEditAudits() throws QException
+   {
+      return (rows(AuditsMetaDataProvider.TABLE_NAME_AUDIT).stream()
+         .filter(audit -> audit.getValueString("message") != null && audit.getValueString("message").contains("Edited"))
+         .count());
+   }
+
+
+
    /*******************************************************************************
     ** Code defaults update rows nobody edited, and never rows an owner edited.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -324,6 +325,45 @@ class BasepullIndexStepTest extends BaseQuickSearchTest
 
 
 
+   /*******************************************************************************
+    ** Basepull bookkeeping on the index row (status, lastBasepullTime,
+    ** documentCount, last run) writes no audit; an admin edit of the same row
+    ** is still audited.
+    *******************************************************************************/
+   @Test
+   void testBasepullBookkeeping_isNotAudited_adminEditIs() throws QException
+   {
+      if(QContext.getQInstance().getTable(AuditsMetaDataProvider.TABLE_NAME_AUDIT) == null)
+      {
+         new AuditsMetaDataProvider().defineAll(QContext.getQInstance(), TEST_BACKEND_NAME, null);
+      }
+      insertTestEntities(3);
+      QRecord row = insertIndexRow(TEST_ENTITY_TABLE, true, null, 5);
+      when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenReturn(3L);
+
+      new BasepullIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
+      assertThat(queryAll(QuickSearchIndex.TABLE_NAME).get(0).getValueInteger("documentCount")).isEqualTo(3);
+      assertThat(indexRowEditAudits()).as("basepull bookkeeping is not audited").isZero();
+
+      new UpdateAction().execute(new UpdateInput().withTableName(QuickSearchIndex.TABLE_NAME)
+         .withRecord(new QRecord().withValue("id", row.getValue("id")).withValue("basepullIntervalMinutes", 15)));
+      assertThat(indexRowEditAudits()).as("an admin configuration edit is audited").isEqualTo(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Audit rows recording an edit of a quick_search_index row.
+    *******************************************************************************/
+   private long indexRowEditAudits() throws QException
+   {
+      return (queryAll(AuditsMetaDataProvider.TABLE_NAME_AUDIT).stream()
+         .filter(audit -> audit.getValueString("message") != null && audit.getValueString("message").contains("Edited"))
+         .count());
+   }
+
+
+
    /*******************************************************************************
     ** Test: no discovered tables, no quickSearchIndex rows - step completes
     ** without calling the OpenSearch client.