diff --git a/CHANGELOG.md b/CHANGELOG.md index d117231..5606977 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project are documented here. The format follows Keep Requires QQQ 4.1.0-RC.8 or later (central health checks, help bundles and the spec-managed UI API extension contract). ### Fixed +- Indexing bookkeeping is no longer record-audited: the `quick_search_index` runtime fields written by basepull, full reindex and real-time indexing (status, last basepull time, document count, last run, real-time error count) and the settings row's priming outcome (`lastPrimedAt`, summary) are written with `omitDmlAudit`. On a production host these wrote about 470 "Record was Edited during process: Update search index" audit rows an hour. Admin edits to an index, its fields or the settings are still audited. - Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write. ### Added diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java b/src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java index bd19b74..2aa5001 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimer.java @@ -412,8 +412,12 @@ public class QuickSearchConfigPrimer } else { + /////////////////////////////////////////////////////////////////////// + // Only the priming outcome (lastPrimedAt, summary) changes here, on // + // every startup: bookkeeping, not a settings change, so not audited. // + /////////////////////////////////////////////////////////////////////// values.withValue("id", rows.get(0).getValue("id")); - updateAll(settingsTable, List.of(values)); + updateAll(settingsTable, List.of(values), true); } } @@ -477,6 +481,17 @@ public class QuickSearchConfigPrimer ** Update rows, failing on any record error. *******************************************************************************/ public static void updateAll(String tableName, List records) throws QException + { + updateAll(tableName, records, false); + } + + + + /******************************************************************************* + ** Update rows, failing on any record error; omitAudit skips the DML audit + ** for runtime bookkeeping writes. + *******************************************************************************/ + public static void updateAll(String tableName, List records, boolean omitAudit) throws QException { if(records.isEmpty()) { @@ -485,6 +500,7 @@ public class QuickSearchConfigPrimer UpdateInput updateInput = new UpdateInput(); updateInput.setTableName(tableName); updateInput.setRecords(records); + updateInput.setOmitDmlAudit(omitAudit); for(QRecord record : CollectionUtils.nonNullList(new UpdateAction().execute(updateInput).getRecords())) { if(CollectionUtils.nullSafeHasContents(record.getErrors())) diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java b/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java index c2c34ab..1c45372 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/listeners/QuickSearchRecordChangeListener.java @@ -334,6 +334,8 @@ public class QuickSearchRecordChangeListener implements RecordChangeListenerInte updateInput.setTableName(indexTable); updateInput.setRecords(List.of(new QRecord().withValue("id", row.getValue("id")) .withValue("realTimeErrorCount", Math.addExact(previous == null ? 0 : previous, entry.getValue())))); + // A runtime counter, not a configuration change: not record-audited. + updateInput.setOmitDmlAudit(true); List updated = QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)).getRecords(); if(CollectionUtils.nullSafeIsEmpty(updated) || CollectionUtils.nullSafeHasContents(updated.get(0).getErrors())) { diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java b/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java index f4fa517..7901971 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/processes/AbstractIndexingStep.java @@ -493,7 +493,10 @@ public abstract class AbstractIndexingStep implements BackendStep /******************************************************************************* - ** Update fields on a quick_search_index row. + ** Update runtime state on a quick_search_index row (status, last run, last + ** basepull time, document count). Indexing bookkeeping, not a configuration + ** change: it is not record-audited, or every basepull tick would write an + ** audit row. Admin edits to the row go through the table and stay audited. *******************************************************************************/ protected void updateIndexRow(Integer indexId, Map values) throws QException { @@ -506,6 +509,7 @@ public abstract class AbstractIndexingStep implements BackendStep UpdateInput updateInput = new UpdateInput(); updateInput.setTableName(getConfig().getQuickSearchIndexTableName()); updateInput.setRecords(List.of(record)); + updateInput.setOmitDmlAudit(true); QuickSearchConfigWrites.asSystem(() -> new UpdateAction().execute(updateInput)); } diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimerTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimerTest.java index 3da3ee3..cb4787f 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimerTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/config/QuickSearchConfigPrimerTest.java @@ -21,7 +21,9 @@ import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -77,6 +79,38 @@ class QuickSearchConfigPrimerTest extends BaseDataDrivenTest + /******************************************************************************* + ** Re-priming only records the priming outcome on the settings row: that is + ** not audited. An admin settings edit still is. + *******************************************************************************/ + @Test + void testRepriming_settingsBookkeepingIsNotAudited() throws QException + { + new AuditsMetaDataProvider().defineAll(QContext.getQInstance(), TEST_BACKEND_NAME, null); + new QuickSearchConfigPrimer(runtime).prime(); + long afterFirst = settingsEditAudits(); + new QuickSearchConfigPrimer(runtime).prime(); + assertThat(settingsEditAudits()).as("re-priming is not audited").isEqualTo(afterFirst); + + new UpdateAction().execute(new UpdateInput().withTableName(QuickSearchSettings.TABLE_NAME) + .withRecord(new QRecord().withValue("id", QuickSearchSettings.SETTINGS_ID).withValue("indexingPaused", true))); + assertThat(settingsEditAudits()).as("an admin settings edit is audited").isEqualTo(afterFirst + 1); + } + + + + /******************************************************************************* + ** Audit rows recording an edit of the settings row. + *******************************************************************************/ + private long settingsEditAudits() throws QException + { + return (rows(AuditsMetaDataProvider.TABLE_NAME_AUDIT).stream() + .filter(audit -> audit.getValueString("message") != null && audit.getValueString("message").contains("Edited")) + .count()); + } + + + /******************************************************************************* ** Code defaults update rows nobody edited, and never rows an owner edited. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java index fc1a6e2..172e48b 100644 --- a/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java +++ b/src/test/java/com/kingsrook/qbits/quicksearch/processes/BasepullIndexStepTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -324,6 +325,45 @@ class BasepullIndexStepTest extends BaseQuickSearchTest + /******************************************************************************* + ** Basepull bookkeeping on the index row (status, lastBasepullTime, + ** documentCount, last run) writes no audit; an admin edit of the same row + ** is still audited. + *******************************************************************************/ + @Test + void testBasepullBookkeeping_isNotAudited_adminEditIs() throws QException + { + if(QContext.getQInstance().getTable(AuditsMetaDataProvider.TABLE_NAME_AUDIT) == null) + { + new AuditsMetaDataProvider().defineAll(QContext.getQInstance(), TEST_BACKEND_NAME, null); + } + insertTestEntities(3); + QRecord row = insertIndexRow(TEST_ENTITY_TABLE, true, null, 5); + when(mockClient.countDocumentsForTable(TEST_ENTITY_TABLE)).thenReturn(3L); + + new BasepullIndexStep().run(new RunBackendStepInput(), new RunBackendStepOutput()); + assertThat(queryAll(QuickSearchIndex.TABLE_NAME).get(0).getValueInteger("documentCount")).isEqualTo(3); + assertThat(indexRowEditAudits()).as("basepull bookkeeping is not audited").isZero(); + + new UpdateAction().execute(new UpdateInput().withTableName(QuickSearchIndex.TABLE_NAME) + .withRecord(new QRecord().withValue("id", row.getValue("id")).withValue("basepullIntervalMinutes", 15))); + assertThat(indexRowEditAudits()).as("an admin configuration edit is audited").isEqualTo(1); + } + + + + /******************************************************************************* + ** Audit rows recording an edit of a quick_search_index row. + *******************************************************************************/ + private long indexRowEditAudits() throws QException + { + return (queryAll(AuditsMetaDataProvider.TABLE_NAME_AUDIT).stream() + .filter(audit -> audit.getValueString("message") != null && audit.getValueString("message").contains("Edited")) + .count()); + } + + + /******************************************************************************* ** Test: no discovered tables, no quickSearchIndex rows - step completes ** without calling the OpenSearch client.