Compare commits

..
Author SHA1 Message Date
james.maesandClaude Opus 5.5 b08d41da7f fix(audit): do not record-audit engine bookkeeping updates
Monitor lease claims and run results, and alert reminder/notification
timers, changed on every scheduler pass and each wrote a FIELD audit row
(~7k alerts_monitor and ~1.8k alerts_alert audits per 6h in production).
AlertsDb.update now omits the DML audit when an update touches only the
RuntimeFields of its table; any update with a configuration or state field
is audited in full.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 07:45:58 -05:00
james.maes 4a7dd2bb71 Merge pull request #42 from QRun-IO/fix/alerts-system-session-instance
fix(alerts): asSystem builds its system session with the runtime instance
2026-10-15 07:38:26 -05:00
4 changed files with 250 additions and 3 deletions

No files matched your search

+1
View File
@@ -5,6 +5,7 @@ All notable changes to this project are documented here. The format follows Keep
## [0.1.0] - Unreleased
### Fixed
- Audit noise: every monitor run (lease claim, then run result) and every reminder check wrote a "Record was Edited" audit (about 7,000 `alerts_monitor` and 1,800 `alerts_alert` audits in 6 hours on one production host, most with no field details). Engine updates that touch only bookkeeping fields (`RuntimeFields`: monitor `claimedBy`, `claimedUntil`, `lastRunAt`, `nextRunAt`, `lastStatus`, `lastValue`, `lastError`, `lastElapsedMs`, `consecutiveFailures`, `conditionSince`; alert `timedNotificationCheckedAt`, `lastNotifiedAt`, `recordRefCount`) are written with `omitDmlAudit`. Configuration edits, raises and alert state changes are audited as before, as are UI and API edits of any field.
- Read the QBit metadata version from the Maven-filtered artifact version so release builds report their release version.
- Record-view child lists (e.g. a notification's Recipients) failed with "Error in JSON Serialization": the child table's meta-data includes the qbit config, and `AlertsQBitConfig.getRuntime()` points at a runtime that points back at the config, so Jackson recursed to its 1000-level nesting limit. `getRuntime()` and `getAudienceResolver()` are now `@JsonIgnore`, as in the other qbits' configs. A regression test renders and serializes every child-list widget.
@@ -40,7 +40,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
** Small helpers over QQQ's table actions, for the engine. They run in
** whatever context the caller set up (the engine uses the system user), so
** customizers, audits and ESB listeners apply as for any other write.
** customizers, audits and ESB listeners apply as for any other write, except
** that bookkeeping-only updates (RuntimeFields) are not record-audited.
*******************************************************************************/
public final class AlertsDb
{
@@ -114,11 +115,14 @@ public final class AlertsDb
/*******************************************************************************
** Update one row (it must carry its id); throws on an error.
** Update one row (it must carry its id); throws on an error. An update of
** engine bookkeeping fields only (RuntimeFields) is not record-audited.
*******************************************************************************/
public static QRecord update(String tableName, QRecord record) throws QException
{
List<QRecord> updated = UpdateAction.executeForRecords(new UpdateInput(tableName).withRecord(record).withTransaction(QContext.getQBackendTransaction()));
UpdateInput input = new UpdateInput(tableName).withRecord(record).withTransaction(QContext.getQBackendTransaction())
.withOmitDmlAudit(RuntimeFields.onlyRuntime(tableName, record));
List<QRecord> updated = UpdateAction.executeForRecords(input);
QRecord result = updated.get(0);
throwIfErrors(tableName, result);
return (result);
@@ -0,0 +1,76 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.alerts.engine;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.alerts.model.AlertsAlert;
import com.kingsrook.qbits.alerts.model.AlertsMonitor;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Engine bookkeeping columns: evaluation leases, run results and notification
** timers that change on every scheduler pass. An update that touches only
** these (plus the primary key and modifyDate) is not record-audited; the core
** FIELD audit would otherwise write an audit row per monitor run and per
** reminder check. Any update that also changes a configuration or alert-state
** field (status, severity, acknowledgement, assignment, a monitor's settings)
** is audited in full, as before. Run history lives in the run fields
** themselves; alert raises, transitions and notifications have their own
** event and delivery tables.
*******************************************************************************/
public final class RuntimeFields
{
private static final Set<String> ALWAYS = Set.of("id", "modifyDate");
private static final Map<String, Set<String>> BY_TABLE = Map.of(
AlertsMonitor.TABLE_NAME, Set.of("claimedBy", "claimedUntil", "lastRunAt", "nextRunAt", "lastStatus", "lastValue", "lastError",
"lastElapsedMs", "consecutiveFailures", "conditionSince"),
AlertsAlert.TABLE_NAME, Set.of("timedNotificationCheckedAt", "lastNotifiedAt", "recordRefCount"));
/*******************************************************************************
** Constants only.
*******************************************************************************/
private RuntimeFields()
{
}
/*******************************************************************************
** The bookkeeping fields of a table (empty when it has none).
*******************************************************************************/
public static Set<String> of(String tableName)
{
return (BY_TABLE.getOrDefault(tableName, Set.of()));
}
/*******************************************************************************
** Whether an update record carries nothing but bookkeeping fields.
*******************************************************************************/
public static boolean onlyRuntime(String tableName, QRecord update)
{
Set<String> runtime = of(tableName);
return (!runtime.isEmpty() && update.getValues().keySet().stream().allMatch(field -> runtime.contains(field) || ALWAYS.contains(field)));
}
}
@@ -0,0 +1,166 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.alerts.engine;
import java.sql.Connection;
import java.sql.Statement;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.alerts.BaseAlertsTest;
import com.kingsrook.qbits.alerts.model.AlertsAlert;
import com.kingsrook.qbits.alerts.model.AlertsMonitor;
import com.kingsrook.qbits.alerts.model.AlertsType;
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Engine bookkeeping writes (monitor leases and run results, alert reminder
** and notification timers) create no audit rows; configuration edits and
** alert state changes still do.
*******************************************************************************/
class RuntimeFieldsAuditTest extends BaseAlertsTest
{
static final String TYPE = "test.widget.broken";
/*******************************************************************************
**
*******************************************************************************/
@Override
protected void customizeInstance(QInstance qInstance)
{
AlertsInstanceMetaData.ofOrWithNew(qInstance)
.withAlertType(new QAlertTypeMetaData().withName(TYPE).withLabel("Widget broken").withGroupName("widgets").withSubjectTableName(WIDGET).withDefaultSeverity(AlertSeverity.WARNING));
}
/*******************************************************************************
** Core audit tables on the test backend.
*******************************************************************************/
@BeforeEach
void auditTables() throws Exception
{
new AuditsMetaDataProvider().defineAll(qInstance, BACKEND_NAME, null);
new QInstanceEnricher(qInstance).enrich();
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) qInstance.getBackend(BACKEND_NAME)); Statement statement = connection.createStatement())
{
for(String table : List.of("auditDetail", "audit", "auditUser", "auditTable"))
{
statement.execute("DROP TABLE IF EXISTS " + table);
}
statement.execute("CREATE TABLE auditUser (id INTEGER AUTO_INCREMENT PRIMARY KEY, name VARCHAR(200), createDate TIMESTAMP, modifyDate TIMESTAMP)");
statement.execute("CREATE TABLE auditTable (id INTEGER AUTO_INCREMENT PRIMARY KEY, name VARCHAR(200), label VARCHAR(200), createDate TIMESTAMP, modifyDate TIMESTAMP)");
statement.execute("CREATE TABLE audit (id BIGINT AUTO_INCREMENT PRIMARY KEY, auditTableId INTEGER, auditUserId INTEGER, recordId INTEGER, message VARCHAR(250), timestamp TIMESTAMP)");
statement.execute("CREATE TABLE auditDetail (id BIGINT AUTO_INCREMENT PRIMARY KEY, auditId BIGINT, message VARCHAR(250), fieldName VARCHAR(100), oldValue VARCHAR(250), newValue VARCHAR(250))");
}
}
/*******************************************************************************
** Edited-record audits for one table.
*******************************************************************************/
private List<QRecord> edits(String tableName) throws Exception
{
List<QRecord> tables = where("auditTable", AlertsDb.eq("name", tableName));
if(tables.isEmpty())
{
return (List.of());
}
return (where("audit", AlertsDb.eq("auditTableId", tables.get(0).getValue("id")).withCriteria("message", com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS, "Record was Edited")));
}
/*******************************************************************************
** Changed field names of one table's edit audits.
*******************************************************************************/
private List<String> editedFields(String tableName) throws Exception
{
List<Object> ids = edits(tableName).stream().map(r -> r.getValue("id")).map(Object.class::cast).toList();
return (all("auditDetail").stream().filter(d -> ids.contains(d.getValue("auditId"))).map(d -> d.getValueString("fieldName")).toList());
}
/*******************************************************************************
** The classifier: bookkeeping only, or not.
*******************************************************************************/
@Test
void classifiesBookkeepingOnlyUpdates()
{
assertThat(RuntimeFields.onlyRuntime(AlertsMonitor.TABLE_NAME, new QRecord().withValue("id", 1).withValue("claimedBy", "x").withValue("claimedUntil", Instant.now()))).isTrue();
assertThat(RuntimeFields.onlyRuntime(AlertsMonitor.TABLE_NAME, new QRecord().withValue("id", 1).withValue("lastRunAt", Instant.now()).withValue("enabled", false))).isFalse();
assertThat(RuntimeFields.onlyRuntime(AlertsAlert.TABLE_NAME, new QRecord().withValue("id", 1).withValue("timedNotificationCheckedAt", Instant.now()))).isTrue();
assertThat(RuntimeFields.onlyRuntime(AlertsAlert.TABLE_NAME, new QRecord().withValue("id", 1).withValue("status", "ACKNOWLEDGED"))).isFalse();
assertThat(RuntimeFields.onlyRuntime(WIDGET, new QRecord().withValue("id", 1).withValue("checkedAt", Instant.now()))).isFalse();
}
/*******************************************************************************
** Evaluations and reminder checks write no monitor or alert edit audits;
** an admin edit and an acknowledgement do.
*******************************************************************************/
@Test
void evaluationsAreNotAuditedButEditsAndTransitionsAre() throws Exception
{
insertWidgets("BROKEN", "OK");
runtime.getTypeSync().syncIfNeeded();
QRecord type = runtime.asSystem(() -> AlertsDb.first(AlertsType.TABLE_NAME, AlertsDb.eq("name", TYPE)));
Integer monitorId = runtime.asSystem(() -> AlertsDb.insert(AlertsMonitor.TABLE_NAME, new QRecord()
.withValue("name", "Broken widgets").withValue("alertTypeId", type.getValue("id")).withValue("kind", "TABLE_QUERY").withValue("enabled", true)
.withValue("definition", JsonUtils.toJson(Map.of("tableName", WIDGET, "filter", Map.of("criteria", List.of(Map.of("fieldName", "status", "operator", "EQUALS", "values", List.of("BROKEN")))))))
.withValue("perRecord", false).withValue("scheduleSeconds", 60).withValue("forMinutes", 0).withValue("consecutiveFailures", 0)).getValueInteger("id"));
for(int run = 0; run < 3; run++)
{
runtime.getEvaluator().evaluateDue();
new EscalationAndReminder(runtime).runDue();
clock.advance(java.time.Duration.ofMinutes(2));
}
assertThat(runtime.asSystem(() -> AlertsDb.byId(AlertsMonitor.TABLE_NAME, monitorId)).getValueInstant("lastRunAt")).isNotNull();
assertThat(edits(AlertsMonitor.TABLE_NAME)).as("monitor runs are bookkeeping").isEmpty();
assertThat(editedFields(AlertsAlert.TABLE_NAME)).as("reminder checks and ref counts are bookkeeping")
.doesNotContain("timedNotificationCheckedAt", "recordRefCount", "lastNotifiedAt");
runtime.asSystem(() -> AlertsDb.update(AlertsMonitor.TABLE_NAME, new QRecord().withValue("id", monitorId).withValue("name", "Broken widgets (renamed)")));
runtime.asSystem(() -> AlertsDb.update(AlertsMonitor.TABLE_NAME, new QRecord().withValue("id", monitorId).withValue("enabled", false).withValue("nextRunAt", null)));
assertThat(edits(AlertsMonitor.TABLE_NAME)).hasSize(2);
assertThat(editedFields(AlertsMonitor.TABLE_NAME)).contains("name", "enabled");
QRecord alert = where(AlertsAlert.TABLE_NAME, AlertsDb.eq("monitorId", monitorId)).get(0);
runtime.getEngine().acknowledge(alert.getValueInteger("id"), ADMIN);
assertThat(editedFields(AlertsAlert.TABLE_NAME)).contains("status");
}
}