Merge pull request 'perf: scheduled jobs and the signal queue issue far fewer statements' (#75) from perf/scheduled-job-statements into develop

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>

.circleci/config.yml

@@ -1,60 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.8
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: '0.8'
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.gitea/workflows/build.yaml

@@ -0,0 +1,7 @@
+name: build
+on:
+  push:
+jobs:
+  maven:
+    uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
+    secrets: inherit

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: '0.8'
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

CHANGELOG.md

@@ -4,7 +4,20 @@ All notable changes to this project are documented here. The format follows Keep
 
 ## [0.1.0] - Unreleased
 
+### Changed
+- Scheduled jobs and the signal queue issue far fewer SQL statements (they run every 15 to 60 seconds, so this is a host's constant database load). Measured by `StatementBudgetTest`, which now fails when a job's statements grow with the rows it holds:
+  - `alertsEscalateAndRemind` with 50 open alerts and nothing due: 452 statements to 1. It reads only alerts a reminder or an escalation policy can apply to and decides from the row; an alert is written only when something is attempted. `timedNotificationCheckedAt` is therefore the last attempt, no longer the last tick (it was rewritten on every open alert every minute, with its modify date).
+  - `alertsEvaluateMonitors`: 20 monitors not due, 5 to 2 (the due monitors are selected in SQL; snooze, quiet-period and flapping upkeep is one read); 20 due clean monitors, 185 to 84; a per-record monitor refreshing 10 alerts, 205 to 120.
+  - `alertsDeliverNotifications`: idle 9 to 7; 60 pending deliveries in 2 groups, 317 to 22 (one claim, one alert read and one outcome write per group); the purge runs hourly instead of on every tick.
+  - Queued producer signals are recorded in batches: 200 resolves for alerts that are not open plus 50 heartbeats, 369 to about 33 (one open-alert lookup for consecutive resolves, one write per heartbeat key). A repeated raise of an open alert, 17 to 10.
+- Alert types, groups, escalation policies, steps and enabled mutes are read from memory (`AlertsReferenceCache`: core `Memoization` registered with core `ClusterCaches`). Any insert, update or delete of those tables through QQQ clears the table on every node through the `ClusterBus`; entries also expire after 60 seconds; a table over 2,000 rows is not held. No configuration and no new dependency.
+- A type's `openCount` and `lastRaisedAt` are engine counters: updating them is no longer record-audited (one audit per raise before).
+- New changeset 55: index `ix_alerts_alert_flapping`, for the upkeep read that otherwise scanned every alert.
+- Reconcile release-branch ancestry after an interrupted RC cut while retaining the develop SNAPSHOT revision. A subsequent release still requires the normal cut, verification and publication workflow.
+
 ### Fixed
+- SQL changelog headers: explanatory "-- Include it from the host..." comments were parsed as a licensed Liquibase `include` directive. Replaced with neutral wording; changeset bodies and schema are unchanged.
+- Under OAuth2, per-recipient checks (alert visibility for external channels, follower record access, audience sessions) never worked: they used token-less automated sessions, which OAuth2 rejects, so external sends fell back to the conservative rule. When the host declares the QQQ `alerts` service principal (`AlertsQBitConfig.withServicePrincipal`, default `alerts`), these checks run on the recipient's behalf through it: the recipient's current access intersected with the principal's. Alert recording and routing bookkeeping stay on the system session, so alerting does not depend on the identity provider being up. With no principal declared, behavior is unchanged. Needs the QQQ release with service identities (qqq#1255, #1256).
 - Audit noise: every monitor run (lease claim, then run result) and every reminder check wrote a "Record was Edited" audit (about 7,000 `alerts_monitor` and 1,800 `alerts_alert` audits in 6 hours on one production host, most with no field details). Engine updates that touch only bookkeeping fields (`RuntimeFields`: monitor `claimedBy`, `claimedUntil`, `lastRunAt`, `nextRunAt`, `lastStatus`, `lastValue`, `lastError`, `lastElapsedMs`, `consecutiveFailures`, `conditionSince`; alert `timedNotificationCheckedAt`, `lastNotifiedAt`, `recordRefCount`) are written with `omitDmlAudit`. Configuration edits, raises and alert state changes are audited as before, as are UI and API edits of any field.
 - Read the QBit metadata version from the Maven-filtered artifact version so release builds report their release version.
 - Record-view child lists (e.g. a notification's Recipients) failed with "Error in JSON Serialization": the child table's meta-data includes the qbit config, and `AlertsQBitConfig.getRuntime()` points at a runtime that points back at the config, so Jackson recursed to its 1000-level nesting limit. `getRuntime()` and `getAudienceResolver()` are now `@JsonIgnore`, as in the other qbits' configs. A regression test renders and serializes every child-list widget.

README.md

@@ -263,7 +263,9 @@ stop timed notifications. Suppressed deliveries still count as escalation attemp
 
 Each alert's delivery rows, timeline and progress commit together. The persisted
 `timedNotificationCheckedAt` prevents duplicate attempts in the same minute,
-including when no recipient is eligible. Run this job on **one scheduler node**;
+including when no recipient is eligible. It is written when a reminder or an
+escalation step is attempted; a tick that finds nothing due for an alert does
+not write the alert. Run this job on **one scheduler node**;
 there is no distributed lock between scheduler nodes. No ESCALATION feature
 switch exists in the current config; enable or disable individual policies.
 

UPGRADE.md

@@ -1,5 +1,15 @@
 # Upgrading
 
+## Unreleased: fewer statements from the scheduled jobs
+
+1. **Nothing to add for a host that includes `alerts.changelog.sql`.** Changeset 55 (index `ix_alerts_alert_flapping`) runs with the host's next migration. A host that copied the DDL instead adds `CREATE INDEX IF NOT EXISTS ix_alerts_alert_flapping ON alerts_alert (flapping);`.
+2. **No configuration.** The reference cache (types, groups, escalation policies, steps, mutes) uses QQQ core's `Memoization` and `ClusterCaches`. A host with more than one node gets changes to those tables on every node at once when it has a `ClusterBus` transport (`ClusterBusMetaData`); without one, the other nodes see a change within 60 seconds.
+3. **Behavior changes to know about.**
+   - `timedNotificationCheckedAt` on an alert is the last reminder or escalation attempt. It used to be rewritten (with the alert's modify date) on every open alert every minute, whether or not anything was due.
+   - Updates of a type's `openCount` and `lastRaisedAt` are no longer record-audited.
+   - Old notifications are purged hourly, not on every delivery tick.
+   - Writes to `alerts_type`, `alerts_group`, `alerts_escalation_policy`, `alerts_escalation_step` and `alerts_mute` that do not go through QQQ (direct SQL) are seen by the engine within 60 seconds, not at once.
+
 ## Unreleased: starter defaults, setup card, table help
 
 1. **Add the defaults changelog** after `alerts.changelog.sql` in the host's master changelog:

docs/DESIGN.md

@@ -5,7 +5,7 @@ The design for this qbit is tracked in [QRun-IO/qqq#1090](https://github.com/QRu
 Deviations from the design, and why:
 - **Timed notifications (#27, part 2):** reminders and escalation are implemented ahead of the remaining P4–P6 features. The minute job uses `stepOrder` and typed `userId` / `teamId` / `channelId` fields instead of the design's generic `targetId`. Policy description and enabled fields make the standard admin records usable. Types override groups; only OPEN, unacknowledged alerts run, even if a step is configured for another status, because acknowledgement must stop escalation.
 - **Escalation progress:** existing ESCALATED events hold level transitions and final-step repeat attempts; repeats keep the same level. Reopen events define each cycle's start and repeat budget. One step runs per minute, including when several steps became overdue. Null/zero repeat settings disable repeats. Suppression consumes an escalation attempt, as the normal router records why no delivery was queued.
-- **Timed notification transactions:** `AlertsDb` passes the current QQQ transaction to table actions. Each alert's timed fan-out, events and progress commit together. A read-only `timedNotificationCheckedAt` field prevents repeated evaluation in the same minute, including attempts with no eligible recipients. Alert and team scans use bounded pages. Runtime ticks are serialized locally; hosts must run the job on one scheduler node, matching the existing single-node scheduling guidance.
+- **Timed notification transactions:** `AlertsDb` passes the current QQQ transaction to table actions. Each alert's timed fan-out, events and progress commit together. A read-only `timedNotificationCheckedAt` field, written on each attempt, prevents a second attempt in the same minute, including attempts with no eligible recipients; alerts with nothing due are not read (when their type has no reminder and no policy) or not written. Alert and team scans use bounded pages. Runtime ticks are serialized locally; hosts must run the job on one scheduler node, matching the existing single-node scheduling guidance.
 - **Reminders and EXPIRED:** null reminder interval means 60 minutes for CRITICAL and off otherwise; explicit zero is off. Reminders use subscription event filters and NOTIFIED timeline entries, without introducing a REMINDER event. Record followers do not receive scheduled reminders without a matching subscription. EXPIRED remains reserved but is excluded from the resolution possible values; the design defines no expiration behavior. No ESCALATION feature switch is present in the current config.
 - **UI API and record following:** the RC.6 alerts extension, collaboration ownership of follow, and machine webhook migration are specified in [UI-API.md](UI-API.md). Existing RECORD watch subscriptions remain readable without migration.
 - **Alerts and notifications use core's RC.6 producer API**, including sink registration in core instance metadata (QRun-IO/qqq#1108 and #1125). `AlertsSinkAdapter` delegates `raise`, `resolve`, `heartbeat` and `notify` to the runtime queue. No local SPI types remain; storage-specific audience and severity conversion belongs to `NotificationService`.

pom.xml

@@ -18,12 +18,13 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
-      <revision>0.1.0-RC.15</revision>
+      <qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
+      <revision>0.3.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
       <jacoco.version>0.8.15</jacoco.version>
       <!-- qbit-email from Maven Central -->
-      <qbit-email.version>0.1.0-RC.2</qbit-email.version>
+      <qbit-email.version>0.1.0-RC.11</qbit-email.version>
       <liquibase.version>4.33.0</liquibase.version>
       <testcontainers.version>2.0.5</testcontainers.version>
       <qbit.guardrails.enforce>true</qbit.guardrails.enforce>
@@ -33,7 +34,7 @@
    <parent>
       <groupId>com.kingsrook</groupId>
       <artifactId>qbit-build-parent</artifactId>
-      <version>2.1.1</version>
+      <version>2.2.1</version>
    </parent>
 
    <groupId>com.kingsrook.qbits</groupId>
@@ -267,7 +268,7 @@
             <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
-            <qqq.snapshot.version>4.1.0-RC.15</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>

src/main/java/com/kingsrook/qbits/alerts/AlertsQBitConfig.java

@@ -80,6 +80,7 @@ public class AlertsQBitConfig implements QBitConfig
    private Boolean legacyAlertmanagerPath = false;
 
    private String backendName;
+   private String servicePrincipal = "alerts";
    private String schedulerName;
    private String messagingProviderName;
    private String fromAddress;
@@ -893,4 +894,30 @@ public class AlertsQBitConfig implements QBitConfig
       this.legacyAlertmanagerPath = legacyAlertmanagerPath;
       return (this);
    }
+
+
+   /*******************************************************************************
+    ** Getter for servicePrincipal: when the host declares this QQQ service
+    ** principal, per-recipient checks (alert visibility, follower record access,
+    ** audience) run on the recipient's behalf through it - their current access
+    ** intersected with the principal's - instead of a token-less automated
+    ** session (rejected under OAuth2). Alert recording and routing bookkeeping
+    ** stay on the system session so alerting never depends on the identity
+    ** provider being up.
+    *******************************************************************************/
+   public String getServicePrincipal()
+   {
+      return (this.servicePrincipal);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for servicePrincipal (default alerts).
+    *******************************************************************************/
+   public AlertsQBitConfig withServicePrincipal(String servicePrincipal)
+   {
+      this.servicePrincipal = servicePrincipal;
+      return (this);
+   }
 }

src/main/java/com/kingsrook/qbits/alerts/AlertsQBitProducer.java

@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Properties;
 import com.kingsrook.qbits.alerts.delivery.DeliveryWorker;
 import com.kingsrook.qbits.alerts.engine.AlertsProcessListener;
+import com.kingsrook.qbits.alerts.engine.AlertsReferenceCacheListener;
 import com.kingsrook.qbits.alerts.engine.AlertsSinkAdapter;
 import com.kingsrook.qbits.alerts.engine.MonitorEvaluator;
 import com.kingsrook.qbits.alerts.engine.QbitAlertsSink;
@@ -243,6 +244,12 @@ public class AlertsQBitProducer implements QBitMetaDataProducer<AlertsQBitConfig
          qInstance.withProcessLifecycleListener(new QCodeReference(AlertsProcessListener.class));
       }
 
+      String cacheListener = AlertsReferenceCacheListener.class.getName();
+      if(qInstance.getRecordChangeListeners() == null || qInstance.getRecordChangeListeners().stream().noneMatch(ref -> ref != null && cacheListener.equals(ref.getName())))
+      {
+         qInstance.withRecordChangeListener(new QCodeReference(AlertsReferenceCacheListener.class));
+      }
+
       if(Boolean.TRUE.equals(qBitConfig.getPreferencesSectionEnabled()) && isPreferencesAvailable())
       {
          com.kingsrook.qbits.alerts.preferences.AlertsNotificationPreferencesProvider.register(qInstance, qBitConfig);

src/main/java/com/kingsrook/qbits/alerts/AlertsRuntime.java

@@ -24,6 +24,7 @@ import com.kingsrook.qbits.alerts.delivery.DeliveryWorker;
 import com.kingsrook.qbits.alerts.engine.AlertRouter;
 import com.kingsrook.qbits.alerts.engine.AlertTypeSync;
 import com.kingsrook.qbits.alerts.engine.AlertsEngine;
+import com.kingsrook.qbits.alerts.engine.AlertsReferenceCache;
 import com.kingsrook.qbits.alerts.engine.MonitorEvaluator;
 import com.kingsrook.qbits.alerts.engine.QbitAlertsSink;
 import com.kingsrook.qbits.alerts.notifications.NotificationService;
@@ -47,6 +48,7 @@ public class AlertsRuntime
 
    private Clock clock = Clock.systemUTC();
 
+   private final AlertsReferenceCache reference;
    private final AlertTypeSync    typeSync;
    private final AlertsEngine     engine;
    private final AlertRouter      router;
@@ -79,6 +81,7 @@ public class AlertsRuntime
    {
       this.config = config;
       this.qInstance = qInstance;
+      this.reference = new AlertsReferenceCache(this);
       this.typeSync = new AlertTypeSync(this);
       this.router = new AlertRouter(this);
       this.engine = new AlertsEngine(this);
@@ -205,6 +208,16 @@ public class AlertsRuntime
 
 
 
+   /*******************************************************************************
+    ** The slow-changing rows (types, groups, policies, steps, mutes), from memory.
+    *******************************************************************************/
+   public AlertsReferenceCache getReference()
+   {
+      return (this.reference);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for typeSync
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/alerts/AlertsSecurity.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
@@ -34,6 +35,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
+import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentities;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -307,4 +311,25 @@ public final class AlertsSecurity
    {
       return (QContext.getQSession() instanceof QSystemUserSession);
    }
+
+
+   /*******************************************************************************
+    ** A session for a recipient: on their behalf through the alerts service
+    ** principal when the host declares it (QQQ service identities), else the
+    ** authentication module's automated session (previous behavior). Null when
+    ** neither can be made - callers fail closed.
+    *******************************************************************************/
+   public static QSession recipientSession(QInstance qInstance, String principalName, String userId) throws QException
+   {
+      if(qInstance == null || qInstance.getAuthentication() == null)
+      {
+         return (null);
+      }
+      if(StringUtils.hasContent(principalName) && ServiceIdentities.isDeclared(qInstance, principalName))
+      {
+         return (ServiceIdentities.sessionOnBehalfOf(qInstance, principalName, userId));
+      }
+      QAuthenticationModuleInterface module = new QAuthenticationModuleDispatcher().getQModule(qInstance.getAuthentication());
+      return (module.createAutomatedSessionForUser(qInstance, userId));
+   }
 }

src/main/java/com/kingsrook/qbits/alerts/delivery/DeliveryWorker.java

@@ -108,6 +108,7 @@ public class DeliveryWorker
             .withLimit(BATCH_LIMIT));
 
          Map<String, List<QRecord>> groups = new LinkedHashMap<>();
+         List<QRecord>              claims = new ArrayList<>();
          for(QRecord row : due)
          {
             Instant claimedUntil = row.getValueInstant("claimedUntil");
@@ -115,11 +116,16 @@ public class DeliveryWorker
             {
                continue;
             }
-            AlertsDb.update(AlertsDelivery.TABLE_NAME, new QRecord().withValue("id", row.getValue("id")).withValue("status", DeliveryStatus.SENDING.name())
+            claims.add(new QRecord().withValue("id", row.getValue("id")).withValue("status", DeliveryStatus.SENDING.name())
                .withValue("claimedUntil", now.plus(Duration.ofMinutes(2))));
             groups.computeIfAbsent(row.getValue("channelId") + "|" + row.getValueString("recipientUserId") + "|" + row.getValueString("kind"), key -> new ArrayList<>()).add(row);
          }
 
+         ////////////////////////////////////////////////////
+         // one claim for the batch, then one send a group //
+         ////////////////////////////////////////////////////
+         AlertsDb.updateAll(AlertsDelivery.TABLE_NAME, claims);
+
          int attempted = 0;
          for(List<QRecord> group : groups.values())
          {
@@ -198,10 +204,7 @@ public class DeliveryWorker
       AlertChannelInterface impl    = channel == null ? null : runtime.getChannels().forType(channel.getValueString("channelType"));
       if(impl == null)
       {
-         for(QRecord row : rows)
-         {
-            finish(row, DeliveryResult.failed(channel == null ? "The channel no longer exists." : "No implementation for channel type " + channel.getValueString("channelType") + "."), now);
-         }
+         finish(rows, DeliveryResult.failed(channel == null ? "The channel no longer exists." : "No implementation for channel type " + channel.getValueString("channelType") + "."), now);
          return (rows.size());
       }
 
@@ -209,9 +212,10 @@ public class DeliveryWorker
       List<AlertMessageRenderer.Item> items = new ArrayList<>();
       Map<Integer, QRecord> types  = new HashMap<>();
       Map<Integer, QRecord> groups = new HashMap<>();
+      Map<Integer, QRecord> alerts = alertsOf(rows);
       for(QRecord row : rows)
       {
-         QRecord alert = row.getValue("alertId") == null ? testAlert(now) : AlertsDb.byId(AlertsAlert.TABLE_NAME, row.getValue("alertId"));
+         QRecord alert = row.getValue("alertId") == null ? testAlert(now) : alerts.get(row.getValueInteger("alertId"));
          if(alert == null)
          {
             continue;
@@ -224,10 +228,7 @@ public class DeliveryWorker
       }
       if(items.isEmpty())
       {
-         for(QRecord row : rows)
-         {
-            finish(row, DeliveryResult.failed("The alert no longer exists."), now);
-         }
+         finish(rows, DeliveryResult.failed("The alert no longer exists."), now);
          return (rows.size());
       }
 
@@ -239,10 +240,7 @@ public class DeliveryWorker
          address = preference == null ? null : preference.getValueString("emailAddress");
          if(!StringUtils.hasContent(address))
          {
-            for(QRecord row : rows)
-            {
-               finish(row, DeliveryResult.failed("The recipient has no email address in My Alert Settings."), now);
-            }
+            finish(rows, DeliveryResult.failed("The recipient has no email address in My Alert Settings."), now);
             return (rows.size());
          }
       }
@@ -257,10 +255,7 @@ public class DeliveryWorker
          result = DeliveryResult.retry(AlertRedactor.redact(e.getClass().getSimpleName() + ": " + e.getMessage()));
       }
 
-      for(QRecord row : rows)
-      {
-         finish(row, result, now);
-      }
+      finish(rows, result, now);
       updateChannelHealth(channel, result, now);
       return (rows.size());
    }
@@ -268,9 +263,43 @@ public class DeliveryWorker
 
 
    /*******************************************************************************
-    ** Record one row's outcome.
+    ** The alerts a group's rows are about, by id, in one query.
     *******************************************************************************/
-   private void finish(QRecord row, DeliveryResult result, Instant now) throws QException
+   private static Map<Integer, QRecord> alertsOf(List<QRecord> rows) throws QException
+   {
+      Map<Integer, QRecord> alerts = new HashMap<>();
+      List<Integer>         ids    = rows.stream().map(row -> row.getValueInteger("alertId")).filter(java.util.Objects::nonNull).distinct().toList();
+      if(!ids.isEmpty())
+      {
+         for(QRecord alert : AlertsDb.query(AlertsAlert.TABLE_NAME, new QQueryFilter().withCriteria("id", QCriteriaOperator.IN, ids)))
+         {
+            alerts.put(alert.getValueInteger("id"), alert);
+         }
+      }
+      return (alerts);
+   }
+
+
+
+   /*******************************************************************************
+    ** Record the outcome of a group's rows in one update action.
+    *******************************************************************************/
+   private void finish(List<QRecord> rows, DeliveryResult result, Instant now) throws QException
+   {
+      List<QRecord> updates = new ArrayList<>();
+      for(QRecord row : rows)
+      {
+         updates.add(outcome(row, result, now));
+      }
+      AlertsDb.updateAll(AlertsDelivery.TABLE_NAME, updates);
+   }
+
+
+
+   /*******************************************************************************
+    ** One row's outcome, as the update to store.
+    *******************************************************************************/
+   private static QRecord outcome(QRecord row, DeliveryResult result, Instant now)
    {
       int     attempts = (row.getValueInteger("attempts") == null ? 0 : row.getValueInteger("attempts")) + 1;
       QRecord update   = new QRecord().withValue("id", row.getValue("id")).withValue("attempts", attempts).withValue("claimedUntil", null);
@@ -291,7 +320,7 @@ public class DeliveryWorker
          }
          default -> throw (new IllegalStateException("Unexpected outcome " + result.outcome()));
       }
-      AlertsDb.update(AlertsDelivery.TABLE_NAME, update);
+      return (update);
    }
 
 
@@ -346,13 +375,13 @@ public class DeliveryWorker
 
 
    /*******************************************************************************
-    ** A row by id, or null (no exception), for caches.
+    ** A type or group by id, or null (no exception).
     *******************************************************************************/
-   private static QRecord quietly(String tableName, Integer id)
+   private QRecord quietly(String tableName, Integer id)
    {
       try
       {
-         return (AlertsDb.byId(tableName, id));
+         return (AlertsType.TABLE_NAME.equals(tableName) ? runtime.getReference().type(id) : runtime.getReference().group(id));
       }
       catch(QException e)
       {

src/main/java/com/kingsrook/qbits/alerts/engine/AlertRouter.java

@@ -34,8 +34,6 @@ import com.kingsrook.qbits.alerts.engine.AlertsEngine.AlertTarget;
 import com.kingsrook.qbits.alerts.model.AlertsAlert;
 import com.kingsrook.qbits.alerts.model.AlertsChannel;
 import com.kingsrook.qbits.alerts.model.AlertsDelivery;
-import com.kingsrook.qbits.alerts.model.AlertsGroup;
-import com.kingsrook.qbits.alerts.model.AlertsMute;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.ChannelType;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.DeliveryKind;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.DeliveryStatus;
@@ -48,7 +46,6 @@ import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.SuppressedReason;
 import com.kingsrook.qbits.alerts.model.AlertsSubscription;
 import com.kingsrook.qbits.alerts.model.AlertsTeam;
 import com.kingsrook.qbits.alerts.model.AlertsTeamMember;
-import com.kingsrook.qbits.alerts.model.AlertsType;
 import com.kingsrook.qbits.alerts.model.AlertsUserPreference;
 import com.kingsrook.qqq.backend.core.actions.collaboration.RecordFollowers;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
@@ -66,8 +63,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.collaboration.FollowEvent;
 import com.kingsrook.qqq.backend.core.model.metadata.collaboration.RecordFollower;
 import com.kingsrook.qqq.backend.core.model.metadata.collaboration.RecordReference;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
-import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -238,7 +233,7 @@ public class AlertRouter
    {
       Instant       now      = runtime.now();
       AlertTarget   target   = AlertTarget.of(alert);
-      QRecord       type     = AlertsDb.byId(AlertsType.TABLE_NAME, alert.getValue("alertTypeId"));
+      QRecord       type     = runtime.getReference().type(alert.getValue("alertTypeId"));
       AlertSeverity severity = AlertSeverity.fromLabel(alert.getValueString("severity"), AlertSeverity.WARNING);
 
       if(destinations.isEmpty())
@@ -394,10 +389,16 @@ public class AlertRouter
     *******************************************************************************/
    boolean isMuted(AlertTarget target, Instant now, MuteEffect effect) throws QException
    {
-      List<QRecord> mutes = AlertsDb.query(AlertsMute.TABLE_NAME, AlertsDb.eq("enabled", true)
-         .withCriteria("effect", QCriteriaOperator.EQUALS, effect.name())
-         .withCriteria("startsAt", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)
-         .withCriteria("endsAt", QCriteriaOperator.GREATER_THAN, now));
+      return (isMuted(target, runtime.getReference().activeMutes(now, effect)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether one of these (active) mutes covers the target.
+    *******************************************************************************/
+   boolean isMuted(AlertTarget target, List<QRecord> mutes) throws QException
+   {
       Map<Integer, List<String>> groupNames = new HashMap<>();
       for(QRecord mute : mutes)
       {
@@ -561,7 +562,7 @@ public class AlertRouter
       int     depth   = 0;
       while(current != null && depth++ < 10)
       {
-         QRecord group = AlertsDb.byId(AlertsGroup.TABLE_NAME, current);
+         QRecord group = runtime.getReference().group(current);
          if(group == null)
          {
             return (null);
@@ -752,7 +753,7 @@ public class AlertRouter
       int          depth   = 0;
       while(current != null && depth++ < 10)
       {
-         QRecord group = AlertsDb.byId(AlertsGroup.TABLE_NAME, current);
+         QRecord group = runtime.getReference().group(current);
          if(group == null)
          {
             break;
@@ -882,18 +883,12 @@ public class AlertRouter
 
 
    /*******************************************************************************
-    ** A session for a user, from the instance's authentication module (null
-    ** when it can't make one).
+    ** A session for a user: on their behalf via the alerts service principal
+    ** when declared, else the authentication module's (null when neither).
     *******************************************************************************/
    private QSession automatedSession(String userId) throws QException
    {
-      QInstance qInstance = runtime.getQInstance();
-      if(qInstance.getAuthentication() == null)
-      {
-         return (null);
-      }
-      QAuthenticationModuleInterface module = new QAuthenticationModuleDispatcher().getQModule(qInstance.getAuthentication());
-      return (module.createAutomatedSessionForUser(qInstance, userId));
+      return (AlertsSecurity.recipientSession(runtime.getQInstance(), runtime.getConfig().getServicePrincipal(), userId));
    }
 
 

src/main/java/com/kingsrook/qbits/alerts/engine/AlertTypeSync.java

@@ -134,7 +134,7 @@ public class AlertTypeSync
     *******************************************************************************/
    public QRecord typeFor(String alertTypeName) throws QException
    {
-      QRecord row = AlertsDb.first(AlertsType.TABLE_NAME, AlertsDb.eq("name", alertTypeName));
+      QRecord row = runtime.getReference().typeByName(alertTypeName);
       if(row != null)
       {
          return (row);

src/main/java/com/kingsrook/qbits/alerts/engine/AlertsDb.java

@@ -130,6 +130,27 @@ public final class AlertsDb
 
 
 
+   /*******************************************************************************
+    ** Update several rows (each must carry its id) in one action; throws on the
+    ** first error. Not record-audited when every row carries bookkeeping fields
+    ** only (RuntimeFields). Nothing happens for an empty list.
+    *******************************************************************************/
+   public static void updateAll(String tableName, List<QRecord> records) throws QException
+   {
+      if(CollectionUtils.nullSafeIsEmpty(records))
+      {
+         return;
+      }
+      UpdateInput input = new UpdateInput(tableName).withRecords(records).withTransaction(QContext.getQBackendTransaction())
+         .withOmitDmlAudit(records.stream().allMatch(record -> RuntimeFields.onlyRuntime(tableName, record)));
+      for(QRecord updated : UpdateAction.executeForRecords(input))
+      {
+         throwIfErrors(tableName, updated);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** A filter: field equals value.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/alerts/engine/AlertsEngine.java

@@ -21,7 +21,9 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -79,6 +81,7 @@ public class AlertsEngine
 
    private static final int REPEATED_EVENT_EVERY_MINUTES = 60;
    private static final int MAX_KEY_LENGTH               = 520;
+   private static final int OPEN_KEY_BATCH               = 500;
 
    private final AlertsRuntime runtime;
 
@@ -166,13 +169,41 @@ public class AlertsEngine
             }
          }
 
-         updateTypeCounts(type.getValueInteger("id"), now);
+         if(open != null)
+         {
+            touchType(type.getValueInteger("id"), now);
+         }
+         else
+         {
+            updateTypeCounts(type.getValueInteger("id"), now);
+         }
          return (alert);
       }));
    }
 
 
 
+   /*******************************************************************************
+    ** Which of these open keys (openKey) have an open alert now: one query per
+    ** OPEN_KEY_BATCH keys, for callers that hold many. Call as the system.
+    *******************************************************************************/
+   public Set<String> openKeys(Collection<String> openKeys) throws QException
+   {
+      Set<String>        open     = new HashSet<>();
+      List<Serializable> distinct = new ArrayList<>(new LinkedHashSet<>(openKeys));
+      for(int start = 0; start < distinct.size(); start += OPEN_KEY_BATCH)
+      {
+         List<Serializable> batch = distinct.subList(start, Math.min(distinct.size(), start + OPEN_KEY_BATCH));
+         for(QRecord alert : AlertsDb.query(AlertsAlert.TABLE_NAME, new QQueryFilter(new QFilterCriteria("openDedupKey", QCriteriaOperator.IN, new ArrayList<>(batch)))))
+         {
+            open.add(alert.getValueString("openDedupKey"));
+         }
+      }
+      return (open);
+   }
+
+
+
    /*******************************************************************************
     ** Resolve the open alert for (type, dedupKey), if there is one.
     *******************************************************************************/
@@ -243,6 +274,46 @@ public class AlertsEngine
 
 
 
+   /*******************************************************************************
+    ** Record several beats at once (a queue drained in one batch): per key the
+    ** time of its latest beat and how many beats it stands for. One read for
+    ** all keys and one update action, instead of a read and an update per beat.
+    *******************************************************************************/
+   public void heartbeats(Map<String, Instant> latestByKey, Map<String, Integer> countByKey) throws QException
+   {
+      if(latestByKey.isEmpty())
+      {
+         return;
+      }
+      runtime.asSystem(() ->
+      {
+         Map<String, QRecord> existing = new HashMap<>();
+         for(QRecord row : AlertsDb.query(AlertsHeartbeat.TABLE_NAME, new QQueryFilter(new QFilterCriteria("heartbeatKey", QCriteriaOperator.IN, new ArrayList<Serializable>(latestByKey.keySet())))))
+         {
+            existing.put(row.getValueString("heartbeatKey"), row);
+         }
+         List<QRecord> updates = new ArrayList<>();
+         for(Map.Entry<String, Instant> beat : latestByKey.entrySet())
+         {
+            QRecord row   = existing.get(beat.getKey());
+            int     count = countByKey.getOrDefault(beat.getKey(), 1);
+            if(row == null)
+            {
+               AlertsDb.insert(AlertsHeartbeat.TABLE_NAME, new QRecord().withValue("heartbeatKey", beat.getKey()).withValue("lastBeatAt", beat.getValue()).withValue("beatCount", count));
+            }
+            else
+            {
+               Integer beats = ValueUtils.getValueAsInteger(row.getValue("beatCount"));
+               updates.add(new QRecord().withValue("id", row.getValue("id")).withValue("lastBeatAt", beat.getValue()).withValue("beatCount", (beats == null ? 0 : beats) + count));
+            }
+         }
+         AlertsDb.updateAll(AlertsHeartbeat.TABLE_NAME, updates);
+         return (null);
+      });
+   }
+
+
+
    /*******************************************************************************
     ** Acknowledge an open alert. Returns false when it was already
     ** acknowledged or is not open.
@@ -372,27 +443,51 @@ public class AlertsEngine
          Instant now     = runtime.now();
          int     changed = 0;
 
-         for(QRecord alert : AlertsDb.query(AlertsAlert.TABLE_NAME, AlertsDb.eq("status", Status.SNOOZED.name())
-            .withCriteria("snoozedUntil", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)))
+         ///////////////////////////////////////////////////////////////////////////
+         // one read for the three kinds of upkeep (it finds nothing on most      //
+         // ticks): snoozes that ended, quiet-period alerts gone quiet, flapping  //
+         ///////////////////////////////////////////////////////////////////////////
+         List<QRecord> quietTypes = quietPeriodTypes();
+         QQueryFilter  upkeep     = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+            .withSubFilter(AlertsDb.eq("status", Status.SNOOZED.name()).withCriteria("snoozedUntil", QCriteriaOperator.LESS_THAN_OR_EQUALS, now))
+            .withSubFilter(AlertsDb.eq("flapping", true));
+         for(QRecord type : quietTypes)
          {
-            unsnooze(alert, now, "The snooze ended");
-            changed++;
-         }
-
-         for(QRecord type : AlertsDb.query(AlertsType.TABLE_NAME, AlertsDb.eq("resolveMode", "QUIET_PERIOD").withCriteria("quietPeriodMinutes", QCriteriaOperator.IS_NOT_BLANK)))
-         {
-            Instant cutoff = now.minus(type.getValueInteger("quietPeriodMinutes"), ChronoUnit.MINUTES);
-            for(QRecord alert : AlertsDb.query(AlertsAlert.TABLE_NAME, AlertsDb.eq("alertTypeId", type.getValue("id"))
+            upkeep.withSubFilter(AlertsDb.eq("alertTypeId", type.getValue("id"))
                .withCriteria("status", QCriteriaOperator.IN, Status.openNames())
-               .withCriteria("lastRaisedAt", QCriteriaOperator.LESS_THAN, cutoff)))
+               .withCriteria("lastRaisedAt", QCriteriaOperator.LESS_THAN, now.minus(type.getValueInteger("quietPeriodMinutes"), ChronoUnit.MINUTES)));
+         }
+         List<QRecord> candidates = AlertsDb.query(AlertsAlert.TABLE_NAME, upkeep.withOrderBy(new QFilterOrderBy("id", true)));
+
+         for(QRecord alert : candidates)
+         {
+            Instant snoozedUntil = alert.getValueInstant("snoozedUntil");
+            if(Status.SNOOZED.name().equals(alert.getValueString("status")) && snoozedUntil != null && !snoozedUntil.isAfter(now))
             {
-               resolveAlert(alert, Resolution.QUIET_PERIOD, null, "No new occurrences for " + type.getValueInteger("quietPeriodMinutes") + " minutes");
+               unsnooze(alert, now, "The snooze ended");
+               alert.setValue("status", alert.getValue("acknowledgedAt") != null ? Status.ACKNOWLEDGED.name() : Status.OPEN.name());
+               alert.setValue("snoozedUntil", null);
                changed++;
             }
          }
 
+         for(QRecord type : quietTypes)
+         {
+            Instant cutoff = now.minus(type.getValueInteger("quietPeriodMinutes"), ChronoUnit.MINUTES);
+            for(QRecord alert : candidates)
+            {
+               Instant lastRaisedAt = alert.getValueInstant("lastRaisedAt");
+               if(type.getValueInteger("id").equals(alert.getValueInteger("alertTypeId")) && Status.openNames().contains(alert.getValueString("status"))
+                  && lastRaisedAt != null && lastRaisedAt.isBefore(cutoff))
+               {
+                  resolveAlert(alert, Resolution.QUIET_PERIOD, null, "No new occurrences for " + type.getValueInteger("quietPeriodMinutes") + " minutes");
+                  changed++;
+               }
+            }
+         }
+
          Instant calm = now.minus(runtime.getConfig().getFlapWindowMinutes(), ChronoUnit.MINUTES);
-         for(QRecord alert : AlertsDb.query(AlertsAlert.TABLE_NAME, AlertsDb.eq("flapping", true)))
+         for(QRecord alert : candidates.stream().filter(candidate -> Boolean.TRUE.equals(candidate.getValueBoolean("flapping"))).toList())
          {
             int recent = AlertsDb.count(AlertsEvent.TABLE_NAME, AlertsDb.eq("alertId", alert.getValue("id"))
                .withCriteria("eventType", QCriteriaOperator.IN, List.of(EventType.REOPENED.name(), EventType.RESOLVED.name()))
@@ -494,7 +589,7 @@ public class AlertsEngine
       }
 
       Integer alertId = alert.getValueInteger("id");
-      syncRecordRefs(alertId, signal, monitorId != null);
+      syncRecordRefs(alertId, signal, monitorId != null, null);
       Integer eventId = event(alertId, EventType.RAISED, null, null, severity.name(), signal.getTitle(), now);
       QRecord fresh   = AlertsDb.byId(AlertsAlert.TABLE_NAME, alertId);
       runtime.getRouter().route(fresh, EventType.RAISED, eventId);
@@ -532,7 +627,14 @@ public class AlertsEngine
          update.setValue("snoozedUntil", null);
       }
       AlertsDb.update(AlertsAlert.TABLE_NAME, update);
-      syncRecordRefs(alertId, signal, monitorId != null);
+      if(!signal.getRecordRefs().isEmpty() || (monitorId != null && !Integer.valueOf(0).equals(open.getValueInteger("recordRefCount"))))
+      {
+         ///////////////////////////////////////////////////////////////////////////
+         // a repeat with no refs has nothing to add, and (for a monitor) nothing //
+         // to deactivate when the alert holds no active ref                      //
+         ///////////////////////////////////////////////////////////////////////////
+         syncRecordRefs(alertId, signal, monitorId != null, open.getValueInteger("recordRefCount"));
+      }
 
       if(severity != previous)
       {
@@ -589,7 +691,7 @@ public class AlertsEngine
       AlertsDb.update(AlertsAlert.TABLE_NAME, update);
       if(signal != null)
       {
-         syncRecordRefs(alertId, signal, monitorId != null);
+         syncRecordRefs(alertId, signal, monitorId != null, null);
       }
 
       Integer eventId = event(alertId, EventType.REOPENED, userId, Status.RESOLVED.name(), Status.OPEN.name(), note, now);
@@ -754,9 +856,10 @@ public class AlertsEngine
    /*******************************************************************************
     ** Add the signal's record refs to the alert (marking them active). For a
     ** monitor (a full evaluation), refs it no longer returns become inactive;
-    ** a producer's refs are merged. recordRefCount is the active count.
+    ** a producer's refs are merged. recordRefCount is the active count, written
+    ** when it differs from knownCount (null: always written).
     *******************************************************************************/
-   private void syncRecordRefs(Integer alertId, AlertSignal signal, boolean fullState) throws QException
+   private void syncRecordRefs(Integer alertId, AlertSignal signal, boolean fullState, Integer knownCount) throws QException
    {
       List<QRecord>        existing = AlertsDb.query(AlertsRecordRef.TABLE_NAME, AlertsDb.eq("alertId", alertId));
       Map<String, QRecord> byKey    = new HashMap<>();
@@ -797,8 +900,11 @@ public class AlertsEngine
          }
       }
 
-      int active = AlertsDb.count(AlertsRecordRef.TABLE_NAME, AlertsDb.eq("alertId", alertId).withCriteria("active", QCriteriaOperator.EQUALS, true));
-      AlertsDb.update(AlertsAlert.TABLE_NAME, new QRecord().withValue("id", alertId).withValue("recordRefCount", active));
+      int active = (int) byKey.values().stream().filter(row -> Boolean.TRUE.equals(row.getValueBoolean("active"))).count();
+      if(knownCount == null || knownCount != active)
+      {
+         AlertsDb.update(AlertsAlert.TABLE_NAME, new QRecord().withValue("id", alertId).withValue("recordRefCount", active));
+      }
    }
 
 
@@ -823,6 +929,35 @@ public class AlertsEngine
 
 
 
+   /*******************************************************************************
+    ** A raise that refreshed an open alert: the type's open count is unchanged,
+    ** so only its last-raised time is written.
+    *******************************************************************************/
+   private void touchType(Integer typeId, Instant lastRaisedAt) throws QException
+   {
+      if(typeId != null && lastRaisedAt != null)
+      {
+         AlertsDb.update(AlertsType.TABLE_NAME, new QRecord().withValue("id", typeId).withValue("lastRaisedAt", lastRaisedAt));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Types that resolve their alerts after a quiet period.
+    *******************************************************************************/
+   private List<QRecord> quietPeriodTypes() throws QException
+   {
+      List<QRecord> types = runtime.getReference().types();
+      if(types == null)
+      {
+         return (AlertsDb.query(AlertsType.TABLE_NAME, AlertsDb.eq("resolveMode", "QUIET_PERIOD").withCriteria("quietPeriodMinutes", QCriteriaOperator.IS_NOT_BLANK)));
+      }
+      return (types.stream().filter(type -> "QUIET_PERIOD".equals(type.getValueString("resolveMode")) && type.getValueInteger("quietPeriodMinutes") != null).toList());
+   }
+
+
+
    /*******************************************************************************
     ** What a subscription or mute scope is matched against.
     *******************************************************************************/
@@ -854,6 +989,17 @@ public class AlertsEngine
          {
             keys.add(ref.getValueString("tableName") + ":" + ref.getValueString("recordId"));
          }
+         return (of(alert, keys));
+      }
+
+
+
+      /***************************************************************************
+       ** From an alert row and the keys ("table:id") of its active record refs.
+       ***************************************************************************/
+      public static AlertTarget of(QRecord alert, Set<String> activeRefKeys)
+      {
+         Set<String> keys = new LinkedHashSet<>(activeRefKeys);
          if(StringUtils.hasContent(alert.getValueString("subjectTableName")) && StringUtils.hasContent(alert.getValueString("subjectRecordId")))
          {
             keys.add(alert.getValueString("subjectTableName") + ":" + alert.getValueString("subjectRecordId"));

src/main/java/com/kingsrook/qbits/alerts/engine/AlertsReferenceCache.java

@@ -0,0 +1,384 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.alerts.engine;
+
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
+import com.kingsrook.qbits.alerts.AlertsRuntime;
+import com.kingsrook.qbits.alerts.model.AlertsEscalationPolicy;
+import com.kingsrook.qbits.alerts.model.AlertsEscalationStep;
+import com.kingsrook.qbits.alerts.model.AlertsGroup;
+import com.kingsrook.qbits.alerts.model.AlertsMute;
+import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.MuteEffect;
+import com.kingsrook.qbits.alerts.model.AlertsType;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
+
+
+/*******************************************************************************
+ ** The slow-changing rows the engine reads on every signal and every job tick:
+ ** alert types, groups, escalation policies with their steps, and enabled
+ ** mutes. Each table is held whole (they are small: a row per declared
+ ** type, group or policy), so a tick or a raise reads them from memory.
+ **
+ ** Mechanism: core Memoization registered with core ClusterCaches.
+ ** - Invalidation: every insert, update and delete of these tables through QQQ
+ **   (AlertsReferenceCacheListener, an instance record-change listener) clears
+ **   that table here and, through the ClusterBus, on every other node.
+ **   Updates of the engine's own counters on alerts_type (openCount,
+ **   lastRaisedAt) do not clear it; cached type rows do not carry those fields.
+ ** - Safety net: an entry lives for TTL (a lost cluster event, a write that did
+ **   not go through QQQ, or a listener that ran before its transaction
+ **   committed is corrected within it).
+ ** - Bound: a table with more than MAX_ROWS rows is not held; its lookups query
+ **   the database, as they did before this cache.
+ ** - Failure: a failed load is not cached and throws to the caller, as the
+ **   query it replaces did.
+ **
+ ** Rows handed out are shared: callers must not change them (single-row
+ ** lookups return a copy).
+ *******************************************************************************/
+public class AlertsReferenceCache
+{
+   public static final String CACHE_NAME = "alerts.reference";
+
+   public static final Set<String> TABLES = Set.of(AlertsType.TABLE_NAME, AlertsGroup.TABLE_NAME, AlertsEscalationPolicy.TABLE_NAME, AlertsEscalationStep.TABLE_NAME, AlertsMute.TABLE_NAME);
+
+   /** the engine's own counters on alerts_type: written on every raise, never read from the cache. */
+   public static final Set<String> TYPE_COUNTER_FIELDS = Set.of("openCount", "lastRaisedAt");
+
+   static final Duration TTL      = Duration.ofSeconds(60);
+   static final int      MAX_ROWS = 2_000;
+
+   private static final Set<AlertsReferenceCache> LIVE = Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>()));
+
+   static
+   {
+      ClusterCaches.shared(CACHE_NAME, LIVE, () -> clearLocal(null), AlertsReferenceCache::clearLocal);
+   }
+
+   private final AlertsRuntime                 runtime;
+   private final Memoization<String, Snapshot> tables = new Memoization<>(TTL, TABLES.size());
+
+
+
+   /*******************************************************************************
+    ** One table's rows as they were read.
+    **
+    ** @param rows the rows (null: the table is too large to hold)
+    ** @param byId the rows by primary key
+    *******************************************************************************/
+   private record Snapshot(List<QRecord> rows, Map<Integer, QRecord> byId)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    *******************************************************************************/
+   public AlertsReferenceCache(AlertsRuntime runtime)
+   {
+      this.runtime = runtime;
+      LIVE.add(this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A table changed: clear it on this node and on every other node.
+    *******************************************************************************/
+   public static void invalidate(String tableName)
+   {
+      ClusterCaches.invalidateKey(CACHE_NAME, tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Clear one table (null: all) in every cache of this JVM.
+    *******************************************************************************/
+   static void clearLocal(String tableName)
+   {
+      List<AlertsReferenceCache> caches;
+      synchronized(LIVE)
+      {
+         caches = new ArrayList<>(LIVE);
+      }
+      for(AlertsReferenceCache cache : caches)
+      {
+         if(tableName == null)
+         {
+            cache.tables.clear();
+         }
+         else
+         {
+            cache.tables.clearKey(tableName);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The alert type with this id, or null.
+    *******************************************************************************/
+   public QRecord type(Serializable id) throws QException
+   {
+      return (byId(AlertsType.TABLE_NAME, id));
+   }
+
+
+
+   /*******************************************************************************
+    ** The alert type with this name, or null.
+    *******************************************************************************/
+   public QRecord typeByName(String name) throws QException
+   {
+      List<QRecord> rows = rows(AlertsType.TABLE_NAME);
+      if(rows == null)
+      {
+         return (withoutCounters(AlertsDb.first(AlertsType.TABLE_NAME, AlertsDb.eq("name", name))));
+      }
+      for(QRecord row : rows)
+      {
+         if(name != null && name.equals(row.getValueString("name")))
+         {
+            return (new QRecord(row));
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Every alert type (shared rows: read only), or null when there are too
+    ** many to hold.
+    *******************************************************************************/
+   public List<QRecord> types() throws QException
+   {
+      return (rows(AlertsType.TABLE_NAME));
+   }
+
+
+
+   /*******************************************************************************
+    ** The group with this id, or null.
+    *******************************************************************************/
+   public QRecord group(Serializable id) throws QException
+   {
+      return (byId(AlertsGroup.TABLE_NAME, id));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every group (shared rows: read only), or null when there are too many.
+    *******************************************************************************/
+   public List<QRecord> groups() throws QException
+   {
+      return (rows(AlertsGroup.TABLE_NAME));
+   }
+
+
+
+   /*******************************************************************************
+    ** The escalation policy with this id, or null.
+    *******************************************************************************/
+   public QRecord policy(Serializable id) throws QException
+   {
+      return (byId(AlertsEscalationPolicy.TABLE_NAME, id));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every escalation policy (shared rows: read only), or null when there are
+    ** too many.
+    *******************************************************************************/
+   public List<QRecord> policies() throws QException
+   {
+      return (rows(AlertsEscalationPolicy.TABLE_NAME));
+   }
+
+
+
+   /*******************************************************************************
+    ** A policy's steps by stepOrder (shared rows: read only).
+    *******************************************************************************/
+   public List<QRecord> steps(Serializable policyId) throws QException
+   {
+      Integer id = ValueUtils.getValueAsInteger(policyId);
+      if(id == null)
+      {
+         return (List.of());
+      }
+      List<QRecord> rows = rows(AlertsEscalationStep.TABLE_NAME);
+      if(rows == null)
+      {
+         return (AlertsDb.query(AlertsEscalationStep.TABLE_NAME, AlertsDb.eq("policyId", id).withOrderBy(new QFilterOrderBy("stepOrder", true)).withOrderBy(new QFilterOrderBy("id", true))));
+      }
+      List<QRecord> steps = new ArrayList<>();
+      for(QRecord row : rows)
+      {
+         if(id.equals(row.getValueInteger("policyId")))
+         {
+            steps.add(row);
+         }
+      }
+      return (steps);
+   }
+
+
+
+   /*******************************************************************************
+    ** Enabled mutes with this effect that are active now (shared rows: read
+    ** only).
+    *******************************************************************************/
+   public List<QRecord> activeMutes(Instant now, MuteEffect effect) throws QException
+   {
+      List<QRecord> rows = rows(AlertsMute.TABLE_NAME);
+      if(rows == null)
+      {
+         return (AlertsDb.query(AlertsMute.TABLE_NAME, AlertsDb.eq("enabled", true)
+            .withCriteria("effect", QCriteriaOperator.EQUALS, effect.name())
+            .withCriteria("startsAt", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)
+            .withCriteria("endsAt", QCriteriaOperator.GREATER_THAN, now)));
+      }
+      List<QRecord> active = new ArrayList<>();
+      for(QRecord row : rows)
+      {
+         Instant startsAt = row.getValueInstant("startsAt");
+         Instant endsAt   = row.getValueInstant("endsAt");
+         if(effect.name().equals(row.getValueString("effect")) && startsAt != null && !startsAt.isAfter(now) && endsAt != null && endsAt.isAfter(now))
+         {
+            active.add(row);
+         }
+      }
+      return (active);
+   }
+
+
+
+   /*******************************************************************************
+    ** A row by primary key, from memory or (for a table too large to hold) the
+    ** database. A copy: the caller may change it.
+    *******************************************************************************/
+   private QRecord byId(String tableName, Serializable id) throws QException
+   {
+      Integer key = ValueUtils.getValueAsInteger(id);
+      if(key == null)
+      {
+         return (null);
+      }
+      Snapshot snapshot = snapshot(tableName);
+      if(snapshot.rows() == null)
+      {
+         QRecord row = AlertsDb.byId(tableName, key);
+         return (AlertsType.TABLE_NAME.equals(tableName) ? withoutCounters(row) : row);
+      }
+      QRecord row = snapshot.byId().get(key);
+      return (row == null ? null : new QRecord(row));
+   }
+
+
+
+   /*******************************************************************************
+    ** A table's rows, or null when it is too large to hold.
+    *******************************************************************************/
+   private List<QRecord> rows(String tableName) throws QException
+   {
+      return (snapshot(tableName).rows());
+   }
+
+
+
+   /*******************************************************************************
+    ** The table as held, read (as the system, which is outside any caller's
+    ** transaction) when it is not held or its entry expired.
+    *******************************************************************************/
+   private Snapshot snapshot(String tableName) throws QException
+   {
+      return (tables.getResultThrowing(tableName, name -> runtime.asSystem(() -> load(name))).orElseThrow());
+   }
+
+
+
+   /*******************************************************************************
+    ** Read a table (of mutes, the enabled ones).
+    *******************************************************************************/
+   private Snapshot load(String tableName) throws QException
+   {
+      QQueryFilter filter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true)).withLimit(MAX_ROWS + 1);
+      if(AlertsMute.TABLE_NAME.equals(tableName))
+      {
+         filter.withCriteria("enabled", QCriteriaOperator.EQUALS, true);
+      }
+      List<QRecord> rows = AlertsDb.query(tableName, filter);
+      if(rows.size() > MAX_ROWS)
+      {
+         return (new Snapshot(null, Map.of()));
+      }
+      if(AlertsEscalationStep.TABLE_NAME.equals(tableName))
+      {
+         rows = new ArrayList<>(rows);
+         rows.sort(Comparator.comparing((QRecord row) -> row.getValueInteger("stepOrder"), Comparator.nullsLast(Comparator.naturalOrder())).thenComparing(row -> row.getValueInteger("id")));
+      }
+      Map<Integer, QRecord> byId = new HashMap<>();
+      for(QRecord row : rows)
+      {
+         if(AlertsType.TABLE_NAME.equals(tableName))
+         {
+            withoutCounters(row);
+         }
+         byId.put(row.getValueInteger("id"), row);
+      }
+      return (new Snapshot(List.copyOf(rows), Map.copyOf(byId)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A type row without the engine's counters, which this cache does not keep
+    ** current (null safe).
+    *******************************************************************************/
+   private static QRecord withoutCounters(QRecord type)
+   {
+      if(type != null)
+      {
+         TYPE_COUNTER_FIELDS.forEach(type::removeValue);
+      }
+      return (type);
+   }
+}

src/main/java/com/kingsrook/qbits/alerts/engine/AlertsReferenceCacheListener.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.alerts.engine;
+
+
+import java.util.Set;
+import com.kingsrook.qbits.alerts.model.AlertsType;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Clears AlertsReferenceCache when one of its tables is written through QQQ
+ ** (the engine, a screen, an import or the API), on this node and, through the
+ ** ClusterBus, on the others. The engine's own counter updates on alerts_type
+ ** are ignored: they happen on every raise and the cache does not hold them.
+ *******************************************************************************/
+public class AlertsReferenceCacheListener implements RecordChangeListenerInterface
+{
+   private static final Set<String> NOT_CONFIGURATION = Set.of("id", "modifyDate");
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean appliesTo(String tableName, RecordChangeType type)
+   {
+      return (AlertsReferenceCache.TABLES.contains(tableName));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void onRecordsChanged(RecordChangeEvent event)
+   {
+      if(event.getType() == RecordChangeType.UPDATE && AlertsType.TABLE_NAME.equals(event.getTableName()) && onlyCounters(event))
+      {
+         return;
+      }
+      AlertsReferenceCache.invalidate(event.getTableName());
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether every updated type row carries nothing but the engine's counters.
+    *******************************************************************************/
+   private static boolean onlyCounters(RecordChangeEvent event)
+   {
+      if(CollectionUtils.nullSafeIsEmpty(event.getRecords()))
+      {
+         return (false);
+      }
+      for(QRecord record : event.getRecords())
+      {
+         for(String field : record.getValues().keySet())
+         {
+            if(!AlertsReferenceCache.TYPE_COUNTER_FIELDS.contains(field) && !NOT_CONFIGURATION.contains(field))
+            {
+               return (false);
+            }
+         }
+      }
+      return (true);
+   }
+}

src/main/java/com/kingsrook/qbits/alerts/engine/EscalationAndReminder.java

@@ -17,19 +17,20 @@
 package com.kingsrook.qbits.alerts.engine;
 
 
+import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qbits.alerts.AlertsRuntime;
 import com.kingsrook.qbits.alerts.model.AlertsAlert;
-import com.kingsrook.qbits.alerts.model.AlertsEscalationPolicy;
-import com.kingsrook.qbits.alerts.model.AlertsEscalationStep;
 import com.kingsrook.qbits.alerts.model.AlertsEvent;
-import com.kingsrook.qbits.alerts.model.AlertsGroup;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.EventType;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.MuteEffect;
+import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.ScopeType;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.Status;
-import com.kingsrook.qbits.alerts.model.AlertsType;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -43,11 +44,22 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 /*******************************************************************************
  ** Timed notifications for open, unacknowledged alerts. Progress and fan-out
  ** commit together per alert. Run on one scheduler node, like the other jobs.
+ **
+ ** A tick reads only the alerts something can be due for (their type has a
+ ** reminder interval, their type or group has an enabled escalation policy, or
+ ** they are CRITICAL, which reminds hourly by default) and decides from the
+ ** row and AlertsReferenceCache whether anything is due now. Only then does an
+ ** alert take its own transaction (apply), which re-reads it and checks again.
+ ** An alert with nothing due is not written: timedNotificationCheckedAt
+ ** records the minute of the last attempt, which is what stops a second
+ ** attempt in the same minute.
  *******************************************************************************/
 public class EscalationAndReminder
 {
    private static final int PAGE_SIZE = 100;
 
+   private static final int DEFAULT_CRITICAL_REMINDER_MINUTES = 60;
+
    private final AlertsRuntime runtime;
 
 
@@ -63,8 +75,9 @@ public class EscalationAndReminder
 
 
    /*******************************************************************************
-    ** Read bounded pages. A persisted checkpoint makes repeated ticks harmless,
-    ** including attempts with no eligible recipients. Serialize this runtime's ticks.
+    ** Read bounded pages of the alerts that can have something due. A persisted
+    ** checkpoint makes repeated ticks harmless, including attempts with no
+    ** eligible recipients. Serialize this runtime's ticks.
     *******************************************************************************/
    public int runDue() throws QException
    {
@@ -72,21 +85,26 @@ public class EscalationAndReminder
       {
          return (runtime.asSystem(() ->
          {
-            int changed = 0;
-            int lastId = 0;
+            Instant      now        = runtime.now();
+            QQueryFilter candidates = candidates(now);
+            int          changed    = 0;
+            int          lastId     = 0;
             while(true)
             {
-               List<QRecord> alerts = AlertsDb.query(AlertsAlert.TABLE_NAME, AlertsDb.eq("status", Status.OPEN.name())
+               List<QRecord> alerts = AlertsDb.query(AlertsAlert.TABLE_NAME, candidates.clone()
                   .withCriteria("id", QCriteriaOperator.GREATER_THAN, lastId)
                   .withOrderBy(new QFilterOrderBy("id", true)).withLimit(PAGE_SIZE));
-               if(alerts.isEmpty())
-               {
-                  break;
-               }
                for(QRecord alert : alerts)
                {
                   lastId = alert.getValueInteger("id");
-                  changed += process(lastId);
+                  if(mayBeDue(alert, now))
+                  {
+                     changed += process(lastId);
+                  }
+               }
+               if(alerts.size() < PAGE_SIZE)
+               {
+                  break;
                }
             }
             return (changed);
@@ -96,6 +114,132 @@ public class EscalationAndReminder
 
 
 
+   /*******************************************************************************
+    ** Open, unacknowledged alerts not attempted this minute, narrowed (when the
+    ** reference tables are held in memory) to those a reminder or an escalation
+    ** can apply to. Everything here is checked again per alert.
+    *******************************************************************************/
+   private QQueryFilter candidates(Instant now) throws QException
+   {
+      QQueryFilter filter = AlertsDb.eq("status", Status.OPEN.name())
+         .withCriteria("acknowledgedAt", QCriteriaOperator.IS_BLANK)
+         .withSubFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+            .withCriteria("timedNotificationCheckedAt", QCriteriaOperator.IS_BLANK)
+            .withCriteria("timedNotificationCheckedAt", QCriteriaOperator.LESS_THAN, now.truncatedTo(ChronoUnit.MINUTES)));
+
+      AlertsReferenceCache reference = runtime.getReference();
+      List<QRecord>        types     = reference.types();
+      List<QRecord>        groups    = reference.groups();
+      List<QRecord>        policies  = reference.policies();
+      if(types == null || groups == null || policies == null)
+      {
+         return (filter);
+      }
+
+      Set<Integer> enabledPolicies = new HashSet<>();
+      for(QRecord policy : policies)
+      {
+         if(Boolean.TRUE.equals(policy.getValueBoolean("enabled")))
+         {
+            enabledPolicies.add(policy.getValueInteger("id"));
+         }
+      }
+      List<Serializable> typeIds = new ArrayList<>();
+      for(QRecord type : types)
+      {
+         Integer interval = type.getValueInteger("reminderIntervalMinutes");
+         if(canNotify(type) && ((interval != null && interval > 0) || enabledPolicies.contains(type.getValueInteger("escalationPolicyId"))))
+         {
+            typeIds.add(type.getValueInteger("id"));
+         }
+      }
+      List<Serializable> groupIds = new ArrayList<>();
+      for(QRecord group : groups)
+      {
+         if(enabledPolicies.contains(group.getValueInteger("escalationPolicyId")))
+         {
+            groupIds.add(group.getValueInteger("id"));
+         }
+      }
+
+      QQueryFilter applicable = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+         .withCriteria("severity", QCriteriaOperator.EQUALS, "CRITICAL");
+      if(!typeIds.isEmpty())
+      {
+         applicable.withCriteria("alertTypeId", QCriteriaOperator.IN, typeIds);
+      }
+      if(!groupIds.isEmpty())
+      {
+         applicable.withCriteria("groupId", QCriteriaOperator.IN, groupIds);
+      }
+      return (filter.withSubFilter(applicable));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether apply could do anything for this alert now, decided from the row
+    ** and the reference tables alone. False only when apply would return
+    ** without an attempt; anything that needs more than that (a record or table
+    ** mute, a reopened alert's step, a repeat of the final step) is true, and
+    ** apply decides.
+    *******************************************************************************/
+   private boolean mayBeDue(QRecord alert, Instant now) throws QException
+   {
+      AlertsReferenceCache reference = runtime.getReference();
+      QRecord              type      = reference.type(alert.getValue("alertTypeId"));
+      if(type == null || !canNotify(type) || Boolean.TRUE.equals(alert.getValueBoolean("muted")) || Boolean.TRUE.equals(alert.getValueBoolean("flapping")))
+      {
+         return (false);
+      }
+
+      List<QRecord> mutes = new ArrayList<>(reference.activeMutes(now, MuteEffect.SILENCE_NOTIFICATIONS));
+      mutes.addAll(reference.activeMutes(now, MuteEffect.SUPPRESS_ALERTS));
+      if(!mutes.isEmpty())
+      {
+         for(QRecord mute : mutes)
+         {
+            String scope = mute.getValueString("scopeType");
+            if(ScopeType.TABLE.name().equals(scope) || ScopeType.RECORD.name().equals(scope))
+            {
+               return (true);
+            }
+         }
+         if(runtime.getRouter().isMuted(AlertsEngine.AlertTarget.of(alert, Set.of()), mutes))
+         {
+            return (false);
+         }
+      }
+
+      QRecord policy = reference.policy(policyId(alert, type));
+      if(policy != null && Boolean.TRUE.equals(policy.getValueBoolean("enabled")))
+      {
+         int     level = alert.getValueInteger("escalationLevel") == null ? 0 : alert.getValueInteger("escalationLevel");
+         QRecord step  = nextStep(policy.getValue("id"), level);
+         if(step == null)
+         {
+            Integer interval   = policy.getValueInteger("repeatAfterMinutes");
+            Integer maxRepeats = policy.getValueInteger("maxRepeats");
+            if(level != 0 && interval != null && interval > 0 && maxRepeats != null && maxRepeats > 0)
+            {
+               return (true);
+            }
+         }
+         else if(stepIsUsable(step))
+         {
+            Integer reopens = alert.getValueInteger("reopenCount");
+            Instant start   = alert.getValueInstant("firstRaisedAt");
+            if((reopens != null && reopens > 0) || (start != null && !start.plus(step.getValueInteger("afterMinutes"), ChronoUnit.MINUTES).isAfter(now)))
+            {
+               return (true);
+            }
+         }
+      }
+      return (reminderIsDue(alert, type, now));
+   }
+
+
+
    /*******************************************************************************
     ** One alert is an atomic unit, including in-app recipients and delivery rows.
     *******************************************************************************/
@@ -139,8 +283,8 @@ public class EscalationAndReminder
       {
          return (0);
       }
-      QRecord type = AlertsDb.byId(AlertsType.TABLE_NAME, alert.getValue("alertTypeId"));
-      if(type == null || Boolean.FALSE.equals(type.getValueBoolean("enabled")) || Boolean.FALSE.equals(type.getValueBoolean("notify")))
+      QRecord type = runtime.getReference().type(alert.getValue("alertTypeId"));
+      if(type == null || !canNotify(type))
       {
          return (0);
       }
@@ -154,14 +298,7 @@ public class EscalationAndReminder
 
       int changed = escalate(alert, type, now) ? 1 : 0;
       QRecord fresh = AlertsDb.byId(AlertsAlert.TABLE_NAME, alert.getValue("id"));
-      Integer interval = type.getValueInteger("reminderIntervalMinutes");
-      if(interval == null && "CRITICAL".equals(fresh.getValueString("severity")))
-      {
-         interval = 60;
-      }
-      Instant raised = latest(fresh.getValueInstant("lastRaisedAt"), fresh.getValueInstant("firstRaisedAt"));
-      Instant since = latest(fresh.getValueInstant("lastNotifiedAt"), raised);
-      if(interval != null && interval > 0 && since != null && !since.plus(interval, ChronoUnit.MINUTES).isAfter(now))
+      if(reminderIsDue(fresh, type, now))
       {
          if(runtime.getRouter().remind(fresh) > 0)
          {
@@ -181,13 +318,8 @@ public class EscalationAndReminder
     *******************************************************************************/
    private boolean escalate(QRecord alert, QRecord type, Instant now) throws QException
    {
-      Integer policyId = type.getValueInteger("escalationPolicyId");
-      if(policyId == null)
-      {
-         QRecord group = AlertsDb.byId(AlertsGroup.TABLE_NAME, alert.getValue("groupId"));
-         policyId = group == null ? null : group.getValueInteger("escalationPolicyId");
-      }
-      QRecord policy = AlertsDb.byId(AlertsEscalationPolicy.TABLE_NAME, policyId);
+      Serializable policyId = policyId(alert, type);
+      QRecord      policy   = runtime.getReference().policy(policyId);
       if(policy == null || !Boolean.TRUE.equals(policy.getValueBoolean("enabled")))
       {
          return (false);
@@ -196,15 +328,12 @@ public class EscalationAndReminder
          .withCriteria("eventType", QCriteriaOperator.EQUALS, EventType.REOPENED.name()).withOrderBy(new QFilterOrderBy("id", false)));
       Instant start = reopened == null ? alert.getValueInstant("firstRaisedAt") : reopened.getValueInstant("at");
       int level = alert.getValueInteger("escalationLevel") == null ? 0 : alert.getValueInteger("escalationLevel");
-      QRecord step = AlertsDb.first(AlertsEscalationStep.TABLE_NAME, AlertsDb.eq("policyId", policyId)
-         .withCriteria("stepOrder", QCriteriaOperator.GREATER_THAN, level).withOrderBy(new QFilterOrderBy("stepOrder", true)));
+      QRecord step = nextStep(policyId, level);
       if(step == null)
       {
          step = repeatStep(alert, policy, reopened, level, now);
       }
-      if(step == null || start == null || step.getValueInteger("afterMinutes") == null || step.getValueInteger("afterMinutes") < 0
-         || !Status.OPEN.name().equals(step.getValueString("onlyIfStatus"))
-         || start.plus(step.getValueInteger("afterMinutes"), ChronoUnit.MINUTES).isAfter(now))
+      if(step == null || start == null || !stepIsUsable(step) || start.plus(step.getValueInteger("afterMinutes"), ChronoUnit.MINUTES).isAfter(now))
       {
          return (false);
       }
@@ -243,8 +372,89 @@ public class EscalationAndReminder
          .withCriteria("id", QCriteriaOperator.GREATER_THAN, reopened == null ? 0 : reopened.getValueInteger("id"))
          .withCriteria("fromValue", QCriteriaOperator.EQUALS, String.valueOf(level))
          .withCriteria("toValue", QCriteriaOperator.EQUALS, String.valueOf(level)));
-      return (repeats >= maxRepeats ? null : AlertsDb.first(AlertsEscalationStep.TABLE_NAME, AlertsDb.eq("policyId", policy.getValue("id"))
-         .withCriteria("stepOrder", QCriteriaOperator.EQUALS, level)));
+      if(repeats >= maxRepeats)
+      {
+         return (null);
+      }
+      for(QRecord step : runtime.getReference().steps(policy.getValue("id")))
+      {
+         if(step.getValueInteger("stepOrder") != null && step.getValueInteger("stepOrder") == level)
+         {
+            return (step);
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The policy that applies: the type's, else the alert's group's (null: none).
+    *******************************************************************************/
+   private Serializable policyId(QRecord alert, QRecord type) throws QException
+   {
+      Integer policyId = type.getValueInteger("escalationPolicyId");
+      if(policyId == null)
+      {
+         QRecord group = runtime.getReference().group(alert.getValue("groupId"));
+         policyId = group == null ? null : group.getValueInteger("escalationPolicyId");
+      }
+      return (policyId);
+   }
+
+
+
+   /*******************************************************************************
+    ** The policy's first step after a level (null: none).
+    *******************************************************************************/
+   private QRecord nextStep(Serializable policyId, int level) throws QException
+   {
+      for(QRecord step : runtime.getReference().steps(policyId))
+      {
+         if(step.getValueInteger("stepOrder") != null && step.getValueInteger("stepOrder") > level)
+         {
+            return (step);
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a step can ever be taken: a delay, and only for OPEN alerts.
+    *******************************************************************************/
+   private static boolean stepIsUsable(QRecord step)
+   {
+      return (step.getValueInteger("afterMinutes") != null && step.getValueInteger("afterMinutes") >= 0 && Status.OPEN.name().equals(step.getValueString("onlyIfStatus")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the type's alerts notify at all.
+    *******************************************************************************/
+   private static boolean canNotify(QRecord type)
+   {
+      return (!Boolean.FALSE.equals(type.getValueBoolean("enabled")) && !Boolean.FALSE.equals(type.getValueBoolean("notify")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the reminder interval (the type's; hourly for CRITICAL when the
+    ** type has none) has passed since the last raise or notification.
+    *******************************************************************************/
+   private static boolean reminderIsDue(QRecord alert, QRecord type, Instant now)
+   {
+      Integer interval = type.getValueInteger("reminderIntervalMinutes");
+      if(interval == null && "CRITICAL".equals(alert.getValueString("severity")))
+      {
+         interval = DEFAULT_CRITICAL_REMINDER_MINUTES;
+      }
+      Instant raised = latest(alert.getValueInstant("lastRaisedAt"), alert.getValueInstant("firstRaisedAt"));
+      Instant since  = latest(alert.getValueInstant("lastNotifiedAt"), raised);
+      return (interval != null && interval > 0 && since != null && !since.plus(interval, ChronoUnit.MINUTES).isAfter(now));
    }
 
 

src/main/java/com/kingsrook/qbits/alerts/engine/MonitorEvaluator.java

@@ -44,7 +44,6 @@ import com.kingsrook.qbits.alerts.model.AlertsMonitor;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.MonitorStatus;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.Resolution;
 import com.kingsrook.qbits.alerts.model.AlertsPossibleValues.Status;
-import com.kingsrook.qbits.alerts.model.AlertsType;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
 import com.kingsrook.qqq.backend.core.alerts.AlertLink;
@@ -145,27 +144,29 @@ public class MonitorEvaluator
       List<QRecord> due = runtime.asSystem(() ->
       {
          Instant now = runtime.now();
-         List<QRecord> candidates = AlertsDb.query(AlertsMonitor.TABLE_NAME, AlertsDb.eq("enabled", true)
+         ///////////////////////////////////////////////////////////////////////////
+         // only the monitors that are due and not claimed: a tick with nothing   //
+         // due reads no rows (ix_alerts_monitor_enabled_next_run_at)             //
+         ///////////////////////////////////////////////////////////////////////////
+         List<QRecord> claimed = AlertsDb.query(AlertsMonitor.TABLE_NAME, AlertsDb.eq("enabled", true)
             .withCriteria("kind", QCriteriaOperator.IN, List.of(AlertMonitorKind.TABLE_QUERY.name(), AlertMonitorKind.AGGREGATE.name(), AlertMonitorKind.HEARTBEAT.name(), AlertMonitorKind.CODE.name()))
+            .withSubFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria("nextRunAt", QCriteriaOperator.IS_BLANK).withCriteria("nextRunAt", QCriteriaOperator.LESS_THAN_OR_EQUALS, now))
+            .withSubFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria("claimedUntil", QCriteriaOperator.IS_BLANK).withCriteria("claimedUntil", QCriteriaOperator.LESS_THAN, now))
             .withOrderBy(new QFilterOrderBy("id")));
-         List<QRecord> claimed = new ArrayList<>();
-         for(QRecord monitor : candidates)
+         List<QRecord> claims = new ArrayList<>();
+         for(QRecord monitor : claimed)
          {
-            Instant next  = monitor.getValueInstant("nextRunAt");
-            Instant until = monitor.getValueInstant("claimedUntil");
-            if((next == null || !next.isAfter(now)) && (until == null || until.isBefore(now)))
-            {
-               AlertsDb.update(AlertsMonitor.TABLE_NAME, new QRecord().withValue("id", monitor.getValue("id")).withValue("claimedBy", evaluatorId)
-                  .withValue("claimedUntil", now.plusSeconds(CLAIM_SECONDS)));
-               claimed.add(monitor);
-            }
+            claims.add(new QRecord().withValue("id", monitor.getValue("id")).withValue("claimedBy", evaluatorId).withValue("claimedUntil", now.plusSeconds(CLAIM_SECONDS)));
          }
+         AlertsDb.updateAll(AlertsMonitor.TABLE_NAME, claims);
          return (claimed);
       });
 
       for(QRecord monitor : due)
       {
-         evaluate(monitor.getValueInteger("id"), true);
+         evaluate(monitor, true);
       }
       runtime.getEngine().maintenance();
       return (due.size());
@@ -197,16 +198,28 @@ public class MonitorEvaluator
     ** Evaluate one monitor, applying the result or only planning it.
     *******************************************************************************/
    Plan evaluate(Integer monitorId, boolean apply) throws QException
+   {
+      runtime.getTypeSync().syncIfNeeded();
+      QRecord monitor = runtime.asSystem(() -> AlertsDb.byId(AlertsMonitor.TABLE_NAME, monitorId));
+      if(monitor == null)
+      {
+         throw (new QUserFacingException("The monitor was not found."));
+      }
+      return (evaluate(monitor, apply));
+   }
+
+
+
+   /*******************************************************************************
+    ** Evaluate a monitor row as read, applying the result or only planning it.
+    *******************************************************************************/
+   private Plan evaluate(QRecord monitor, boolean apply) throws QException
    {
       runtime.getTypeSync().syncIfNeeded();
       return (runtime.asSystem(() ->
       {
-         QRecord monitor = AlertsDb.byId(AlertsMonitor.TABLE_NAME, monitorId);
-         if(monitor == null)
-         {
-            throw (new QUserFacingException("The monitor was not found."));
-         }
-         QRecord type = AlertsDb.byId(AlertsType.TABLE_NAME, monitor.getValue("alertTypeId"));
+         Integer monitorId = monitor.getValueInteger("id");
+         QRecord type      = runtime.getReference().type(monitor.getValue("alertTypeId"));
          if(type == null)
          {
             throw (new QUserFacingException("The monitor's alert type was not found."));
@@ -265,10 +278,13 @@ public class MonitorEvaluator
          Set<String>   keys         = new HashSet<>();
          for(AlertSignal signal : signals)
          {
-            String signalType = signal.getAlertTypeName() == null ? typeName : signal.getAlertTypeName();
-            signal.setAlertTypeName(signalType);
-            keys.add(AlertsEngine.openKey(signalType, signal.getEffectiveDedupKey()));
-            if(AlertsDb.first(AlertsAlert.TABLE_NAME, AlertsDb.eq("openDedupKey", AlertsEngine.openKey(signalType, signal.getEffectiveDedupKey()))) != null)
+            signal.setAlertTypeName(signal.getAlertTypeName() == null ? typeName : signal.getAlertTypeName());
+            keys.add(AlertsEngine.openKey(signal.getAlertTypeName(), signal.getEffectiveDedupKey()));
+         }
+         Set<String> alreadyOpen = runtime.getEngine().openKeys(keys);
+         for(AlertSignal signal : signals)
+         {
+            if(alreadyOpen.contains(AlertsEngine.openKey(signal.getAlertTypeName(), signal.getEffectiveDedupKey())))
             {
                wouldUpdate.add(signal.getEffectiveDedupKey());
             }
@@ -320,7 +336,7 @@ public class MonitorEvaluator
    {
       for(QRecord monitor : processMonitors(processName))
       {
-         QRecord type = runtime.asSystem(() -> AlertsDb.byId(AlertsType.TABLE_NAME, monitor.getValue("alertTypeId")));
+         QRecord type = runtime.getReference().type(monitor.getValue("alertTypeId"));
          if(type == null)
          {
             continue;
@@ -344,7 +360,7 @@ public class MonitorEvaluator
    {
       for(QRecord monitor : processMonitors(processName))
       {
-         QRecord type = runtime.asSystem(() -> AlertsDb.byId(AlertsType.TABLE_NAME, monitor.getValue("alertTypeId")));
+         QRecord type = runtime.getReference().type(monitor.getValue("alertTypeId"));
          if(type != null)
          {
             runtime.getEngine().resolve(type.getValueString("name"), processName, Resolution.MONITOR, null, "A later run completed");

src/main/java/com/kingsrook/qbits/alerts/engine/QbitAlertsSink.java

@@ -18,7 +18,12 @@ package com.kingsrook.qbits.alerts.engine;
 
 
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
@@ -34,6 +39,7 @@ import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
 import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
 import com.kingsrook.qqq.backend.core.alerts.NotificationSignal;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -45,6 +51,9 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** signals a minute for one type, signals are counted, not stored, and
  ** alerts.producer.flood is raised (section 7.5).
  **
+ ** The queue is drained in batches (process), so a burst costs fewer
+ ** statements than the same signals one at a time.
+ **
  ** With synchronousSink (tests), signals are processed on the caller's
  ** thread.
  *******************************************************************************/
@@ -59,9 +68,10 @@ public class QbitAlertsSink implements AlertSinkInterface
    static final int FLOOD_PER_MINUTE    = 1_000;
    static final long POLL_MILLIS        = 200;
    static final long STOP_WAIT_MILLIS   = 10_000;
+   static final int MAX_BATCH           = 500;
 
    private final AlertsRuntime          runtime;
-   private final BlockingQueue<Runnable> queue = new ArrayBlockingQueue<>(QUEUE_CAPACITY);
+   private final BlockingQueue<Queued>  queue = new ArrayBlockingQueue<>(QUEUE_CAPACITY);
    private final AtomicBoolean          started = new AtomicBoolean(false);
    private final AtomicBoolean          overflowRaised = new AtomicBoolean(false);
    private final AtomicLong             dropped = new AtomicLong();
@@ -72,6 +82,60 @@ public class QbitAlertsSink implements AlertSinkInterface
 
 
 
+   /*******************************************************************************
+    ** One queued producer call.
+    *******************************************************************************/
+   sealed interface Queued permits Raise, Resolve, Beat, Notify
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A raise.
+    **
+    ** @param signal the signal
+    *******************************************************************************/
+   record Raise(AlertSignal signal) implements Queued
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A resolve.
+    **
+    ** @param resolution the resolution
+    *******************************************************************************/
+   record Resolve(AlertResolution resolution) implements Queued
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A heartbeat.
+    **
+    ** @param heartbeatKey the key
+    ** @param at           when it beat
+    *******************************************************************************/
+   record Beat(String heartbeatKey, Instant at) implements Queued
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** A notification.
+    **
+    ** @param signal the signal
+    *******************************************************************************/
+   record Notify(NotificationSignal signal) implements Queued
+   {
+   }
+
+
+
    /*******************************************************************************
     ** Constructor
     *******************************************************************************/
@@ -101,7 +165,7 @@ public class QbitAlertsSink implements AlertSinkInterface
          dropped.incrementAndGet();
          return;
       }
-      submit(() -> runtime.getEngine().raise(signal));
+      submit(new Raise(signal));
    }
 
 
@@ -116,7 +180,7 @@ public class QbitAlertsSink implements AlertSinkInterface
       {
          return;
       }
-      submit(() -> runtime.getEngine().resolve(resolution.getAlertTypeName(), resolution.getEffectiveDedupKey(), Resolution.PRODUCER, null, resolution.getNote()));
+      submit(new Resolve(resolution));
    }
 
 
@@ -128,7 +192,7 @@ public class QbitAlertsSink implements AlertSinkInterface
    public void heartbeat(String heartbeatKey, Instant at)
    {
       Instant when = at == null ? runtime.now() : at;
-      submit(() -> runtime.getEngine().heartbeat(heartbeatKey, when));
+      submit(new Beat(heartbeatKey, when));
    }
 
 
@@ -148,7 +212,7 @@ public class QbitAlertsSink implements AlertSinkInterface
       {
          signal.setOccurredAt(runtime.now());
       }
-      submit(() -> runtime.getNotifications().notify(signal));
+      submit(new Notify(signal));
    }
 
 
@@ -158,10 +222,11 @@ public class QbitAlertsSink implements AlertSinkInterface
     *******************************************************************************/
    public void flush()
    {
-      Runnable work;
-      while((work = queue.poll()) != null)
+      List<Queued> batch = new ArrayList<>();
+      while(queue.drainTo(batch, MAX_BATCH) > 0)
       {
-         work.run();
+         process(batch);
+         batch.clear();
       }
    }
 
@@ -211,30 +276,166 @@ public class QbitAlertsSink implements AlertSinkInterface
 
 
    /*******************************************************************************
-    ** Queue (or, synchronously, run) one piece of engine work.
+    ** Record what was queued, in order, with two savings for a batch (what
+    ** producers send all day is resolves for alerts that are not open and
+    ** heartbeats, a statement or three each when taken one at a time):
+    ** - consecutive resolves are checked against the open alerts in one query,
+    **   and only those with an open alert are resolved;
+    ** - heartbeats (which nothing else in the batch reads) are written once per
+    **   key, after the rest.
+    ** A batch of one is recorded exactly as a single call. Never throws.
     *******************************************************************************/
-   private void submit(EngineWork work)
+   void process(List<Queued> batch)
    {
-      Runnable runnable = () ->
+      Map<String, Instant> latestBeat = new LinkedHashMap<>();
+      Map<String, Integer> beatCount  = new HashMap<>();
+      List<Queued>         rest       = new ArrayList<>();
+      for(Queued item : batch)
+      {
+         if(item instanceof Beat beat)
+         {
+            if(StringUtils.hasContent(beat.heartbeatKey()))
+            {
+               latestBeat.put(beat.heartbeatKey(), beat.at());
+               beatCount.merge(beat.heartbeatKey(), 1, Integer::sum);
+            }
+         }
+         else
+         {
+            rest.add(item);
+         }
+      }
+
+      int index = 0;
+      while(index < rest.size())
+      {
+         int end = index;
+         while(end < rest.size() && rest.get(end) instanceof Resolve)
+         {
+            end++;
+         }
+         if(end - index > 1)
+         {
+            resolveRun(rest.subList(index, end));
+            index = end;
+         }
+         else
+         {
+            one(rest.get(index));
+            index++;
+         }
+      }
+
+      if(latestBeat.size() == 1 && beatCount.values().iterator().next() == 1)
+      {
+         Map.Entry<String, Instant> beat = latestBeat.entrySet().iterator().next();
+         one(new Beat(beat.getKey(), beat.getValue()));
+      }
+      else if(!latestBeat.isEmpty())
       {
          try
          {
-            work.run();
+            runtime.getEngine().heartbeats(latestBeat, beatCount);
          }
          catch(Exception e)
          {
-            LOG.warn("Error recording an alert signal", e);
+            LOG.warn("Error recording heartbeats together; recording each", e);
+            latestBeat.forEach((key, at) -> one(new Beat(key, at)));
          }
-      };
+      }
+   }
 
+
+
+   /*******************************************************************************
+    ** Several resolves in a row: one query finds which of them have an open
+    ** alert; the others have nothing to resolve.
+    *******************************************************************************/
+   private void resolveRun(List<Queued> run)
+   {
+      Set<String> open;
+      try
+      {
+         List<String> keys = new ArrayList<>();
+         for(Queued item : run)
+         {
+            keys.add(openKey(((Resolve) item).resolution()));
+         }
+         open = runtime.asSystem(() -> runtime.getEngine().openKeys(keys));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error checking open alerts for queued resolves; resolving each", e);
+         run.forEach(this::one);
+         return;
+      }
+      for(Queued item : run)
+      {
+         if(open.contains(openKey(((Resolve) item).resolution())))
+         {
+            one(item);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The open key a resolution closes (as AlertsEngine.resolve computes it).
+    *******************************************************************************/
+   private static String openKey(AlertResolution resolution)
+   {
+      String dedupKey = resolution.getEffectiveDedupKey();
+      return (AlertsEngine.openKey(resolution.getAlertTypeName(), StringUtils.hasContent(dedupKey) ? dedupKey : resolution.getAlertTypeName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record one queued call; a failure is logged, never thrown.
+    *******************************************************************************/
+   private void one(Queued item)
+   {
+      try
+      {
+         if(item instanceof Raise raise)
+         {
+            runtime.getEngine().raise(raise.signal());
+         }
+         else if(item instanceof Resolve resolve)
+         {
+            runtime.getEngine().resolve(resolve.resolution().getAlertTypeName(), resolve.resolution().getEffectiveDedupKey(), Resolution.PRODUCER, null, resolve.resolution().getNote());
+         }
+         else if(item instanceof Beat beat)
+         {
+            runtime.getEngine().heartbeat(beat.heartbeatKey(), beat.at());
+         }
+         else if(item instanceof Notify notify)
+         {
+            runtime.getNotifications().notify(notify.signal());
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error recording an alert signal", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Queue (or, synchronously, record) one producer call.
+    *******************************************************************************/
+   private void submit(Queued item)
+   {
       if(Boolean.TRUE.equals(runtime.getConfig().getSynchronousSink()))
       {
-         runnable.run();
+         process(List.of(item));
          return;
       }
 
       startIfNeeded();
-      if(!queue.offer(runnable))
+      if(!queue.offer(item))
       {
          dropped.incrementAndGet();
          if(overflowRaised.compareAndSet(false, true))
@@ -265,7 +466,7 @@ public class QbitAlertsSink implements AlertSinkInterface
       int count = perMinute.computeIfAbsent(alertTypeName, key -> new AtomicInteger()).incrementAndGet();
       if(count == FLOOD_PER_MINUTE + 1 && !FLOOD_TYPE.equals(alertTypeName))
       {
-         submit(() -> runtime.getEngine().raise(new AlertSignal(FLOOD_TYPE)
+         submit(new Raise(new AlertSignal(FLOOD_TYPE)
             .withDedupKey(alertTypeName)
             .withSeverity(AlertSeverity.WARNING)
             .withTitle("A producer is flooding alert signals")
@@ -291,10 +492,13 @@ public class QbitAlertsSink implements AlertSinkInterface
             {
                try
                {
-                  Runnable work = queue.poll(POLL_MILLIS, TimeUnit.MILLISECONDS);
-                  if(work != null)
+                  Queued first = queue.poll(POLL_MILLIS, TimeUnit.MILLISECONDS);
+                  if(first != null)
                   {
-                     work.run();
+                     List<Queued> batch = new ArrayList<>();
+                     batch.add(first);
+                     queue.drainTo(batch, MAX_BATCH - 1);
+                     process(batch);
                   }
                }
                catch(InterruptedException e)

src/main/java/com/kingsrook/qbits/alerts/engine/RuntimeFields.java

@@ -21,12 +21,14 @@ import java.util.Map;
 import java.util.Set;
 import com.kingsrook.qbits.alerts.model.AlertsAlert;
 import com.kingsrook.qbits.alerts.model.AlertsMonitor;
+import com.kingsrook.qbits.alerts.model.AlertsType;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
- ** Engine bookkeeping columns: evaluation leases, run results and notification
- ** timers that change on every scheduler pass. An update that touches only
+ ** Engine bookkeeping columns: evaluation leases, run results, notification
+ ** timers and a type's open count and last-raised time, which change on every
+ ** scheduler pass or raise. An update that touches only
  ** these (plus the primary key and modifyDate) is not record-audited; the core
  ** FIELD audit would otherwise write an audit row per monitor run and per
  ** reminder check. Any update that also changes a configuration or alert-state
@@ -42,7 +44,8 @@ public final class RuntimeFields
    private static final Map<String, Set<String>> BY_TABLE = Map.of(
       AlertsMonitor.TABLE_NAME, Set.of("claimedBy", "claimedUntil", "lastRunAt", "nextRunAt", "lastStatus", "lastValue", "lastError",
          "lastElapsedMs", "consecutiveFailures", "conditionSince"),
-      AlertsAlert.TABLE_NAME, Set.of("timedNotificationCheckedAt", "lastNotifiedAt", "recordRefCount"));
+      AlertsAlert.TABLE_NAME, Set.of("timedNotificationCheckedAt", "lastNotifiedAt", "recordRefCount"),
+      AlertsType.TABLE_NAME, AlertsReferenceCache.TYPE_COUNTER_FIELDS);
 
 
 

src/main/java/com/kingsrook/qbits/alerts/notifications/AlertsAudienceResolver.java

@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qbits.alerts.AlertsRuntime;
+import com.kingsrook.qbits.alerts.AlertsSecurity;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.alerts.NotificationAudience;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -33,8 +34,6 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
-import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -218,20 +217,14 @@ public class AlertsAudienceResolver
 
 
    /*******************************************************************************
-    ** A session for a user from the instance's authentication module (null
-    ** when it can't make one).
+    ** A session for a user: on their behalf via the alerts service principal
+    ** when declared, else the authentication module's (null when neither).
     *******************************************************************************/
    private QSession automatedSession(String userId)
    {
       try
       {
-         QInstance qInstance = runtime.getQInstance();
-         if(qInstance.getAuthentication() == null)
-         {
-            return (null);
-         }
-         QAuthenticationModuleInterface module = new QAuthenticationModuleDispatcher().getQModule(qInstance.getAuthentication());
-         return (module.createAutomatedSessionForUser(qInstance, userId));
+         return (AlertsSecurity.recipientSession(runtime.getQInstance(), runtime.getConfig().getServicePrincipal(), userId));
       }
       catch(Exception e)
       {

src/main/java/com/kingsrook/qbits/alerts/notifications/NotificationEmailer.java

@@ -26,6 +26,7 @@ import java.time.ZoneOffset;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -159,12 +160,14 @@ public class NotificationEmailer
          }
 
          Map<String, QRecord> settings = new HashMap<>();
+         Set<String> settingsRead = new HashSet<>();
          int sent = 0;
          int skipped = 0;
          int failed = 0;
          Map<String, List<QRecord[]>> digests = new LinkedHashMap<>();
          int afterId = 0;
-         while(true)
+         boolean lastPage = false;
+         while(!lastPage)
          {
             List<QRecord> recipients = AlertsDb.query(AlertsNotificationRecipient.TABLE_NAME, new QQueryFilter()
                .withSubFilter(eligible)
@@ -174,15 +177,30 @@ public class NotificationEmailer
             {
                break;
             }
+            lastPage = recipients.size() < SCAN;
             afterId = recipients.get(recipients.size() - 1).getValueInteger("id");
             Map<Integer, QRecord> notifications = notifications(recipients, now);
-            Set<String> pageUsers = recipients.stream().map(recipient -> recipient.getValueString("userId")).collect(Collectors.toSet());
-            settings.putAll(userSettings(pageUsers));
-            Map<String, Map<String, QRecord>> requestedPreferences = new HashMap<>();
-            for(QRecord preference : AlertsDb.query(AlertsNotificationPreference.TABLE_NAME, new QQueryFilter()
-               .withCriteria("userId", QCriteriaOperator.IN, new ArrayList<>(pageUsers))))
+            ///////////////////////////////////////////////////////////////////////////
+            // settings: only for users not read on an earlier page; preferences of  //
+            // every mode: only for the users of REQUESTED rows, which are the only  //
+            // rows that use them                                                    //
+            ///////////////////////////////////////////////////////////////////////////
+            Set<String> pageUsers = recipients.stream().map(recipient -> recipient.getValueString("userId")).filter(userId -> !settingsRead.contains(userId)).collect(Collectors.toSet());
+            if(!pageUsers.isEmpty())
             {
-               requestedPreferences.computeIfAbsent(preference.getValueString("userId"), k -> new HashMap<>()).put(preference.getValueString("category"), preference);
+               settings.putAll(userSettings(pageUsers));
+               settingsRead.addAll(pageUsers);
+            }
+            Set<String> requestedUsers = recipients.stream().filter(recipient -> EmailStatus.REQUESTED.name().equals(recipient.getValueString("emailStatus")))
+               .map(recipient -> recipient.getValueString("userId")).collect(Collectors.toSet());
+            Map<String, Map<String, QRecord>> requestedPreferences = new HashMap<>();
+            if(!requestedUsers.isEmpty())
+            {
+               for(QRecord preference : AlertsDb.query(AlertsNotificationPreference.TABLE_NAME, new QQueryFilter()
+                  .withCriteria("userId", QCriteriaOperator.IN, new ArrayList<>(requestedUsers))))
+               {
+                  requestedPreferences.computeIfAbsent(preference.getValueString("userId"), k -> new HashMap<>()).put(preference.getValueString("category"), preference);
+               }
             }
             for(QRecord recipient : recipients)
             {

src/main/java/com/kingsrook/qbits/alerts/notifications/NotificationService.java

@@ -19,6 +19,7 @@ package com.kingsrook.qbits.alerts.notifications;
 
 
 import java.io.Serializable;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
@@ -75,9 +76,13 @@ public class NotificationService
 
    static final int BATCH = 500;
 
+   static final Duration PURGE_EVERY = Duration.ofHours(1);
+
    private final AlertsRuntime          runtime;
    private final AlertsAudienceResolver resolver;
 
+   private volatile Instant lastPurgeAt;
+
 
 
    /*******************************************************************************
@@ -305,6 +310,25 @@ public class NotificationService
 
 
 
+   /*******************************************************************************
+    ** purge(), at most once every PURGE_EVERY on this node (the delivery job
+    ** calls this on every tick; retention is counted in days, so an hourly
+    ** purge deletes the same rows).
+    *******************************************************************************/
+   public int purgeIfDue() throws QException
+   {
+      Instant now  = runtime.now();
+      Instant last = lastPurgeAt;
+      if(last != null && !now.isBefore(last) && now.isBefore(last.plus(PURGE_EVERY)))
+      {
+         return (0);
+      }
+      lastPurgeAt = now;
+      return (purge());
+   }
+
+
+
    /*******************************************************************************
     ** Delete notifications and their recipient rows (as the system user).
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/alerts/notifications/NotificationSlackSender.java

@@ -124,7 +124,8 @@ public class NotificationSlackSender
          int                          failed  = 0;
          Map<String, List<QRecord[]>> digests = new LinkedHashMap<>();
          int afterId = 0;
-         while(true)
+         boolean lastPage = false;
+         while(!lastPage)
          {
             List<QRecord> recipients = AlertsDb.query(AlertsNotificationRecipient.TABLE_NAME, new QQueryFilter()
                .withCriteria("userId", QCriteriaOperator.IN, new ArrayList<>(preferences.keySet()))
@@ -136,6 +137,7 @@ public class NotificationSlackSender
             {
                break;
             }
+            lastPage = recipients.size() < NotificationEmailer.SCAN;
             afterId = recipients.get(recipients.size() - 1).getValueInteger("id");
             Map<Integer, QRecord> notifications = NotificationEmailer.notifications(recipients, now);
             for(QRecord recipient : recipients)

src/main/java/com/kingsrook/qbits/alerts/processes/AlertsScheduledSteps.java

@@ -78,7 +78,7 @@ public final class AlertsScheduledSteps
          AlertsRuntime runtime = AlertsRuntime.get();
          runtime.getSink().flush();
          runtime.getNotifications().sendDue();
-         runtime.getNotifications().purge();
+         runtime.getNotifications().purgeIfDue();
          new NotificationEmailer(runtime).run();
          new NotificationSlackSender(runtime).run();
          output.addValue(OUTPUT_COUNT, runtime.getDeliveryWorker().deliverDue());

src/main/resources/db/liquibase/qbit-alerts/alerts-defaults.changelog.sql

@@ -3,7 +3,7 @@
 -- qbit-alerts starter defaults: the two channels every install needs, and the
 -- two groups the qbit itself files alerts under, so Delivery health and the
 -- group lists are not empty before the first alert.
--- Include it from the host's master changelog AFTER alerts.changelog.sql:
+-- Load this file from the host's master changelog AFTER alerts.changelog.sql:
 --
 --   - include: { file: db/liquibase/qbit-alerts/alerts-defaults.changelog.sql }
 --

src/main/resources/db/liquibase/qbit-alerts/alerts.changelog.sql

@@ -1,7 +1,7 @@
 -- liquibase formatted sql
 
 -- qbit-alerts schema (QRun-IO/qqq docs/specs/2026-10-12-qbit-alerts-design.md section 4).
--- Include it from the host's master changelog:
+-- Load this file from the host's master changelog:
 --
 --   - include: { file: db/liquibase/qbit-alerts/alerts.changelog.sql }
 --
@@ -775,3 +775,7 @@ INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_i
 DELETE FROM alerts_escalation_step c WHERE c.channel_id IS NOT NULL AND NOT EXISTS (SELECT 1 FROM alerts_channel p WHERE p.id = c.channel_id);
 ALTER TABLE alerts_escalation_step ADD CONSTRAINT fk_alerts_escalation_step_channel_id FOREIGN KEY (channel_id) REFERENCES alerts_channel (id) ON DELETE RESTRICT;
 -- rollback ALTER TABLE alerts_escalation_step DROP CONSTRAINT IF EXISTS fk_alerts_escalation_step_channel_id;
+
+-- changeset qbit-alerts:55
+-- comment: The upkeep that follows every monitor evaluation (every evaluateEverySeconds, AlertsEngine.maintenance) reads the flapping alerts. No index led with flapping, so that read scanned every alert ever raised, resolved ones included.
+CREATE INDEX IF NOT EXISTS ix_alerts_alert_flapping ON alerts_alert (flapping);

src/main/resources/help/qbit-alerts.yaml

@@ -1433,7 +1433,7 @@ entries:
 - key: table:alerts_escalation_step;field:teamId
   summary: The team whose members are notified when the target type is Team.
 - key: table:alerts_alert;field:timedNotificationCheckedAt
-  summary: The last timed-notification check, used to avoid processing the alert twice in one minute.
+  summary: The last reminder or escalation attempt, used to avoid attempting the alert twice in one minute.
 - key: table:alerts_type;field:escalationPolicyId
   summary: The escalation policy for this alert type; takes precedence over the group policy.
 - key: table:alerts_group;field:escalationPolicyId

src/test/java/com/kingsrook/qbits/alerts/AlertsServicePrincipalTest.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.alerts;
+
+
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.serviceidentity.QDelegatedSession;
+import com.kingsrook.qqq.backend.core.serviceidentity.QServicePrincipalMetaData;
+import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentities;
+import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentitiesMetaData;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Per-recipient checks run on the recipient's behalf through the alerts
+ ** service principal when the host declares it; otherwise unchanged.
+ *******************************************************************************/
+class AlertsServicePrincipalTest extends BaseAlertsTest
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void recipientSessionsGoThroughTheDeclaredPrincipal() throws Exception
+   {
+      QSession before = AlertsSecurity.recipientSession(qInstance, config.getServicePrincipal(), "alice");
+      assertThat(before).isNotInstanceOf(QDelegatedSession.class);
+      assertThat(before.getUser().getIdReference()).isEqualTo("alice");
+
+      ServiceIdentitiesMetaData.declare(qInstance, new QServicePrincipalMetaData().withName("alerts").withMayActOnBehalfOfUsers(true));
+      QSession after = AlertsSecurity.recipientSession(qInstance, config.getServicePrincipal(), "alice");
+      assertThat(after).isInstanceOf(QDelegatedSession.class);
+      assertThat(after.getUser().getIdReference()).isEqualTo("alice");
+      assertThat(ServiceIdentities.principalOf(after)).isEqualTo("alerts");
+      assertThat(ServiceIdentities.attributionName(after)).endsWith("via service:alerts");
+   }
+}

src/test/java/com/kingsrook/qbits/alerts/StatementBudgetTest.java

@@ -0,0 +1,550 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.alerts;
+
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.alerts.api.AlertsApi;
+import com.kingsrook.qbits.alerts.engine.AlertsDb;
+import com.kingsrook.qbits.alerts.model.AlertsAlert;
+import com.kingsrook.qbits.alerts.model.AlertsChannel;
+import com.kingsrook.qbits.alerts.model.AlertsDelivery;
+import com.kingsrook.qbits.alerts.model.AlertsEscalationPolicy;
+import com.kingsrook.qbits.alerts.model.AlertsEscalationStep;
+import com.kingsrook.qbits.alerts.model.AlertsHeartbeat;
+import com.kingsrook.qbits.alerts.model.AlertsMonitor;
+import com.kingsrook.qbits.alerts.model.AlertsMute;
+import com.kingsrook.qbits.alerts.model.AlertsNotificationPreference;
+import com.kingsrook.qbits.alerts.model.AlertsNotificationRecipient;
+import com.kingsrook.qbits.alerts.model.AlertsSubscription;
+import com.kingsrook.qbits.alerts.model.AlertsType;
+import com.kingsrook.qbits.alerts.model.AlertsUserPreference;
+import com.kingsrook.qbits.alerts.processes.AlertsScheduledSteps;
+import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.NotificationAudience;
+import com.kingsrook.qqq.backend.core.alerts.NotificationCategory;
+import com.kingsrook.qqq.backend.core.alerts.NotificationSeverity;
+import com.kingsrook.qqq.backend.core.alerts.NotificationSignal;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
+import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData;
+import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface;
+import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** How many SQL statements the scheduled jobs and the polled requests issue.
+ ** The jobs run every few seconds to a minute around the clock, so their
+ ** statement count is a constant load on the host's database: each scenario
+ ** states a budget that must not grow with the number of alerts, monitors,
+ ** notifications or users it holds. "Before" in the comments is the same
+ ** scenario measured on 0.1.0-RC.18. Add -Dalerts.statements.dump=(scenario
+ ** name) to print a scenario's statements.
+ *******************************************************************************/
+class StatementBudgetTest extends BaseAlertsTest
+{
+   private static final String TYPE = "test.budget";
+
+
+
+   /*******************************************************************************
+    ** A provider that accepts every message.
+    *******************************************************************************/
+   public static class AcceptingProvider implements MessagingProviderInterface
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public String getType()
+      {
+         return ("alerts-budget");
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public SendMessageOutput sendMessage(SendMessageInput sendMessageInput)
+      {
+         return (new SendMessageOutput());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Email is configured (as on a host with an email service), and a chat
+    ** channel type exists.
+    *******************************************************************************/
+   @Override
+   protected void customizeConfig(AlertsQBitConfig config)
+   {
+      QMessagingProviderDispatcher.registerMessagingProvider(new AcceptingProvider());
+      qInstance.addMessagingProvider(new QMessagingProviderMetaData().withName("budget").withType("alerts-budget"));
+      config.withMessagingProviderName("budget").withFromAddress("alerts@example.com");
+      config.withChannelType(new QCodeReference(TestSteps.RecordingChannel.class));
+      TestSteps.SENT.clear();
+      TestSteps.nextOutcome = com.kingsrook.qbits.alerts.delivery.DeliveryResult.Outcome.SENT;
+   }
+
+
+
+   /*******************************************************************************
+    ** One declared type; every statement is recorded.
+    *******************************************************************************/
+   @Override
+   protected void customizeInstance(QInstance instance)
+   {
+      AlertsInstanceMetaData.ofOrWithNew(instance).withAlertType(new QAlertTypeMetaData()
+         .withName(TYPE).withLabel("Budget alert").withGroupName("budget").withDefaultSeverity(AlertSeverity.ERROR));
+      ((RDBMSBackendMetaData) instance.getBackend(BACKEND_NAME)).setActionStrategyCodeReference(new QCodeReference(StatementCounter.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** Something to measure.
+    *******************************************************************************/
+   @FunctionalInterface
+   private interface Work
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      void run() throws Exception;
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the work and report how many statements it issued.
+    *******************************************************************************/
+   private int measure(String scenario, Work work) throws Exception
+   {
+      StatementCounter.reset();
+      work.run();
+      int count = StatementCounter.count();
+      System.out.println("STATEMENTS " + scenario + " total=" + count + " " + StatementCounter.byTable());
+      if(scenario.equals(System.getProperty("alerts.statements.dump")))
+      {
+         StatementCounter.statements().forEach(sql -> System.out.println("   SQL " + sql.replaceAll("\\s+", " ")));
+      }
+      return (count);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord insert(String table, QRecord record) throws QException
+   {
+      return (runtime.asSystem(() -> AlertsDb.insert(table, record)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void update(String table, QRecord record) throws QException
+   {
+      runtime.asSystem(() -> AlertsDb.update(table, record));
+   }
+
+
+
+   /*******************************************************************************
+    ** Open alerts of the test type, one per key.
+    *******************************************************************************/
+   private List<QRecord> raise(int howMany) throws QException
+   {
+      List<QRecord> alerts = new ArrayList<>();
+      for(int i = 0; i < howMany; i++)
+      {
+         alerts.add(runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("key-" + i).withTitle("Alert " + i)));
+      }
+      return (alerts);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int escalateAndRemind() throws Exception
+   {
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new AlertsScheduledSteps.EscalateAndRemind().run(new RunBackendStepInput(), output);
+      return ((Integer) output.getValue(AlertsScheduledSteps.OUTPUT_COUNT));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int deliver() throws Exception
+   {
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new AlertsScheduledSteps.DeliverNotifications().run(new RunBackendStepInput(), output);
+      return ((Integer) output.getValue(AlertsScheduledSteps.OUTPUT_COUNT));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int evaluate() throws Exception
+   {
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new AlertsScheduledSteps.EvaluateMonitors().run(new RunBackendStepInput(), output);
+      return ((Integer) output.getValue(AlertsScheduledSteps.OUTPUT_COUNT));
+   }
+
+
+
+   /*******************************************************************************
+    ** A clean TABLE_QUERY monitor of the test type.
+    *******************************************************************************/
+   private Integer monitor(String name, String status, boolean perRecord) throws QException
+   {
+      runtime.getTypeSync().syncIfNeeded();
+      QRecord type = runtime.asSystem(() -> AlertsDb.first(AlertsType.TABLE_NAME, AlertsDb.eq("name", TYPE)));
+      Map<String, Object> definition = Map.of("tableName", WIDGET, "labelField", "name",
+         "filter", Map.of("criteria", List.of(Map.of("fieldName", "status", "operator", "EQUALS", "values", List.of(status)))));
+      return (insert(AlertsMonitor.TABLE_NAME, new QRecord()
+         .withValue("name", name).withValue("alertTypeId", type.getValue("id")).withValue("kind", "TABLE_QUERY").withValue("enabled", true)
+         .withValue("definition", JsonUtils.toJson(definition)).withValue("perRecord", perRecord).withValue("scheduleSeconds", 60).withValue("forMinutes", 0)
+         .withValue("consecutiveFailures", 0)).getValueInteger("id"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Email settings and one category preference for a user.
+    *******************************************************************************/
+   private void emailSettings(String userId, String category, String mode, int digestHour) throws QException
+   {
+      AlertsPreferences.ensure(runtime, loginAs(userId));
+      runtime.asSystem(() ->
+      {
+         QRecord settings = AlertsDb.first(AlertsUserPreference.TABLE_NAME, AlertsDb.eq("userId", userId));
+         AlertsDb.update(AlertsUserPreference.TABLE_NAME, new QRecord().withValue("id", settings.getValue("id")).withValue("emailAddress", userId + "@example.com")
+            .withValue("timeZone", "UTC").withValue("digestHour", digestHour));
+         AlertsDb.insert(AlertsNotificationPreference.TABLE_NAME, new QRecord().withValue("userId", userId).withValue("category", category)
+            .withValue("emailMode", mode).withValue("emailMinSeverity", "INFO"));
+         return (null);
+      });
+      loginAs(ADMIN);
+   }
+
+
+
+   /*******************************************************************************
+    ** Open, unacknowledged alerts with nothing due (no policy, no reminder):
+    ** the minute tick reads none of them and writes nothing.
+    ** Before: 452 statements (9 per alert, every minute). Now: 1.
+    *******************************************************************************/
+   @Test
+   void testEscalateAndRemindWithNothingDue() throws Exception
+   {
+      raise(50);
+      clock.advance(Duration.ofMinutes(2));
+      escalateAndRemind();
+      clock.advance(Duration.ofMinutes(1));
+      int count = measure("escalate.idle.50alerts", () -> assertThat(escalateAndRemind()).isZero());
+      assertThat(all(AlertsAlert.TABLE_NAME)).hasSize(50).allSatisfy(row -> assertThat(row.getValueInstant("timedNotificationCheckedAt")).isNull());
+      assertThat(count).isEqualTo(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** The same with an escalation policy and a reminder that are not due yet,
+    ** and an active mute that covers something else: one read of the page.
+    ** Before: 552 statements. Now: 1.
+    *******************************************************************************/
+   @Test
+   void testEscalateAndRemindWithPolicyNotDue() throws Exception
+   {
+      List<QRecord> alerts = raise(50);
+      QRecord policy = insert(AlertsEscalationPolicy.TABLE_NAME, new QRecord().withValue("name", "On call").withValue("enabled", true)
+         .withValue("repeatAfterMinutes", 30).withValue("maxRepeats", 2));
+      insert(AlertsEscalationStep.TABLE_NAME, new QRecord().withValue("policyId", policy.getValue("id")).withValue("stepOrder", 1).withValue("afterMinutes", 600)
+         .withValue("targetType", "USER").withValue("userId", "viewer-1").withValue("onlyIfStatus", "OPEN"));
+      update(AlertsType.TABLE_NAME, new QRecord().withValue("id", alerts.get(0).getValue("alertTypeId")).withValue("escalationPolicyId", policy.getValue("id"))
+         .withValue("reminderIntervalMinutes", 600));
+      insert(AlertsMute.TABLE_NAME, new QRecord().withValue("name", "Other").withValue("scopeType", "ALERT_TYPE").withValue("scopeValue", "another.type")
+         .withValue("enabled", true).withValue("effect", "SILENCE_NOTIFICATIONS").withValue("startsAt", clock.instant()).withValue("endsAt", clock.instant().plusSeconds(86_400)));
+      clock.advance(Duration.ofMinutes(2));
+      escalateAndRemind();
+      clock.advance(Duration.ofMinutes(1));
+      int count = measure("escalate.policyNotDue.50alerts", () -> assertThat(escalateAndRemind()).isZero());
+      assertThat(all(AlertsDelivery.TABLE_NAME)).isEmpty();
+      assertThat(count).isEqualTo(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Monitors that are not due cost one statement that reads no rows, whatever
+    ** their number, and the upkeep that follows costs one.
+    ** Before: 5 statements, reading every enabled monitor. Now: 2.
+    *******************************************************************************/
+   @Test
+   void testEvaluateMonitorsWithNothingDue() throws Exception
+   {
+      for(int i = 0; i < 20; i++)
+      {
+         monitor("Monitor " + i, "BROKEN", false);
+      }
+      assertThat(evaluate()).isEqualTo(20);
+      clock.advance(Duration.ofSeconds(30));
+      int count = measure("evaluate.idle.20monitors", () -> assertThat(evaluate()).isZero());
+      assertThat(count).isEqualTo(2);
+   }
+
+
+
+   /*******************************************************************************
+    ** Due monitors whose checks are clean: one claim for all, then per monitor
+    ** its check, its open alerts and its run result.
+    ** Before: 185 statements. Now: 84.
+    *******************************************************************************/
+   @Test
+   void testEvaluateDueCleanMonitors() throws Exception
+   {
+      for(int i = 0; i < 20; i++)
+      {
+         monitor("Monitor " + i, "BROKEN", false);
+      }
+      assertThat(evaluate()).isEqualTo(20);
+      clock.advance(Duration.ofSeconds(61));
+      int count = measure("evaluate.due.20cleanMonitors", () -> assertThat(evaluate()).isEqualTo(20));
+      assertThat(count).isLessThanOrEqualTo(5 + 20 * 4);
+   }
+
+
+
+   /*******************************************************************************
+    ** One per-record monitor that keeps firing for the same ten records.
+    ** Before: 205 statements for the refresh. Now: 120.
+    *******************************************************************************/
+   @Test
+   void testEvaluateFiringPerRecordMonitor() throws Exception
+   {
+      insertWidgets("BROKEN", "BROKEN", "BROKEN", "BROKEN", "BROKEN", "BROKEN", "BROKEN", "BROKEN", "BROKEN", "BROKEN");
+      Integer monitorId = monitor("Broken widgets", "BROKEN", true);
+      measure("evaluate.firing.open10", () -> assertThat(evaluate()).isEqualTo(1));
+      clock.advance(Duration.ofSeconds(61));
+      int count = measure("evaluate.firing.refresh10", () -> assertThat(evaluate()).isEqualTo(1));
+      assertThat(where(AlertsAlert.TABLE_NAME, AlertsDb.eq("monitorId", monitorId))).hasSize(10).allSatisfy(row ->
+      {
+         assertThat(row.getValueString("status")).isEqualTo("OPEN");
+         assertThat(row.getValueInteger("occurrenceCount")).isEqualTo(2);
+      });
+      assertThat(count).isLessThanOrEqualTo(25 + 10 * 10);
+   }
+
+
+
+   /*******************************************************************************
+    ** The delivery tick with nothing to deliver, on a host with email and
+    ** users who chose email and Slack.
+    ** Before: 9 statements. Now: 7 (the purge runs hourly, not every tick).
+    *******************************************************************************/
+   @Test
+   void testDeliverWithNothingDue() throws Exception
+   {
+      emailSettings("viewer-1", "JOB", "IMMEDIATE", 8);
+      emailSettings("viewer-2", "JOB", "IMMEDIATE", 8);
+      insert(AlertsNotificationPreference.TABLE_NAME, new QRecord().withValue("userId", "viewer-3").withValue("category", "JOB").withValue("slackMode", "IMMEDIATE"));
+      deliver();
+      clock.advance(Duration.ofSeconds(15));
+      int count = measure("deliver.idle", () -> assertThat(deliver()).isZero());
+      assertThat(count).isLessThanOrEqualTo(7);
+   }
+
+
+
+   /*******************************************************************************
+    ** Ten users who chose a daily digest, each holding twenty notifications,
+    ** outside their digest hour: nothing can be sent.
+    ** Before: 11 statements. Now: 7. (The 200 rows are still read on each tick.)
+    *******************************************************************************/
+   @Test
+   void testDeliverWithDigestBacklog() throws Exception
+   {
+      List<String> users = new ArrayList<>();
+      for(int i = 0; i < 10; i++)
+      {
+         users.add("viewer-" + i);
+         emailSettings("viewer-" + i, "JOB", "DAILY_DIGEST", 3);
+      }
+      for(int i = 0; i < 20; i++)
+      {
+         runtime.getNotifications().notify(new NotificationSignal("Job " + i + " finished").withCategory(NotificationCategory.JOB).withSeverity(NotificationSeverity.WARNING)
+            .withAudience(NotificationAudience.users(users.toArray(new String[0]))));
+      }
+      deliver();
+      clock.advance(Duration.ofSeconds(15));
+      int count = measure("deliver.digestBacklog.10users.200recipients", () -> assertThat(deliver()).isZero());
+      assertThat(all(AlertsNotificationRecipient.TABLE_NAME)).hasSize(200).allSatisfy(row -> assertThat(row.getValueString("emailStatus")).isEqualTo("NONE"));
+      assertThat(count).isLessThanOrEqualTo(7);
+   }
+
+
+
+   /*******************************************************************************
+    ** Sixty pending deliveries for two chat channels (two messages).
+    ** Before: 317 statements. Now: 22.
+    *******************************************************************************/
+   @Test
+   void testDeliverPendingToChannels() throws Exception
+   {
+      for(String name : List.of("ops", "dev"))
+      {
+         QRecord channel = insert(AlertsChannel.TABLE_NAME, new QRecord().withValue("name", name).withValue("label", "Chat " + name)
+            .withValue("channelType", "SLACK").withValue("enabled", true).withValue("detailLevel", "STANDARD").withValue("healthStatus", "HEALTHY").withValue("consecutiveFailures", 0));
+         insert(AlertsSubscription.TABLE_NAME, new QRecord().withValue("subscriberType", "CHANNEL").withValue("channelId", channel.getValue("id"))
+            .withValue("scopeType", "ALL").withValue("minSeverity", "WARNING").withValue("events", "RAISED,RESOLVED").withValue("deliveryMode", "IMMEDIATE").withValue("enabled", true));
+      }
+      raise(30);
+      assertThat(where(AlertsDelivery.TABLE_NAME, AlertsDb.eq("status", "PENDING"))).hasSize(60);
+      int count = measure("deliver.pending.60rows.2groups", () -> assertThat(deliver()).isEqualTo(60));
+      assertThat(all(AlertsDelivery.TABLE_NAME)).hasSize(60).allSatisfy(row ->
+      {
+         assertThat(row.getValueString("status")).isEqualTo("SENT");
+         assertThat(row.getValueInteger("attempts")).isEqualTo(1);
+         assertThat(row.getValue("claimedUntil")).isNull();
+      });
+      assertThat(TestSteps.SENT).hasSize(2);
+      assertThat(count).isLessThanOrEqualTo(25);
+   }
+
+
+
+   /*******************************************************************************
+    ** What producers send all day: resolves for alerts that are not open (a
+    ** process completed, a health check passed) and heartbeats. How the queue
+    ** is cut into batches depends on timing, so the budget is loose.
+    ** Before: 369 statements. Now: about 33.
+    *******************************************************************************/
+   @Test
+   void testQueuedResolvesAndHeartbeats() throws Exception
+   {
+      raise(2);
+      runtime.getConfig().withSynchronousSink(false);
+      int count = measure("sink.200resolves.50heartbeats", () ->
+      {
+         for(int i = 0; i < 200; i++)
+         {
+            runtime.getSink().resolve(new com.kingsrook.qqq.backend.core.alerts.AlertResolution(TYPE, i == 100 ? "key-1" : "not-open-" + i, "Fine"));
+            if(i % 4 == 0)
+            {
+               runtime.getSink().heartbeat("beat-" + (i % 10), null);
+            }
+         }
+         runtime.getSink().stop();
+      });
+      runtime.getConfig().withSynchronousSink(true);
+      assertThat(where(AlertsAlert.TABLE_NAME, AlertsDb.eq("status", "OPEN"))).singleElement().satisfies(row -> assertThat(row.getValueString("dedupKey")).isEqualTo("key-0"));
+      assertThat(all(AlertsHeartbeat.TABLE_NAME)).hasSize(5);
+      assertThat(all(AlertsHeartbeat.TABLE_NAME).stream().mapToInt(row -> row.getValueInteger("beatCount")).sum()).isEqualTo(50);
+      assertThat(count).isLessThanOrEqualTo(120);
+   }
+
+
+
+   /*******************************************************************************
+    ** A producer raising the same open alert again and again.
+    ** Before: 340 statements (17 a raise). Now: 200 (10 a raise).
+    *******************************************************************************/
+   @Test
+   void testRepeatedRaise() throws Exception
+   {
+      QRecord widget = insert(WIDGET, new QRecord().withValue("name", "W").withValue("status", "OK"));
+      AlertRecordRef ref = AlertRecordRef.of(WIDGET, widget.getValue("id"));
+      runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("same").withRecordRef(ref));
+      int count = measure("raise.repeat.20", () ->
+      {
+         for(int i = 0; i < 20; i++)
+         {
+            runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("same").withRecordRef(ref));
+         }
+      });
+      assertThat(all(AlertsAlert.TABLE_NAME)).singleElement().satisfies(row ->
+      {
+         assertThat(row.getValueInteger("occurrenceCount")).isEqualTo(21);
+         assertThat(row.getValueInteger("recordRefCount")).isEqualTo(1);
+      });
+      assertThat(runtime.asSystem(() -> AlertsDb.first(AlertsType.TABLE_NAME, AlertsDb.eq("name", TYPE))).getValueInteger("openCount")).isEqualTo(1);
+      assertThat(count).isLessThanOrEqualTo(20 * 10);
+   }
+
+
+
+   /*******************************************************************************
+    ** The bell every browser tab polls, and the record panel: a fixed number of
+    ** statements (3 and 6, unchanged), whatever the user holds.
+    *******************************************************************************/
+   @Test
+   void testBellAndRecordPanel() throws Exception
+   {
+      QRecord widget = insert(WIDGET, new QRecord().withValue("name", "W").withValue("status", "OK"));
+      insert(AlertsSubscription.TABLE_NAME, new QRecord().withValue("subscriberType", "USER").withValue("userId", "viewer-1")
+         .withValue("scopeType", "ALL").withValue("minSeverity", "WARNING").withValue("events", "RAISED").withValue("deliveryMode", "IMMEDIATE").withValue("enabled", true));
+      for(int i = 0; i < 5; i++)
+      {
+         runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("panel-" + i).withRecordRef(AlertRecordRef.of(WIDGET, widget.getValue("id"))));
+      }
+      for(int i = 0; i < 100; i++)
+      {
+         runtime.getNotifications().notify(new NotificationSignal("Job " + i + " finished").withCategory(NotificationCategory.JOB).withAudience(NotificationAudience.user("viewer-1")));
+      }
+
+      loginAs("viewer-1");
+      AlertsApi api = new AlertsApi(runtime);
+      assertThat(api.inboxSummary().get("unreadCount")).isEqualTo(105);
+      int bell = measure("bell.inboxSummary.105entries", () -> assertThat(api.inboxSummary().get("unreadCount")).isEqualTo(105));
+      int panel = measure("recordPanel.5alerts", () -> assertThat(api.recordPanel(WIDGET, String.valueOf(widget.getValue("id"))).get("openCount")).isEqualTo(5));
+      assertThat(bell).isLessThanOrEqualTo(3);
+      assertThat(panel).isLessThanOrEqualTo(6);
+   }
+}

src/test/java/com/kingsrook/qbits/alerts/StatementCounter.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.alerts;
+
+
+import java.io.Serializable;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy;
+
+
+/*******************************************************************************
+ ** The standard RDBMS action strategy, recording every statement it sends to
+ ** the database, so a test can state how many statements a job or a request
+ ** issues. Statements are what a host pays for: each one is a round trip and a
+ ** connection check-out.
+ *******************************************************************************/
+public class StatementCounter extends BaseRDBMSActionStrategy
+{
+   private static final List<String> STATEMENTS = Collections.synchronizedList(new ArrayList<>());
+
+   private static final Pattern TABLE = Pattern.compile("(?i)\\b(?:FROM|INTO|UPDATE)\\s+[`\"]?([a-z_]+)");
+
+
+
+   /*******************************************************************************
+    ** Forget what was recorded.
+    *******************************************************************************/
+   public static void reset()
+   {
+      STATEMENTS.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** How many statements ran since the last reset.
+    *******************************************************************************/
+   public static int count()
+   {
+      return (STATEMENTS.size());
+   }
+
+
+
+   /*******************************************************************************
+    ** How many statements ran against one table since the last reset.
+    *******************************************************************************/
+   public static int count(String tableName)
+   {
+      return (byTable().getOrDefault(tableName, 0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Statements by the first table each names.
+    *******************************************************************************/
+   public static Map<String, Integer> byTable()
+   {
+      Map<String, Integer> counts = new TreeMap<>();
+      synchronized(STATEMENTS)
+      {
+         for(String sql : STATEMENTS)
+         {
+            Matcher matcher = TABLE.matcher(sql);
+            counts.merge(matcher.find() ? matcher.group(1) : "?", 1, Integer::sum);
+         }
+      }
+      return (counts);
+   }
+
+
+
+   /*******************************************************************************
+    ** The recorded statements, in order.
+    *******************************************************************************/
+   public static List<String> statements()
+   {
+      synchronized(STATEMENTS)
+      {
+         return (new ArrayList<>(STATEMENTS));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public PreparedStatement executeUpdate(Connection connection, String sql, List<Object> params) throws SQLException
+   {
+      STATEMENTS.add(sql);
+      return (super.executeUpdate(connection, sql, params));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void executeBatchUpdate(Connection connection, String updateSQL, List<List<Serializable>> values) throws SQLException
+   {
+      STATEMENTS.add(updateSQL);
+      super.executeBatchUpdate(connection, updateSQL, values);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<Serializable> executeInsertForGeneratedIds(Connection connection, String sql, List<Object> params, QFieldMetaData primaryKeyField) throws SQLException
+   {
+      STATEMENTS.add(sql);
+      return (super.executeInsertForGeneratedIds(connection, sql, params, primaryKeyField));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Integer executeUpdateForRowCount(Connection connection, String sql, Object... params) throws SQLException
+   {
+      STATEMENTS.add(sql);
+      return (super.executeUpdateForRowCount(connection, sql, params));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException
+   {
+      STATEMENTS.add(String.valueOf(sql));
+      super.executeStatement(statement, sql, processor, params);
+   }
+}

src/test/java/com/kingsrook/qbits/alerts/engine/AlertsReferenceCacheTest.java

@@ -0,0 +1,203 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.alerts.engine;
+
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.Statement;
+import com.kingsrook.qbits.alerts.BaseAlertsTest;
+import com.kingsrook.qbits.alerts.StatementCounter;
+import com.kingsrook.qbits.alerts.model.AlertsGroup;
+import com.kingsrook.qbits.alerts.model.AlertsMute;
+import com.kingsrook.qbits.alerts.model.AlertsType;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** AlertsReferenceCache: what is held, what clears it (a write through QQQ, a
+ ** cluster event), what does not (the engine's own type counters), and what
+ ** happens to a table too large to hold.
+ *******************************************************************************/
+class AlertsReferenceCacheTest extends BaseAlertsTest
+{
+   private static final String TYPE = "test.cached";
+
+
+
+   /*******************************************************************************
+    ** One declared type; every statement is recorded.
+    *******************************************************************************/
+   @Override
+   protected void customizeInstance(QInstance instance)
+   {
+      AlertsInstanceMetaData.ofOrWithNew(instance).withAlertType(new QAlertTypeMetaData()
+         .withName(TYPE).withLabel("Cached alert").withGroupName("cached").withDefaultSeverity(AlertSeverity.ERROR));
+      ((RDBMSBackendMetaData) instance.getBackend(BACKEND_NAME)).setActionStrategyCodeReference(new QCodeReference(StatementCounter.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** Change rows behind QQQ's back (no record-change event).
+    *******************************************************************************/
+   private void sql(String sql) throws Exception
+   {
+      try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) qInstance.getBackend(BACKEND_NAME)); Statement statement = connection.createStatement())
+      {
+         statement.execute(sql);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void update(String table, QRecord record) throws QException
+   {
+      runtime.asSystem(() -> AlertsDb.update(table, record));
+   }
+
+
+
+   /*******************************************************************************
+    ** An admin's change to a type, a group or a mute (any write through QQQ) is
+    ** seen by the next signal.
+    *******************************************************************************/
+   @Test
+   void testWritesThroughQqqAreSeenAtOnce() throws Exception
+   {
+      QRecord alert = runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("one"));
+      assertThat(alert).isNotNull();
+
+      update(AlertsType.TABLE_NAME, new QRecord().withValue("id", alert.getValue("alertTypeId")).withValue("enabled", false));
+      assertThat(runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("two"))).isNull();
+
+      update(AlertsType.TABLE_NAME, new QRecord().withValue("id", alert.getValue("alertTypeId")).withValue("enabled", true));
+      assertThat(runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("two"))).isNotNull();
+
+      QRecord mute = runtime.asSystem(() -> AlertsDb.insert(AlertsMute.TABLE_NAME, new QRecord().withValue("name", "Window").withValue("scopeType", "ALERT_TYPE").withValue("scopeValue", TYPE)
+         .withValue("enabled", true).withValue("effect", "SUPPRESS_ALERTS").withValue("startsAt", clock.instant()).withValue("endsAt", clock.instant().plusSeconds(3600))));
+      assertThat(runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("three"))).isNull();
+
+      update(AlertsMute.TABLE_NAME, new QRecord().withValue("id", mute.getValue("id")).withValue("enabled", false));
+      assertThat(runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("three"))).isNotNull();
+
+      update(AlertsGroup.TABLE_NAME, new QRecord().withValue("id", alert.getValue("groupId")).withValue("label", "Renamed"));
+      assertThat(runtime.getReference().group(alert.getValue("groupId")).getValueString("label")).isEqualTo("Renamed");
+   }
+
+
+
+   /*******************************************************************************
+    ** A raise updates its type's open count and last-raised time; that does not
+    ** clear the held types (it would on every raise), and held type rows do not
+    ** carry those counters.
+    *******************************************************************************/
+   @Test
+   void testTypeCountersDoNotClear() throws Exception
+   {
+      QRecord alert = runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("same"));
+      runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("same"));
+      StatementCounter.reset();
+      runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("same"));
+      runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("other"));
+      assertThat(StatementCounter.statements()).noneMatch(sql -> sql.contains("FROM `alerts_type`") && !sql.contains("WHERE"));
+
+      QRecord stored = runtime.asSystem(() -> AlertsDb.byId(AlertsType.TABLE_NAME, alert.getValue("alertTypeId")));
+      assertThat(stored.getValueInteger("openCount")).isEqualTo(2);
+      assertThat(stored.getValueInstant("lastRaisedAt")).isEqualTo(clock.instant());
+      QRecord held = runtime.getReference().type(alert.getValue("alertTypeId"));
+      assertThat(held.getValueString("name")).isEqualTo(TYPE);
+      assertThat(held.getValues()).doesNotContainKeys("openCount", "lastRaisedAt");
+   }
+
+
+
+   /*******************************************************************************
+    ** A change that did not go through QQQ on this node (another node's write)
+    ** is held until the cluster event for that table arrives, or the entry
+    ** expires. A lookup returns a copy, so a caller cannot change what is held.
+    *******************************************************************************/
+   @Test
+   void testClusterEventClearsTheTable() throws Exception
+   {
+      runtime.getTypeSync().syncIfNeeded();
+      QRecord type = runtime.getReference().typeByName(TYPE);
+      assertThat(type.getValueString("label")).isEqualTo("Cached alert");
+      type.setValue("label", "Changed by a caller");
+      assertThat(runtime.getReference().typeByName(TYPE).getValueString("label")).isEqualTo("Cached alert");
+
+      sql("UPDATE alerts_type SET label = 'Changed elsewhere' WHERE name = '" + TYPE + "'");
+      assertThat(runtime.getReference().typeByName(TYPE).getValueString("label")).isEqualTo("Cached alert");
+
+      ClusterCaches.invalidateKey(AlertsReferenceCache.CACHE_NAME, AlertsGroup.TABLE_NAME);
+      assertThat(runtime.getReference().typeByName(TYPE).getValueString("label")).isEqualTo("Cached alert");
+
+      ClusterCaches.invalidateKey(AlertsReferenceCache.CACHE_NAME, AlertsType.TABLE_NAME);
+      assertThat(runtime.getReference().typeByName(TYPE).getValueString("label")).isEqualTo("Changed elsewhere");
+
+      sql("UPDATE alerts_type SET label = 'Changed again' WHERE name = '" + TYPE + "'");
+      ClusterCaches.invalidate(AlertsReferenceCache.CACHE_NAME);
+      assertThat(runtime.getReference().typeByName(TYPE).getValueString("label")).isEqualTo("Changed again");
+   }
+
+
+
+   /*******************************************************************************
+    ** A table with more rows than the cache holds is read from the database on
+    ** each lookup, as before the cache; the engine works the same.
+    *******************************************************************************/
+   @Test
+   void testTableTooLargeIsNotHeld() throws Exception
+   {
+      QRecord alert = runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("one"));
+      try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) qInstance.getBackend(BACKEND_NAME));
+         PreparedStatement insert = connection.prepareStatement("INSERT INTO alerts_group (name, label) VALUES (?, ?)"))
+      {
+         for(int i = 0; i <= AlertsReferenceCache.MAX_ROWS; i++)
+         {
+            insert.setString(1, "bulk-" + i);
+            insert.setString(2, "Bulk " + i);
+            insert.addBatch();
+         }
+         insert.executeBatch();
+      }
+      ClusterCaches.invalidateKey(AlertsReferenceCache.CACHE_NAME, AlertsGroup.TABLE_NAME);
+
+      assertThat(runtime.getReference().groups()).isNull();
+      assertThat(runtime.getReference().group(alert.getValue("groupId")).getValueString("name")).isEqualTo("cached");
+      sql("UPDATE alerts_group SET label = 'Read each time' WHERE name = 'cached'");
+      assertThat(runtime.getReference().group(alert.getValue("groupId")).getValueString("label")).isEqualTo("Read each time");
+      assertThat(runtime.getReference().group(-1)).isNull();
+      assertThat(new EscalationAndReminder(runtime).runDue()).isZero();
+      assertThat(runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("two"))).isNotNull();
+   }
+}

src/test/java/com/kingsrook/qbits/alerts/engine/EscalationAndReminderTest.java

@@ -18,6 +18,7 @@ package com.kingsrook.qbits.alerts.engine;
 
 
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import com.kingsrook.qbits.alerts.BaseAlertsTest;
 import com.kingsrook.qbits.alerts.model.AlertsAlert;
@@ -499,16 +500,19 @@ class EscalationAndReminderTest extends BaseAlertsTest
 
 
    /*******************************************************************************
-    ** The scan crosses page boundaries without missing alerts or retaining all rows.
+    ** The scan crosses page boundaries without missing alerts or retaining all rows:
+    ** every alert with a reminder due is attempted, which is what the minute
+    ** checkpoint records.
     *******************************************************************************/
    @Test
    void testAlertPages() throws Exception
    {
-      QRecord first = alert(0, AlertSeverity.ERROR);
+      QRecord first = alert(1, AlertSeverity.ERROR);
       for(int i = 1; i <= 100; i++)
       {
          runtime.getEngine().raise(new AlertSignal(TYPE).withDedupKey("page-" + i));
       }
+      clock.advance(Duration.ofMinutes(1));
       tick();
       assertThat(all(AlertsAlert.TABLE_NAME)).hasSize(101).allSatisfy(row ->
          assertThat(row.getValueInstant("timedNotificationCheckedAt")).isEqualTo(clock.instant()));
@@ -517,6 +521,31 @@ class EscalationAndReminderTest extends BaseAlertsTest
 
 
 
+   /*******************************************************************************
+    ** An alert with nothing due (no reminder, no policy) is not written by a tick:
+    ** no checkpoint, no new modify date. When its type gains a reminder, the next
+    ** tick attempts it.
+    *******************************************************************************/
+   @Test
+   void testNothingDueIsNotWritten() throws Exception
+   {
+      QRecord alert = alert(0, AlertSeverity.ERROR);
+      subscribe("viewer", "REMINDER");
+      Instant modified = fresh(alert).getValueInstant("modifyDate");
+      clock.advance(Duration.ofHours(3));
+      tick();
+      assertThat(fresh(alert).getValueInstant("timedNotificationCheckedAt")).isNull();
+      assertThat(fresh(alert).getValueInstant("modifyDate")).isEqualTo(modified);
+      assertThat(all(AlertsDelivery.TABLE_NAME)).isEmpty();
+
+      update(AlertsType.TABLE_NAME, new QRecord().withValue("id", alert.getValue("alertTypeId")).withValue("reminderIntervalMinutes", 60));
+      tick();
+      assertThat(all(AlertsDelivery.TABLE_NAME)).singleElement().satisfies(row -> assertThat(row.getValueString("kind")).isEqualTo("REMINDER"));
+      assertThat(fresh(alert).getValueInstant("timedNotificationCheckedAt")).isEqualTo(clock.instant());
+   }
+
+
+
    /*******************************************************************************
     ** A deliberate storage failure for transaction verification.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/alerts/engine/SinkAndScheduleTest.java

@@ -17,6 +17,7 @@
 package com.kingsrook.qbits.alerts.engine;
 
 
+import java.time.Instant;
 import java.util.List;
 import com.kingsrook.qbits.alerts.AlertsQBitConfig;
 import com.kingsrook.qbits.alerts.BaseAlertsTest;
@@ -139,6 +140,41 @@ class SinkAndScheduleTest extends BaseAlertsTest
 
 
 
+   /*******************************************************************************
+    ** One drained batch is recorded in queue order: a resolve closes only what
+    ** was raised before it, resolves for alerts that are not open do nothing,
+    ** and each heartbeat of a key is counted.
+    *******************************************************************************/
+   @Test
+   void testBatchKeepsOrder() throws Exception
+   {
+      Instant at = clock.instant();
+      runtime.getSink().process(List.of(
+         new QbitAlertsSink.Resolve(new AlertResolution("test.batch.a", null, "nothing is open yet")),
+         new QbitAlertsSink.Raise(new AlertSignal("test.batch.a").withTitle("A")),
+         new QbitAlertsSink.Beat("batch.beat", at),
+         new QbitAlertsSink.Raise(new AlertSignal("test.batch.b").withTitle("B")),
+         new QbitAlertsSink.Resolve(new AlertResolution("test.batch.a", null, "done")),
+         new QbitAlertsSink.Resolve(new AlertResolution("test.batch.never", null, "never raised")),
+         new QbitAlertsSink.Beat("batch.beat", at.plusSeconds(5)),
+         new QbitAlertsSink.Resolve(new AlertResolution("test.batch.b", "another-key", "not this key")),
+         new QbitAlertsSink.Beat("batch.other", at),
+         new QbitAlertsSink.Raise(new AlertSignal("test.batch.c").withTitle("C")),
+         new QbitAlertsSink.Resolve(new AlertResolution("test.batch.c", null, "done"))));
+
+      List<QRecord> alerts = all(AlertsAlert.TABLE_NAME);
+      assertThat(alerts).extracting(alert -> alert.getValueString("title") + " " + alert.getValueString("status")).containsExactlyInAnyOrder("A RESOLVED", "B OPEN", "C RESOLVED");
+      assertThat(alerts).filteredOn(alert -> "A".equals(alert.getValueString("title"))).singleElement().satisfies(alert -> assertThat(alert.getValueString("resolutionNote")).isEqualTo("done"));
+      assertThat(all(AlertsHeartbeat.TABLE_NAME)).extracting(beat -> beat.getValueString("heartbeatKey") + " " + beat.getValueInteger("beatCount") + " " + beat.getValueInstant("lastBeatAt"))
+         .containsExactlyInAnyOrder("batch.beat 2 " + at.plusSeconds(5), "batch.other 1 " + at);
+
+      runtime.getSink().process(List.of(new QbitAlertsSink.Beat("batch.beat", at.plusSeconds(9)), new QbitAlertsSink.Beat("batch.other", at.plusSeconds(9)), new QbitAlertsSink.Beat("batch.new", at)));
+      assertThat(all(AlertsHeartbeat.TABLE_NAME)).extracting(beat -> beat.getValueString("heartbeatKey") + " " + beat.getValueInteger("beatCount"))
+         .containsExactlyInAnyOrder("batch.beat 3", "batch.other 2", "batch.new 1");
+   }
+
+
+
    /*******************************************************************************
     ** Past 1,000 signals a minute for a type, the rest are counted, not stored.
     *******************************************************************************/

src/test/resources/alerts-h2.sql

@@ -445,6 +445,7 @@ ALTER TABLE alerts_alert ADD COLUMN timed_notification_checked_at TIMESTAMP;
 -- Foreign keys and their indexes (changesets 30-53).
 CREATE INDEX IF NOT EXISTS ix_alerts_alert_group_id ON alerts_alert (group_id);
 CREATE INDEX IF NOT EXISTS ix_alerts_alert_assigned_team_id ON alerts_alert (assigned_team_id);
+CREATE INDEX IF NOT EXISTS ix_alerts_alert_flapping ON alerts_alert (flapping);
 CREATE INDEX IF NOT EXISTS ix_alerts_type_escalation_policy_id ON alerts_type (escalation_policy_id);
 CREATE INDEX IF NOT EXISTS ix_alerts_group_parent_group_id ON alerts_group (parent_group_id);
 CREATE INDEX IF NOT EXISTS ix_alerts_group_owner_team_id ON alerts_group (owner_team_id);