ci: centralize branch verification and source scanning

Co-Authored-By: Codex <noreply@openai.com>
ci: adopt central orb 0.7.12 and stable gates (#21 )
2026-10-21 12:10:54 -05:00 · 2026-10-21 08:56:36 -05:00 · 2026-10-21 02:25:42 -05:00 · 2026-10-21 00:57:50 -05:00 · 2026-10-20 18:57:48 -05:00 · 2026-10-20 18:52:28 -05:00
32 changed files with 2633 additions and 122 deletions
@@ -1,60 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.15
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.15
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/
@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
  workflow_dispatch:
    inputs:
      dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
        type: boolean
        default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
  pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
  contents: write
-
+  actions: read
+  checks: read
 jobs:
  release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
    with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit
@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.15
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true
@@ -4,7 +4,16 @@ All notable changes are documented here. This project follows Keep a Changelog a

 ## [Unreleased]

+### Fixed
+- Query-spike alerts no longer hold seven days of query snapshots as records on every snapshot evaluation (~230,000 rows at kof22's rate). `SnapshotEvaluator` reads the window in 1,000-row keyset pages and keeps only each statement's current interval time, latest capture time and earlier interval times as primitive doubles; the spike rule (twelve prior samples, median, absence after an hour) is unchanged.
+- The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. The hourly analysis (`dbPerfAnalyze`) reads attribution the same way: `RuleContext.window("attribution", ...)` returns database sums per unit, table and operation instead of every detail row (it filled the kof22 worker heap to ~1.8 GB every hour).
+- Collection no longer stops at RUNNING on large production statement sets: list folding in the SQL normalizer is a linear token scan (the nested-repetition regular expressions overflowed the stack on 60 KB statements with long IN/VALUES lists). Any failure, including an `Error`, now closes the run as FAILED with the exception class, logs it at ERROR (class and frame only, never SQL text) and raises the collection alert; runs left RUNNING by a crash are closed as FAILED after three intervals.
+- Only one runtime attributes per JVM: starting attribution on a newer instance (a hot swap, or repeated server starts in tests) stops the previous one, so old instances are never pinned by a forgotten listener and flusher.
+- Each table has its own icon (the app icon is now `storage`), so nav entries are distinguishable and do not repeat icons used elsewhere.
+- Home tables keep at most four columns with plain headers (readable at every width): secondary facts such as calls and mean time, the session's query text, and a unit's kind and top table move into a second line of the first cell.
+
 ### Added
+- QQQ attribution and N+1 detection: a core statement execution listener (QQQ 4.1.0-RC.13) aggregates database time per route, scheduled job, process, table and statement shape in bounded memory, records repeated statement shapes per unit of work, and each pod flushes one batched insert per table every `attribution_flush_seconds`. Hosts without QQQ's application launcher call `DbPerfRuntime.of(instance).startAttribution()` after startup.

 - Secured database-performance tables, PostgreSQL statistics snapshots, daily rollups and configurable retention.
 - Audited PLAN/GENERIC and read-only ANALYZE query viewer with SQL validation, timeout, rollback and transient parameter handling.
@@ -4,7 +4,7 @@ Database performance for QQQ apps: PostgreSQL statement deltas, table and index

 ## Requirements

- Java 21 and QQQ **4.1.0-RC.11+**.
+- Java 21 and QQQ **4.1.0-RC.13+** (the statement execution listener SPI used for QQQ attribution).
 - PostgreSQL **13+**; **16+** for parameterized generic plans, with the pgJDBC driver (built against 42.7.14).
 - `pg_stat_statements` preloaded, created in the monitored database, and readable by the application role.
 - `pg_read_all_stats` granted to that role for complete statistics and other sessions' visibility.
@@ -0,0 +1,58 @@
+# Attribution runtime (phase B2)
+
+The producer registers `AttributionRuntimeService` with the QQQ application
+launcher. It starts one daemon per application instance after initialization,
+independently of scheduled-job leadership. Hosts without `QApplicationLauncher`
+should call `DbPerfRuntime.of(instance).startAttribution()` after migrations and
+instance validation, and `stopAttribution()` before disposing of the instance.
+`attributionEnabled=false` prevents listener registration and background work.
+Metadata production itself starts no threads, so existing metadata tests remain
+isolated.
+
+`AttributionFlusher` can also be constructed directly for deterministic tests:
+`refreshSettings()`, register `getListener()` with `StatementExecutionListeners`,
+and call `flushNow()`. Always close it to unregister. Production `start()` loads
+settings, registers the listener, schedules flushing, and installs a JVM shutdown
+hook. Closing stops scheduling, unregisters, and flushes once. Shutdown only
+persists completed units; hosts should drain application requests first.
+
+Statements never read settings from the database. A successful settings refresh
+publishes an immutable cached copy; a failed refresh keeps the previous copy.
+The initial copy disables recording until settings can be read. The configured
+flush interval takes effect when the next pass is scheduled. Disabled attribution
+still refreshes settings so administrators can re-enable it.
+
+Units are sampled once and buffered until completion, allowing HTTP route names
+to be finalized. A completed unit belongs to the window in which it ends; an
+abandoned unit is merged when that worker sees a different unit. No-unit work is
+sampled per statement and counted as one unit per statement. Sampling transitions
+retain separate aggregates for each effective rate. Values are observed counts,
+not extrapolated estimates. Daily rollups sum those counts across windows and
+pods and take the maximum duration.
+
+Each global map includes its overflow bucket within `attribution_max_keys`.
+One slot is reserved for a universal `(other)` key (all dimensions collapse,
+fingerprint is SHA-256 of `(other)`). Per-unit maps use the same bound with a
+minimum of two slots. Collapsed per-unit shapes cannot establish N+1 evidence
+and are excluded from repeat detection, avoiding false positives. Global repeat
+overflow sums qualifying repetitions but counts each affected unit only once.
+A mixed-rate overflow row has a null sample rate because there is no single
+valid rate for it. The SQL normalization cache holds at most 5,000 raw SQL keys
+in memory and clears on overflow; only normalized SQL is persisted, capped at
+4,000 characters.
+
+Window swaps and counter merges synchronize only in memory. Database work runs
+under the system user on the flusher thread with attribution suppressed. Each
+nonempty table receives one `InsertAction`, even above 1,000 rows; QQQ may page
+the underlying SQL. Both tables are written in one transaction. Failed windows
+are dropped, with rate-limited fixed-text logging, and never queued for retry.
+The repeat-pattern migration adds `sample_rate` without changing prior
+changesets. Existing retention already includes repeat patterns.
+
+Build using the locally installed statement-listener API:
+
+```powershell
+.\mvnw.cmd -o verify '-Dqqq.snapshot.version=4.1.0-dbperf-SNAPSHOT'
+```
+
+The normal POM default is `4.1.0-RC.13`.
@@ -241,14 +241,14 @@
   <profiles>
      <!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent  -->
      <!-- imports the released 4.1 BOM, this profile is active by default and     -->
-      <!-- imports qqq-bom-pom 4.1.0-RC.11 ahead of the parent's 4.0.0 BOM.         -->
+      <!-- imports qqq-bom-pom 4.1.0-RC.13 ahead of the parent's 4.0.0 BOM.         -->
      <profile>
         <id>qqq-snapshot</id>
         <activation>
            <activeByDefault>true</activeByDefault>
         </activation>
         <properties>
-            <qqq.snapshot.version>4.1.0-RC.11</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.13</qqq.snapshot.version>
         </properties>
         <dependencyManagement>
            <dependencies>
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qbits.dbperformance.alerts.SnapshotEvaluator;
 import com.kingsrook.qbits.dbperformance.api.DbPerformanceUiApi;
+import com.kingsrook.qbits.dbperformance.attribution.AttributionRuntimeService;
 import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess;
 import com.kingsrook.qbits.dbperformance.health.DbPerfHealthChecks;
 import com.kingsrook.qbits.dbperformance.metadata.DbPerfHelp;
@@ -34,6 +35,7 @@ import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgets;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
@@ -144,6 +146,7 @@ public class DbPerfQBitProducer implements QBitMetaDataProducer<DbPerfQBitConfig
      instance.addQBit(getQBitMetaData());
      DbPerfMetaData.addTo(instance, qBitConfig);
      new DbPerfRuntime(instance, qBitConfig).addSelfToInstance(instance);
+      instance.withRuntimeService(new QCodeReference(AttributionRuntimeService.class));
      DbPerfProcesses.addTo(instance, qBitConfig);
      SnapshotEvaluator.register(instance, qBitConfig);
      DbPerfHealthChecks.register(instance);
@@ -22,6 +22,7 @@ package com.kingsrook.qbits.dbperformance;


 import java.util.concurrent.atomic.AtomicReference;
+import com.kingsrook.qbits.dbperformance.attribution.AttributionFlusher;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -39,6 +40,13 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
   public static final String NAME = "dbPerformance";
   private final QInstance instance;
   private final DbPerfQBitConfig config;
+   private AttributionFlusher attributionFlusher;
+
+   /////////////////////////////////////////////////////////////////////////////
+   // statement listeners are JVM-wide: only one runtime attributes at a time //
+   // (a newer instance, e.g. after a hot swap or in tests, replaces the old) //
+   /////////////////////////////////////////////////////////////////////////////
+   private static final AtomicReference<DbPerfRuntime> ACTIVE_ATTRIBUTION = new AtomicReference<>();


   /*******************************************************************************
@@ -140,6 +148,38 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
   }


+   /*******************************************************************************
+    ** Called by the launcher, or explicitly by hosts without a launcher.
+    *******************************************************************************/
+   public synchronized void startAttribution()
+   {
+      if(attributionFlusher == null && Boolean.TRUE.equals(config.getAttributionEnabled()))
+      {
+         DbPerfRuntime previous = ACTIVE_ATTRIBUTION.getAndSet(this);
+         if(previous != null && previous != this)
+         {
+            previous.stopAttribution();
+         }
+         attributionFlusher = new AttributionFlusher(this);
+         attributionFlusher.start();
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Stop the per-instance listener and its daemon; safe to call repeatedly.
+    *******************************************************************************/
+   public synchronized void stopAttribution()
+   {
+      ACTIVE_ATTRIBUTION.compareAndSet(this, null);
+      if(attributionFlusher != null)
+      {
+         attributionFlusher.close();
+         attributionFlusher = null;
+      }
+   }
+
+
   /*******************************************************************************
    ** Checked service operation.
    *******************************************************************************/
@@ -22,9 +22,11 @@ package com.kingsrook.qbits.dbperformance.alerts;


 import java.time.Instant;
-import java.util.Comparator;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
 import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
@@ -38,6 +40,9 @@ import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
 import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
 import com.kingsrook.qqq.backend.core.alerts.QAlerts;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;

@@ -159,33 +164,31 @@ public class SnapshotEvaluator
    *******************************************************************************/
   private void spikes(RuleContext ctx, QRecord snapshot) throws Exception
   {
-      Map<Integer, List<QRecord>> history = ctx.window("query_snapshot", 7).stream().collect(Collectors.groupingBy(r -> r.getValueInteger("statementId")));
+      Map<Integer, StatementHistory> history = history(ctx, snapshot.getValueInteger("id"));
      for(QRecord statement : ctx.identities("statement"))
      {
-         List<QRecord> samples = history.getOrDefault(statement.getValueInteger("id"), List.of());
-         QRecord current = samples.stream().filter(r -> snapshot.getValueInteger("id").equals(r.getValueInteger("snapshotId"))).findFirst().orElse(null);
-         if(current == null)
+         StatementHistory samples = history.getOrDefault(statement.getValueInteger("id"), new StatementHistory());
+         if(samples.current == null)
         {
-            Instant last = samples.stream().map(r -> r.getValueInstant("capturedAt")).max(Comparator.naturalOrder()).orElse(Instant.EPOCH);
+            Instant last = samples.last == null ? Instant.EPOCH : samples.last;
            if(!last.plusSeconds(3600).isAfter(ctx.now))
            {
               signal(QUERY_SPIKE, ctx.backend, statement.getValueString("id"), false, "Query absent", "Statement absent from snapshots for at least one hour", statement, "db_perf_statement");
            }
            continue;
         }
-         List<Double> previous = samples.stream().filter(r -> r.getValueInstant("capturedAt").isBefore(ctx.now))
-            .map(r -> RuleContext.number(r, "totalMs")).sorted().toList();
-         if(previous.size() < 12)
+         double[] previous = samples.sortedPrevious();
+         if(previous.length < 12)
         {
-            if(RuleContext.number(current, "totalMs") < RuleContext.number(ctx.settings, "spikeMinTotalMs"))
+            if(samples.current < RuleContext.number(ctx.settings, "spikeMinTotalMs"))
            {
               signal(QUERY_SPIKE, ctx.backend, statement.getValueString("id"), false, "Query recovered", "Interval time is below the configured floor", statement, "db_perf_statement");
            }
            continue;
         }
-         int middle = previous.size() / 2;
-         double median = previous.size() % 2 == 0 ? (previous.get(middle - 1) + previous.get(middle)) / 2 : previous.get(middle);
-         double time = RuleContext.number(current, "totalMs");
+         int middle = previous.length / 2;
+         double median = previous.length % 2 == 0 ? (previous[middle - 1] + previous[middle]) / 2 : previous[middle];
+         double time = samples.current;
         boolean spike = time >= RuleContext.number(ctx.settings, "spikeMinTotalMs") && time >= median * RuleContext.number(ctx.settings, "spikeFactor");
         String normalized = SqlNormalizer.normalize(statement.getValueString("queryText"));
         String text = normalized.substring(0, Math.min(200, normalized.length()));
@@ -195,6 +198,90 @@ public class SnapshotEvaluator
   }


+   /*******************************************************************************
+    ** Seven days of per-statement interval times, read in keyset pages and kept
+    ** as primitive doubles. kof22 writes ~33,000 query snapshots a day, so the
+    ** window is ~230,000 rows; holding them as QRecords on every snapshot
+    ** evaluation cost hundreds of MB of worker heap.
+    *******************************************************************************/
+   static Map<Integer, StatementHistory> history(RuleContext ctx, Integer currentSnapshotId) throws Exception
+   {
+      Set<Integer> valid = ctx.window("snapshot", 7).stream().filter(r -> List.of("OK", "PARTIAL").contains(r.getValueString("status")))
+         .map(r -> r.getValueInteger("id")).collect(Collectors.toSet());
+      QQueryFilter filter = new QQueryFilter(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN, ctx.now.minusSeconds(7 * 86400L)),
+         new QFilterCriteria("capturedAt", QCriteriaOperator.LESS_THAN_OR_EQUALS, ctx.now));
+      Map<Integer, StatementHistory> history = new HashMap<>();
+      DbPerfStore.forEachPage("db_perf_query_snapshot", filter, DbPerfStore.BATCH_SIZE, page ->
+      {
+         for(QRecord row : page)
+         {
+            Integer snapshotId = row.getValueInteger("snapshotId");
+            if(!valid.contains(snapshotId))
+            {
+               continue;
+            }
+            StatementHistory statement = history.computeIfAbsent(row.getValueInteger("statementId"), ignored -> new StatementHistory());
+            Instant at = row.getValueInstant("capturedAt");
+            double totalMs = RuleContext.number(row, "totalMs");
+            if(statement.last == null || at.isAfter(statement.last))
+            {
+               statement.last = at;
+            }
+            if(snapshotId.equals(currentSnapshotId) && statement.current == null)
+            {
+               statement.current = totalMs;
+            }
+            if(at.isBefore(ctx.now))
+            {
+               statement.add(totalMs);
+            }
+         }
+      });
+      return (history);
+   }
+
+
+
+   /*******************************************************************************
+    ** One statement's samples: the current interval, the latest capture time
+    ** and earlier interval times.
+    *******************************************************************************/
+   static final class StatementHistory
+   {
+      private Double   current;
+      private Instant  last;
+      private double[] previous = new double[16];
+      private int      size;
+
+
+
+      /***************************************************************************
+       ** Append an earlier interval time.
+       ***************************************************************************/
+      void add(double value)
+      {
+         if(size == previous.length)
+         {
+            previous = Arrays.copyOf(previous, size * 2);
+         }
+         previous[size++] = value;
+      }
+
+
+
+      /***************************************************************************
+       ** Earlier interval times, ascending.
+       ***************************************************************************/
+      double[] sortedPrevious()
+      {
+         double[] sorted = Arrays.copyOf(previous, size);
+         Arrays.sort(sorted);
+         return (sorted);
+      }
+   }
+
+
+
   /*******************************************************************************
    ** Stable backend/subject keys make raises and resolutions restart-safe.
    *******************************************************************************/
@@ -25,17 +25,27 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
 import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;


 /*******************************************************************************
@@ -130,7 +140,7 @@ public class RuleContext
         {
            filter.withCriteria(new QFilterCriteria("backendName", QCriteriaOperator.EQUALS, backend));
         }
-         List<QRecord> rows = read("db_perf_" + suffix, filter);
+         List<QRecord> rows = suffix.equals("attribution") ? attributionTotals(filter) : read("db_perf_" + suffix, filter);
         if(!Set.of("snapshot", "attribution", "repeat_pattern").contains(suffix))
         {
            Set<Integer> ids = window("snapshot", days).stream()
@@ -144,6 +154,52 @@ public class RuleContext
   }


+   /*******************************************************************************
+    ** Most attribution groups the rules receive (prod: ~2,000 per day).
+    *******************************************************************************/
+   public static final int MAX_ATTRIBUTION_GROUPS = 50_000;
+
+
+
+   /*******************************************************************************
+    ** Attribution summed in the database per unit, table and operation: the
+    ** fields the rules read (statements, totalMs and the other counters), never
+    ** the day's detail rows. A day is ~570,000 rows in kof22 and loading it made
+    ** every hourly dbPerfAnalyze fill the worker heap (2026-10-16).
+    *******************************************************************************/
+   private static List<QRecord> attributionTotals(QQueryFilter filter) throws QException
+   {
+      List<String> dimensions = List.of("unitKind", "unitName", "tableName", "operation");
+      AggregateInput input = new AggregateInput("db_perf_attribution").withFilter(filter).withLimit(MAX_ATTRIBUTION_GROUPS)
+         .withTransaction(QContext.getQBackendTransaction());
+      Map<String, GroupBy> groupBys = new LinkedHashMap<>();
+      for(String dimension : dimensions)
+      {
+         GroupBy groupBy = new GroupBy(QFieldType.STRING, dimension);
+         groupBys.put(dimension, groupBy);
+         input.withGroupBy(groupBy);
+      }
+      Map<String, Aggregate> aggregates = new LinkedHashMap<>();
+      for(String counter : List.of("units", "statements", "rowsCount", "errors", "totalMs"))
+      {
+         aggregates.put(counter, new Aggregate(counter, AggregateOperator.SUM));
+      }
+      aggregates.put("maxMs", new Aggregate("maxMs", AggregateOperator.MAX));
+      aggregates.values().forEach(input::withAggregate);
+
+      List<QRecord> rows = new ArrayList<>();
+      for(AggregateResult result : new AggregateAction().execute(input).getResults())
+      {
+         QRecord row = new QRecord();
+         groupBys.forEach((field, groupBy) -> row.setValue(field, result.getGroupByValue(groupBy)));
+         aggregates.forEach((field, aggregate) -> row.setValue(field, result.getAggregateValue(aggregate)));
+         rows.add(row);
+      }
+      return (rows);
+   }
+
+
+
   /*******************************************************************************
    ** Latest inventory only: retired tables and dropped indexes cannot recur.
    *******************************************************************************/
@@ -0,0 +1,270 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.dbperformance.attribution;
+
+
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
+import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListeners;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** One daemon per runtime. Windows are detached before any database operation;
+ ** failed windows are dropped and later windows continue without retry queues.
+ *******************************************************************************/
+public class AttributionFlusher implements AutoCloseable
+{
+   private static final QLogger LOG = QLogger.getLogger(AttributionFlusher.class);
+   private final DbPerfRuntime runtime;
+   private final AttributionListener listener;
+   private ScheduledExecutorService executor;
+   private Thread shutdownHook;
+   private boolean closed;
+   private long failures;
+
+
+   /*******************************************************************************
+    ** Construct without starting threads, for hosts and deterministic tests.
+    *******************************************************************************/
+   public AttributionFlusher(DbPerfRuntime runtime)
+   {
+      this.runtime = runtime;
+      this.listener = new AttributionListener(runtime);
+   }
+
+
+   /*******************************************************************************
+    ** Register once after the instance and schema are ready.
+    *******************************************************************************/
+   public synchronized void start()
+   {
+      if(executor != null || closed || !Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()))
+      {
+         return;
+      }
+      refreshSettings();
+      StatementExecutionListeners.register(listener);
+      executor = Executors.newSingleThreadScheduledExecutor(task ->
+      {
+         Thread thread = new Thread(task, "db-perf-attribution");
+         thread.setDaemon(true);
+         return (thread);
+      });
+      shutdownHook = new Thread(this::close, "db-perf-attribution-shutdown");
+      Runtime.getRuntime().addShutdownHook(shutdownHook);
+      schedule();
+   }
+
+
+   /*******************************************************************************
+    ** Reschedule after each pass so administrative interval changes take effect.
+    *******************************************************************************/
+   private synchronized void schedule()
+   {
+      if(!closed)
+      {
+         executor.schedule(() ->
+         {
+            flushNow();
+            schedule();
+         }, listener.settings().flushSeconds(), TimeUnit.SECONDS);
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Exactly one InsertAction per nonempty table, with all rows in the window.
+    ** Serialize flushers only; statement threads never acquire this monitor.
+    *******************************************************************************/
+   public synchronized void flushNow()
+   {
+      boolean previous = AttributionListener.suppress(true);
+      try
+      {
+         Instant end = Instant.now();
+         AttributionListener.Window window = listener.drain(end);
+         List<QRecord> attribution = records(window.attribution(), window.start(), end, false);
+         List<QRecord> repeats = records(window.repeats(), window.start(), end, true);
+         if(!attribution.isEmpty() || !repeats.isEmpty())
+         {
+            runtime.transaction(() ->
+            {
+               insert("db_perf_attribution", attribution);
+               insert("db_perf_repeat_pattern", repeats);
+               return (null);
+            });
+         }
+      }
+      catch(Exception e)
+      {
+         failed();
+      }
+      finally
+      {
+         refreshSettings();
+         AttributionListener.suppress(previous);
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Refresh even after failed writes, including while attribution is disabled.
+    *******************************************************************************/
+   public synchronized void refreshSettings()
+   {
+      boolean previous = AttributionListener.suppress(true);
+      try
+      {
+         listener.refresh(new DbPerfSettingsService(runtime).ensure(runtime.getConfig().getBackendName()));
+      }
+      catch(Exception e)
+      {
+         failed();
+      }
+      finally
+      {
+         AttributionListener.suppress(previous);
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Rate-limited fixed text; database errors may contain sensitive SQL.
+    *******************************************************************************/
+   private void failed()
+   {
+      failures++;
+      if(failures == 1 || failures % 1000 == 0)
+      {
+         LOG.warn("Database Performance attribution write or settings refresh failed; the failed window is dropped");
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Convert nanoseconds to persisted milliseconds without floating point loss.
+    *******************************************************************************/
+   private List<QRecord> records(Map<AttributionListener.Key, AttributionListener.Counters> values, Instant start, Instant end, boolean repeat)
+   {
+      List<QRecord> records = new ArrayList<>(values.size());
+      for(Map.Entry<AttributionListener.Key, AttributionListener.Counters> entry : values.entrySet())
+      {
+         AttributionListener.Key key = entry.getKey();
+         AttributionListener.Counters counts = entry.getValue();
+         QRecord row = new QRecord().withValue("windowStart", start).withValue("windowEnd", end)
+            .withValue("podName", truncate(runtime.getConfig().getPodName(), 200)).withValue("unitKind", key.kind())
+            .withValue("unitName", truncate(key.name(), 300)).withValue("processName", truncate(key.process(), 200))
+            .withValue("tableName", truncate(key.table(), 200)).withValue("fingerprint", key.fingerprint())
+            .withValue("sqlText", truncate(counts.sql, 4000)).withValue("sampleRate", counts.sampleRate < 0 ? null : BigDecimal.valueOf(counts.sampleRate));
+         if(repeat)
+         {
+            row.withValue("unitsAffected", counts.units.sum()).withValue("totalRepeats", counts.statements.sum())
+               .withValue("maxRepeats", (int) Math.min(Integer.MAX_VALUE, counts.max.get()));
+         }
+         else
+         {
+            row.withValue("operation", key.operation()).withValue("units", counts.units.sum()).withValue("statements", counts.statements.sum())
+               .withValue("totalMs", BigDecimal.valueOf(counts.nanos.sum(), 6)).withValue("maxMs", BigDecimal.valueOf(counts.max.get(), 6))
+               .withValue("rowsCount", counts.rows.sum()).withValue("errors", counts.errors.sum());
+         }
+         records.add(row);
+      }
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Bound text at the persistence boundary after normalization.
+    *******************************************************************************/
+   private static String truncate(String value, int length)
+   {
+      return (value == null || value.length() <= length ? value : value.substring(0, length));
+   }
+
+
+   /*******************************************************************************
+    ** Reject record-level failures so the whole window transaction rolls back.
+    *******************************************************************************/
+   private static void insert(String table, List<QRecord> rows) throws Exception
+   {
+      if(!rows.isEmpty())
+      {
+         List<QRecord> inserted = InsertAction.executeForRecords(new InsertInput(table).withRecords(rows).withTransaction(QContext.getQBackendTransaction()));
+         if(inserted.size() != rows.size() || inserted.stream().anyMatch(row -> row.getErrors() != null && !row.getErrors().isEmpty()))
+         {
+            throw (new IllegalStateException("Attribution insert failed"));
+         }
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Listener exposed for deterministic delivery and registration tests.
+    *******************************************************************************/
+   public AttributionListener getListener()
+   {
+      return (listener);
+   }
+
+
+   /*******************************************************************************
+    ** Unregister first, stop scheduling, and flush one final detached window.
+    *******************************************************************************/
+   @Override
+   public synchronized void close()
+   {
+      if(closed)
+      {
+         return;
+      }
+      closed = true;
+      StatementExecutionListeners.unregister(listener);
+      if(executor != null)
+      {
+         executor.shutdownNow();
+      }
+      if(shutdownHook != null && Thread.currentThread() != shutdownHook)
+      {
+         try
+         {
+            Runtime.getRuntime().removeShutdownHook(shutdownHook);
+         }
+         catch(IllegalStateException expected)
+         {
+            // JVM shutdown is already running the hook.
+         }
+      }
+      flushNow();
+   }
+}
@@ -0,0 +1,425 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.dbperformance.attribution;
+
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
+import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
+import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementExecution;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListener;
+import com.kingsrook.qqq.backend.core.instrumentation.UnitOfWork;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Bounded, in-memory attribution. No table actions run on statement threads.
+ ** Completed units belong to the window in which they end, including long units.
+ *******************************************************************************/
+public class AttributionListener implements StatementExecutionListener
+{
+   private static final String OTHER = "(other)";
+   private static final Key OVERFLOW = new Key("OTHER", OTHER, OTHER, OTHER, "OTHER", SqlNormalizer.fingerprint(OTHER), -1);
+   private static final ThreadLocal<Boolean> SUPPRESSED = ThreadLocal.withInitial(() -> false);
+   private final DbPerfRuntime runtime;
+   private final ConcurrentHashMap<String, Shape> shapes = new ConcurrentHashMap<>();
+   private final ThreadLocal<Buffer> current = new ThreadLocal<>();
+   private volatile Settings settings = new Settings(false, 1.0, 60, 10000, 10);
+   private Window window = new Window(Instant.now(), new HashMap<>(), new HashMap<>());
+
+
+   /*******************************************************************************
+    ** Bind to one instance and monitored backend.
+    *******************************************************************************/
+   public AttributionListener(DbPerfRuntime runtime)
+   {
+      this.runtime = runtime;
+   }
+
+
+   /*******************************************************************************
+    ** Ignore self-observation, then sample and count without database access.
+    *******************************************************************************/
+   @Override
+   public void onStatement(StatementExecution execution)
+   {
+      Settings copy = settings;
+      if(!copy.enabled() || !Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()) || SUPPRESSED.get()
+         || !runtime.getConfig().getBackendName().equals(execution.backendName())
+         || (execution.tableName() != null && execution.tableName().startsWith(DbPerfQBitConfig.TABLE_NAME_PREFIX))
+         || (QContext.getQInstance() != null && QContext.getQInstance() != runtime.getInstance()))
+      {
+         return;
+      }
+      UnitOfWork unit = execution.unitOfWork();
+      Buffer buffer = current.get();
+      if(buffer != null && (unit == null || buffer.unit.getId() != unit.getId()))
+      {
+         merge(buffer);
+         current.remove();
+         buffer = null;
+      }
+      if(unit != null && buffer == null)
+      {
+         buffer = new Buffer(unit, copy, sampled(unit.getId(), copy.sampleRate()));
+         current.set(buffer);
+      }
+      if(buffer == null ? !sampled(ThreadLocalRandom.current().nextLong(), copy.sampleRate()) : !buffer.sampled)
+      {
+         return;
+      }
+      Shape shape = shape(execution.sql());
+      Key key = new Key(unit == null ? "OTHER" : unit.getKind().name(), unit == null ? name(execution.processName()) : "",
+         execution.processName(), execution.tableName(), execution.operation() == null ? "OTHER" : execution.operation().name(), shape.fingerprint(), buffer == null ? copy.sampleRate() : buffer.settings.sampleRate());
+      if(buffer == null)
+      {
+         synchronized(this)
+         {
+            Key target = bounded(window.attribution(), key, copy.maxKeys());
+            Counters counts = window.attribution().computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : shape.sql(), key.sampleRate()));
+            counts.sampleRate(key.sampleRate());
+            counts.add(execution);
+            counts.units.increment();
+         }
+      }
+      else
+      {
+         Key target = bounded(buffer.counts, key, Math.max(2, buffer.settings.maxKeys()));
+         buffer.counts.computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : shape.sql(), key.sampleRate())).add(execution);
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Read the final route name, then release all per-thread counters.
+    *******************************************************************************/
+   @Override
+   public void onUnitOfWorkEnd(UnitOfWork unit)
+   {
+      Buffer buffer = current.get();
+      if(buffer != null && buffer.unit.getId() == unit.getId())
+      {
+         current.remove();
+         if(settings.enabled() && Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()))
+         {
+            merge(buffer);
+         }
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Merge each unit once per destination, even when several shapes overflow.
+    ** Repeat counts are combined across operations before threshold evaluation.
+    *******************************************************************************/
+   private synchronized void merge(Buffer buffer)
+   {
+      Set<Key> touched = new HashSet<>();
+      Map<Key, Counters> repeats = new HashMap<>();
+      for(Map.Entry<Key, Counters> entry : buffer.counts.entrySet())
+      {
+         Key local = entry.getKey();
+         Key key = local == OVERFLOW ? OVERFLOW : new Key(local.kind(), name(buffer.unit.getName()), local.process(), local.table(), local.operation(), local.fingerprint(), buffer.settings.sampleRate());
+         Key target = bounded(window.attribution(), key, settings.maxKeys());
+         Counters counts = window.attribution().computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : entry.getValue().sql, entry.getValue().sampleRate));
+         counts.merge(entry.getValue());
+         if(touched.add(target))
+         {
+            counts.units.increment();
+         }
+         // Collapsed shapes cannot prove repetition of any one fingerprint.
+         if(local != OVERFLOW)
+         {
+            Key repeat = new Key(key.kind(), key.name(), key.process(), key.table(), null, key.fingerprint(), key.sampleRate());
+            repeats.computeIfAbsent(repeat, ignored -> new Counters(entry.getValue().sql, entry.getValue().sampleRate)).merge(entry.getValue());
+         }
+      }
+      Map<Key, Long> totals = new HashMap<>();
+      Map<Key, Long> maxima = new HashMap<>();
+      for(Map.Entry<Key, Counters> entry : repeats.entrySet())
+      {
+         long count = entry.getValue().statements.sum();
+         if(count >= buffer.settings.threshold())
+         {
+            Key target = bounded(window.repeats(), entry.getKey(), settings.maxKeys());
+            window.repeats().computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : entry.getValue().sql, entry.getValue().sampleRate));
+            window.repeats().get(target).sampleRate(entry.getValue().sampleRate);
+            totals.merge(target, count, Long::sum);
+            maxima.merge(target, count, Math::max);
+         }
+      }
+      for(Map.Entry<Key, Long> entry : totals.entrySet())
+      {
+         Counters counts = window.repeats().get(entry.getKey());
+         counts.units.increment();
+         counts.statements.add(entry.getValue());
+         counts.max.accumulateAndGet(maxima.get(entry.getKey()), Math::max);
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Reserve one slot for a universal overflow key, preserving the strict cap.
+    *******************************************************************************/
+   private static Key bounded(Map<Key, Counters> counts, Key key, int limit)
+   {
+      return (counts.containsKey(key) || (key != OVERFLOW && counts.size() < limit - (counts.containsKey(OVERFLOW) ? 0 : 1)) ? key : OVERFLOW);
+   }
+
+
+   /*******************************************************************************
+    ** Normalize once per raw SQL shape; cold-cache admission alone is serialized.
+    *******************************************************************************/
+   private Shape shape(String sql)
+   {
+      String raw = sql == null ? "" : sql;
+      Shape cached = shapes.get(raw);
+      if(cached != null)
+      {
+         return (cached);
+      }
+      String normalized = SqlNormalizer.normalize(raw);
+      Shape created = new Shape(normalized, SqlNormalizer.fingerprint(normalized));
+      synchronized(shapes)
+      {
+         if(shapes.size() >= 5000)
+         {
+            shapes.clear();
+         }
+         shapes.put(raw, created);
+      }
+      return (created);
+   }
+
+
+   /*******************************************************************************
+    ** SplitMix64 finalizer distributes sequential unit IDs without allocations.
+    *******************************************************************************/
+   private static boolean sampled(long id, double rate)
+   {
+      long hash = (id ^ (id >>> 30)) * 0xbf58476d1ce4e5b9L;
+      hash = (hash ^ (hash >>> 27)) * 0x94d049bb133111ebL;
+      hash ^= hash >>> 31;
+      return ((hash >>> 11) * 0x1.0p-53 < rate);
+   }
+
+
+   /*******************************************************************************
+    ** Use a stable label for units without a name.
+    *******************************************************************************/
+   private static String name(String value)
+   {
+      return (value == null || value.isBlank() ? "(none)" : value);
+   }
+
+
+   /*******************************************************************************
+    ** Detach an immutable-in-practice window; no writers retain its counters.
+    *******************************************************************************/
+   synchronized Window drain(Instant end)
+   {
+      Window previous = window;
+      window = new Window(end, new HashMap<>(), new HashMap<>());
+      return (previous);
+   }
+
+
+   /*******************************************************************************
+    ** Publish a validated immutable settings copy only from the flusher.
+    *******************************************************************************/
+   void refresh(QRecord row)
+   {
+      double rate = row.getValueBigDecimal("attributionSampleRate") == null ? 1.0 : row.getValueBigDecimal("attributionSampleRate").doubleValue();
+      settings = new Settings(!Boolean.FALSE.equals(row.getValueBoolean("attributionEnabled")), Math.max(0, Math.min(1, rate)),
+         positive(row, "attributionFlushSeconds", 60), positive(row, "attributionMaxKeys", 10000), positive(row, "nPlusOneThreshold", 10));
+   }
+
+
+   /*******************************************************************************
+    ** Invalid administrative values use defaults.
+    *******************************************************************************/
+   private static int positive(QRecord row, String field, int fallback)
+   {
+      Integer value = row.getValueInteger(field);
+      return (value == null || value <= 0 ? fallback : value);
+   }
+
+
+   /*******************************************************************************
+    ** Current settings for scheduling and window annotations.
+    *******************************************************************************/
+   Settings settings()
+   {
+      return (settings);
+   }
+
+
+   /*******************************************************************************
+    ** Set thread suppression, returning the previous state for nested callers.
+    *******************************************************************************/
+   static boolean suppress(boolean value)
+   {
+      boolean previous = SUPPRESSED.get();
+      if(value)
+      {
+         SUPPRESSED.set(true);
+      }
+      else
+      {
+         SUPPRESSED.remove();
+      }
+      return (previous);
+   }
+
+
+   /*******************************************************************************
+    ** Immutable administrative settings.
+    *******************************************************************************/
+   record Settings(boolean enabled, double sampleRate, int flushSeconds, int maxKeys, int threshold)
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Attribution dimensions; sampling transitions remain separately interpretable.
+    ** Null operation denotes a repeat pattern.
+    *******************************************************************************/
+   record Key(String kind, String name, String process, String table, String operation, String fingerprint, double sampleRate)
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Cached literal-free text and its digest.
+    *******************************************************************************/
+   private record Shape(String sql, String fingerprint)
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Detached window transferred to the flusher.
+    *******************************************************************************/
+   record Window(Instant start, Map<Key, Counters> attribution, Map<Key, Counters> repeats)
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Thread-confined unit, including its once-only sampling decision.
+    *******************************************************************************/
+   private static class Buffer
+   {
+      private final UnitOfWork unit;
+      private final Settings settings;
+      private final boolean sampled;
+      private final Map<Key, Counters> counts = new HashMap<>();
+
+
+      /*******************************************************************************
+       ** Capture unit settings so refresh cannot partially sample a unit.
+       *******************************************************************************/
+      private Buffer(UnitOfWork unit, Settings settings, boolean sampled)
+      {
+         this.unit = unit;
+         this.settings = settings;
+         this.sampled = sampled;
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Additive counters plus an atomic maximum, shared by both window tables.
+    *******************************************************************************/
+   static class Counters
+   {
+      final String sql;
+      double sampleRate;
+      final LongAdder units = new LongAdder();
+      final LongAdder statements = new LongAdder();
+      final LongAdder nanos = new LongAdder();
+      final AtomicLong max = new AtomicLong();
+      final LongAdder rows = new LongAdder();
+      final LongAdder errors = new LongAdder();
+
+
+      /*******************************************************************************
+       ** Retain only normalized SQL.
+       *******************************************************************************/
+      Counters(String sql, double sampleRate)
+      {
+         this.sql = sql;
+         this.sampleRate = sampleRate;
+      }
+
+
+      /*******************************************************************************
+       ** Count one completed statement, including failures and unknown row counts.
+       *******************************************************************************/
+      void add(StatementExecution execution)
+      {
+         statements.increment();
+         nanos.add(execution.elapsedNanos());
+         max.accumulateAndGet(execution.elapsedNanos(), Math::max);
+         rows.add(execution.rowCount() == null ? 0 : execution.rowCount());
+         if(!execution.succeeded())
+         {
+            errors.increment();
+         }
+      }
+
+
+      /*******************************************************************************
+       ** An overflow bucket spanning sampling rates has no single valid rate.
+       *******************************************************************************/
+      void sampleRate(double rate)
+      {
+         if(sampleRate != rate)
+         {
+            sampleRate = -1;
+         }
+      }
+
+
+      /*******************************************************************************
+       ** Fold statement totals; distinct units are managed by the caller.
+       *******************************************************************************/
+      void merge(Counters other)
+      {
+         sampleRate(other.sampleRate);
+         statements.add(other.statements.sum());
+         nanos.add(other.nanos.sum());
+         max.accumulateAndGet(other.max.get(), Math::max);
+         rows.add(other.rows.sum());
+         errors.add(other.errors.sum());
+      }
+   }
+}
@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.dbperformance.attribution;
+
+
+import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
+import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** QApplicationLauncher lifecycle, after instance initialization and migrations.
+ *******************************************************************************/
+public class AttributionRuntimeService implements QRuntimeServiceInterface
+{
+   private DbPerfRuntime runtime;
+
+
+   /*******************************************************************************
+    ** Name used by the host launcher.
+    *******************************************************************************/
+   @Override
+   public String getName()
+   {
+      return ("dbPerformanceAttribution");
+   }
+
+
+   /*******************************************************************************
+    ** Start on each application pod, independently of the job scheduler.
+    *******************************************************************************/
+   @Override
+   public void start(QInstance instance)
+   {
+      runtime = DbPerfRuntime.of(instance);
+      runtime.startAttribution();
+   }
+
+
+   /*******************************************************************************
+    ** Stop listener delivery and persist the final completed units.
+    *******************************************************************************/
+   @Override
+   public void stop()
+   {
+      if(runtime != null)
+      {
+         runtime.stopAttribution();
+      }
+   }
+}
@@ -66,6 +66,48 @@ public final class DbPerfStore
   }


+   /*******************************************************************************
+    ** Keyset-paged read for tables that can hold more rows than fit in the heap
+    ** (attribution writes hundreds of thousands per day). At most pageSize
+    ** records are held at once; the consumer must not keep the page.
+    *******************************************************************************/
+   public static void forEachPage(String table, QQueryFilter filter, int pageSize, PageConsumer consumer) throws Exception
+   {
+      Serializable lastId = 0;
+      while(true)
+      {
+         QQueryFilter page = new QQueryFilter().withSubFilter(filter).withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, lastId))
+            .withOrderBy(new QFilterOrderBy("id", true)).withLimit(pageSize);
+         List<QRecord> rows = query(table, page);
+         if(rows.isEmpty())
+         {
+            return;
+         }
+         consumer.accept(rows);
+         if(rows.size() < pageSize)
+         {
+            return;
+         }
+         lastId = rows.getLast().getValue("id");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Receives one bounded page from forEachPage.
+    *******************************************************************************/
+   @FunctionalInterface
+   public interface PageConsumer
+   {
+      /***************************************************************************
+       ** Handle one page.
+       ***************************************************************************/
+      void accept(List<QRecord> page) throws Exception;
+   }
+
+
+
   /*******************************************************************************
    ** First matching record.
    *******************************************************************************/
@@ -168,7 +168,19 @@ public class RollupService
   {
      Map<String, QRecord> groups = new LinkedHashMap<>();
      List<String> dimensions = List.of("unitKind", "unitName", "processName", "tableName", "operation", "fingerprint");
-      for(QRecord row : DbPerfStore.query("db_perf_attribution", range("windowStart", start, end)))
+      DbPerfStore.forEachPage("db_perf_attribution", range("windowStart", start, end), DbPerfStore.BATCH_SIZE, page -> mergeAttribution(day, page, dimensions, groups));
+      replace("db_perf_attribution_daily", day, new ArrayList<>(groups.values()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Fold one page of attribution detail into the day's groups. Memory is
+    ** bounded by distinct dimension keys, never by the day's detail row count.
+    *******************************************************************************/
+   private static void mergeAttribution(LocalDate day, List<QRecord> page, List<String> dimensions, Map<String, QRecord> groups)
+   {
+      for(QRecord row : page)
      {
         StringBuilder key = new StringBuilder();
         for(String field : dimensions)
@@ -187,7 +199,6 @@ public class RollupService
         sumDecimal(aggregate, row, "totalMs");
         aggregate.setValue("maxMs", SnapshotCollector.decimal(aggregate, "maxMs").max(SnapshotCollector.decimal(row, "maxMs")));
      }
-      replace("db_perf_attribution_daily", day, new ArrayList<>(groups.values()));
   }


@@ -48,6 +48,7 @@ import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
 import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
 import com.kingsrook.qbits.dbperformance.provider.StatementStats;
 import com.kingsrook.qbits.dbperformance.provider.TableStats;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;


 /*******************************************************************************
@@ -62,6 +64,8 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat
 *******************************************************************************/
 public class SnapshotCollector
 {
+   private static final QLogger LOG = QLogger.getLogger(SnapshotCollector.class);
+
   private final DbPerfRuntime runtime;
   private final Clock clock;
   private final DbStatsProvider providerOverride;
@@ -104,6 +108,7 @@ public class SnapshotCollector
               throw (new IllegalArgumentException("Backend is not configured for this runtime"));
            }
            QRecord settings = new DbPerfSettingsService(runtime).ensure(backendName);
+            failAbandonedRuns(backendName, settings);
            QRecord inserted = runtime.asSystem(() -> DbPerfStore.insert("db_perf_snapshot", List.of(snapshot)).getFirst());
            snapshot.setValue("id", inserted.getValueInteger("id"));
            runtime.transaction(() ->
@@ -121,9 +126,19 @@ public class SnapshotCollector
               return (null);
            });
         }
-         catch(Exception e)
+         catch(Throwable e)
         {
-            // Never persist a JDBC exception message: it may contain SQL literals.
+            //////////////////////////////////////////////////////////////////////////
+            // Errors too (a StackOverflowError once left runs RUNNING with no log). //
+            // Never persist or log a JDBC exception message: it may contain SQL    //
+            // literals. The class and the throwing frame identify the cause.       //
+            //////////////////////////////////////////////////////////////////////////
+            if(e instanceof OutOfMemoryError outOfMemory)
+            {
+               throw (outOfMemory);
+            }
+            LOG.error("Database performance snapshot failed", logPair("backend", backendName), logPair("snapshotId", snapshot.getValueInteger("id")),
+               logPair("exception", e.getClass().getName()), logPair("at", e.getStackTrace().length == 0 ? null : e.getStackTrace()[0].toString()));
            snapshot.withValue("status", "FAILED").withValue("errorMessage", "Collection failed (" + e.getClass().getSimpleName() + "). Check database connectivity, schema and statistics access.")
               .withValue("durationMs", (int) Math.min(Integer.MAX_VALUE, (System.nanoTime() - started) / 1000000));
            if(snapshot.getValueInteger("id") != null)
@@ -149,6 +164,33 @@ public class SnapshotCollector
   }


+   /*******************************************************************************
+    ** Runs left RUNNING by a crashed pod or an earlier failure are closed as
+    ** FAILED once they are older than three collection intervals.
+    *******************************************************************************/
+   private void failAbandonedRuns(String backendName, QRecord settings) throws Exception
+   {
+      int     interval = positive(settings, "snapshotIntervalMinutes", 5);
+      Instant cutoff   = clock.instant().minus(Duration.ofMinutes(3L * interval));
+      List<QRecord> abandoned = runtime.asSystem(() -> DbPerfStore.query("db_perf_snapshot", DbPerfStore.eq("backendName", backendName)
+         .withCriteria(new QFilterCriteria("status", QCriteriaOperator.EQUALS, "RUNNING"))
+         .withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.LESS_THAN, cutoff))));
+      if(abandoned.isEmpty())
+      {
+         return;
+      }
+      List<QRecord> updates = abandoned.stream().map(row -> new QRecord().withValue("id", row.getValueInteger("id")).withValue("status", "FAILED")
+         .withValue("errorMessage", "Collection did not finish (the run was interrupted or its pod stopped).")).toList();
+      runtime.asSystem(() ->
+      {
+         DbPerfStore.update("db_perf_snapshot", updates);
+         return (null);
+      });
+      LOG.warn("Closed abandoned database performance snapshots", logPair("backend", backendName), logPair("count", updates.size()));
+   }
+
+
+
   /*******************************************************************************
    ** Statistics read and writes under the ambient system transaction.
    *******************************************************************************/
@@ -23,6 +23,7 @@ package com.kingsrook.qbits.dbperformance.metadata;

 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qbits.dbperformance.DbPerfPermissions;
 import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
 import com.kingsrook.qbits.dbperformance.model.DbPerfActivitySample;
@@ -549,9 +550,10 @@ public final class DbPerfMetaData
      repeatPattern.getField("unitsAffected").setBackendName("units_affected");
      repeatPattern.getField("maxRepeats").setBackendName("max_repeats");
      repeatPattern.getField("totalRepeats").setBackendName("total_repeats");
+      repeatPattern.getField("sampleRate").setBackendName("sample_rate");
      repeatPattern.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
      repeatPattern.withSection(new QFieldSection("identity", new QIcon("table_rows"), Tier.T1, List.of("id", "windowStart", "windowEnd", "podName", "unitKind", "unitName", "processName")));
-      repeatPattern.withSection(new QFieldSection("details", new QIcon("table_rows"), Tier.T2, List.of("tableName", "fingerprint", "sqlText", "unitsAffected", "maxRepeats", "totalRepeats")));
+      repeatPattern.withSection(new QFieldSection("details", new QIcon("table_rows"), Tier.T2, List.of("tableName", "fingerprint", "sqlText", "unitsAffected", "maxRepeats", "totalRepeats", "sampleRate")));
      instance.addTable(repeatPattern);

      QTableMetaData finding = base(config, DbPerfFinding.TABLE_NAME, "Findings", DbPerfFinding.class);
@@ -668,7 +670,7 @@ public final class DbPerfMetaData
      child(instance, "db_perf_statement", "db_perf_finding", "statementId");
      child(instance, "db_perf_table_stat", "db_perf_finding", "tableStatId");
      child(instance, "db_perf_statement", "db_perf_explain", "statementId");
-      QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel("Database Performance").withIcon(new QIcon("speed"))
+      QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel("Database Performance").withIcon(new QIcon("storage"))
         .withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"))
         .withSection(new QAppSection().withName("data").withLabel("Data").withIcon(new QIcon("table_rows"))
            .withTables(List.of("db_perf_finding", "db_perf_statement", "db_perf_table_stat", "db_perf_index_stat", "db_perf_snapshot", "db_perf_settings", "db_perf_rule")));
@@ -680,12 +682,36 @@ public final class DbPerfMetaData
   }


+   /*******************************************************************************
+    ** A distinct icon per table, so nav entries are told apart at a glance.
+    *******************************************************************************/
+   static final Map<String, String> TABLE_ICONS = Map.ofEntries(
+      Map.entry("db_perf_finding", "tips_and_updates"),
+      Map.entry("db_perf_statement", "query_stats"),
+      Map.entry("db_perf_table_stat", "grid_on"),
+      Map.entry("db_perf_index_stat", "bookmarks"),
+      Map.entry("db_perf_snapshot", "photo_camera"),
+      Map.entry("db_perf_settings", "settings_suggest"),
+      Map.entry("db_perf_rule", "playlist_add_check"),
+      Map.entry("db_perf_explain", "device_hub"),
+      Map.entry("db_perf_query_snapshot", "timeline"),
+      Map.entry("db_perf_query_daily", "calendar_month"),
+      Map.entry("db_perf_table_snapshot", "stacked_line_chart"),
+      Map.entry("db_perf_table_daily", "date_range"),
+      Map.entry("db_perf_activity_sample", "monitor_heart"),
+      Map.entry("db_perf_connection_sample", "lan"),
+      Map.entry("db_perf_attribution", "hub"),
+      Map.entry("db_perf_attribution_daily", "event_note"),
+      Map.entry("db_perf_repeat_pattern", "repeat"));
+
+
+
   /*******************************************************************************
    ** Common storage, permissions and write guard.
    *******************************************************************************/
   private static QTableMetaData base(DbPerfQBitConfig config, String name, String label, Class<? extends QRecordEntity> entity) throws QException
   {
-      QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withIcon(new QIcon("speed"))
+      QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withIcon(new QIcon(TABLE_ICONS.getOrDefault(name, "table_rows")))
         .withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
         .withPrimaryKeyField("id").withFieldsFromEntity(entity)
         .withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"));
@@ -21,6 +21,7 @@
 package com.kingsrook.qbits.dbperformance.model;


+import java.math.BigDecimal;
 import java.time.Instant;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -73,6 +74,9 @@ public class DbPerfRepeatPattern extends QRecordEntity
   @QField
   private Long totalRepeats;

+   @QField
+   private BigDecimal sampleRate;
+

   /*******************************************************************************
    ** Default constructor.
@@ -453,4 +457,31 @@ public class DbPerfRepeatPattern extends QRecordEntity
      this.totalRepeats = totalRepeats;
      return (this);
   }
+
+   /*******************************************************************************
+    ** Getter for sampleRate.
+    *******************************************************************************/
+   public BigDecimal getSampleRate()
+   {
+      return (sampleRate);
+   }
+
+
+   /*******************************************************************************
+    ** Setter for sampleRate.
+    *******************************************************************************/
+   public void setSampleRate(BigDecimal sampleRate)
+   {
+      this.sampleRate = sampleRate;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sampleRate.
+    *******************************************************************************/
+   public DbPerfRepeatPattern withSampleRate(BigDecimal sampleRate)
+   {
+      this.sampleRate = sampleRate;
+      return (this);
+   }
 }
@@ -24,7 +24,10 @@ package com.kingsrook.qbits.dbperformance.normalize;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HexFormat;
+import java.util.List;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -200,12 +203,86 @@ public final class SqlNormalizer
    *******************************************************************************/
   private static String collapseLists(String normalized)
   {
-      normalized = normalized.replaceAll("\\bin \\( \\?(?: , \\?)* \\)", "in ( ? )");
-      normalized = normalized.replaceAll("\\bvalues (\\( \\?(?: , \\?)* \\))(?: , \\( \\?(?: , \\?)* \\))+", "values $1");
-      return (normalized);
+      ///////////////////////////////////////////////////////////////////////////
+      // a linear scan over the space-separated tokens: regular expressions    //
+      // with nested repetition overflow the stack on very long IN and VALUES  //
+      // lists (seen with 60 KB statements in production)                      //
+      ///////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////////////////////
+      // segments between quoted identifiers keep their edge spaces          //
+      ////////////////////////////////////////////////////////////////////////
+      String trimmed = normalized.strip();
+      if(trimmed.isEmpty())
+      {
+         return (normalized);
+      }
+      String       leading  = normalized.substring(0, normalized.indexOf(trimmed.charAt(0)));
+      String       trailing = normalized.substring(leading.length() + trimmed.length());
+      String[]     tokens   = trimmed.split(" ");
+      List<String> out    = new ArrayList<>(tokens.length);
+      int          i      = 0;
+      while(i < tokens.length)
+      {
+         String token = tokens[i];
+         if("in".equals(token))
+         {
+            int end = placeholderGroupEnd(tokens, i + 1);
+            if(end > 0)
+            {
+               out.addAll(List.of("in", "(", "?", ")"));
+               i = end;
+               continue;
+            }
+         }
+         if("values".equals(token))
+         {
+            int first = placeholderGroupEnd(tokens, i + 1);
+            if(first > 0)
+            {
+               int end = first;
+               while(end + 1 < tokens.length && ",".equals(tokens[end]))
+               {
+                  int next = placeholderGroupEnd(tokens, end + 1);
+                  if(next < 0)
+                  {
+                     break;
+                  }
+                  end = next;
+               }
+               out.add("values");
+               out.addAll(Arrays.asList(tokens).subList(i + 1, first));
+               i = end;
+               continue;
+            }
+         }
+         out.add(token);
+         i++;
+      }
+      return (leading + String.join(" ", out) + trailing);
   }


+
+   /*******************************************************************************
+    ** If tokens[start] opens "( ? , ? ... )" made only of placeholders, the index
+    ** just after its ")"; otherwise -1.
+    *******************************************************************************/
+   private static int placeholderGroupEnd(String[] tokens, int start)
+   {
+      if(start + 2 >= tokens.length || !"(".equals(tokens[start]) || !"?".equals(tokens[start + 1]))
+      {
+         return (-1);
+      }
+      int i = start + 2;
+      while(i + 1 < tokens.length && ",".equals(tokens[i]) && "?".equals(tokens[i + 1]))
+      {
+         i += 2;
+      }
+      return (i < tokens.length && ")".equals(tokens[i]) ? i + 1 : -1);
+   }
+
+
+
   /*******************************************************************************
    ** Consume a quote with SQL doubled-quote and optional escape handling.
    *******************************************************************************/
@@ -29,6 +29,7 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
@@ -38,8 +39,15 @@ import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
 import com.kingsrook.qbits.dbperformance.provider.ConnectionCount;
 import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -55,11 +63,12 @@ import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockLink;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextStyles;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
-import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.fields;
 import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.map;
 import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.number;

@@ -169,10 +178,11 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
      List<Map<String, Object>> rows = new ArrayList<>();
      for(Map<String, Object> statement : data.statements(100, null))
      {
-         rows.add(map("Query", link((String) statement.get("queryText"), (String) statement.get("historyPath")), "Calls", statement.get("calls24h"),
-            "Mean ms", statement.get("meanMs24h"), "Total ms", statement.get("totalMs24h"), "% DB time", statement.get("shareOfDbTime24h"), "Hit ratio", statement.get("hitRatio")));
+         rows.add(map("Query", stack((String) statement.get("queryText"), (String) statement.get("historyPath"),
+               format(statement.get("calls24h")) + " calls, mean " + format(statement.get("meanMs24h")) + " ms"),
+            "Total ms (24h)", statement.get("totalMs24h"), "Share of DB time %", statement.get("shareOfDbTime24h"), "Cache hit ratio", statement.get("hitRatio")));
      }
-      TableData table = table("Top queries appear after two snapshots establish interval deltas.", rows, "Query", "Calls", "Mean ms", "Total ms", "% DB time", "Hit ratio");
+      TableData table = table("Top queries appear after two snapshots establish interval deltas.", rows, "Query", "Total ms (24h)", "Share of DB time %", "Cache hit ratio");
      table.getColumns().getFirst().setType("composite");
      return (table);
   }
@@ -210,7 +220,20 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
      List<Map<String, Object>> rows = sessions.stream().filter(row -> !((List<?>) row.get("blockedBy")).isEmpty() || ((Number) row.get("blockingCount")).intValue() > 0
         || "active".equals(row.get("state")) && numeric(row.get("querySeconds")) >= longRunning
         || Objects.toString(row.get("state"), "").startsWith("idle in transaction") && numeric(row.get("xactSeconds")) >= idle).toList();
-      return (table("No long-running or blocked sessions right now", rows, "pid", "user", "application", "state", "querySeconds", "xactSeconds", "blockedBy", "blockingCount", "normalizedText"));
+      List<Map<String, Object>> cells = new ArrayList<>();
+      for(Map<String, Object> row : rows)
+      {
+         boolean blocked = !((List<?>) row.get("blockedBy")).isEmpty();
+         int     blocking = ((Number) row.get("blockingCount")).intValue();
+         cells.add(map("Session", stack("pid " + row.get("pid") + ", " + Objects.toString(row.get("user"), "?") + ", " + Objects.toString(row.get("application"), ""), null,
+               Objects.toString(row.get("normalizedText"), "")),
+            "State", Objects.toString(row.get("state"), "") + (row.get("waitEvent") == null ? "" : " (" + row.get("waitEvent") + ")"),
+            "Query / transaction s", format(row.get("querySeconds")) + " / " + format(row.get("xactSeconds")),
+            "Blocking", blocked ? "blocked by " + row.get("blockedBy") : (blocking > 0 ? "blocks " + blocking : "")));
+      }
+      TableData table = table("No long-running or blocked sessions right now", cells, "Session", "State", "Query / transaction s", "Blocking");
+      table.getColumns().getFirst().setType("composite");
+      return (table);
   }


@@ -265,33 +288,64 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer



+   /*******************************************************************************
+    ** Most (unit, table) groups summed for the attribution panel. Production
+    ** shows about a thousand; the cap only guards pathological unit names.
+    *******************************************************************************/
+   public static final int MAX_ATTRIBUTION_GROUPS = 5000;
+
+
+
   /*******************************************************************************
    ** Aggregate units over 24 hours and show repeated-statement patterns below them.
+    ** The sums run in the database: a day of attribution detail is hundreds of
+    ** thousands of rows (about 540 MB), and loading it into the heap on a widget
+    ** request ran the web pod out of memory (2026-10-16).
    *******************************************************************************/
   public MultiTableData attribution() throws QException
   {
-      List<QRecord> records = DbPerfStore.query("db_perf_attribution", since("windowEnd"));
+      GroupBy kindGroup = new GroupBy(QFieldType.STRING, "unitKind");
+      GroupBy nameGroup = new GroupBy(QFieldType.STRING, "unitName");
+      GroupBy tableGroup = new GroupBy(QFieldType.STRING, "tableName");
+      Aggregate statementsSum = new Aggregate("statements", AggregateOperator.SUM);
+      Aggregate totalMsSum = new Aggregate("totalMs", AggregateOperator.SUM);
+      List<AggregateResult> results = new AggregateAction().execute(new AggregateInput("db_perf_attribution")
+         .withFilter(since("windowEnd").withOrderBy(new QFilterOrderByAggregate(totalMsSum, false)))
+         .withGroupBy(kindGroup).withGroupBy(nameGroup).withGroupBy(tableGroup)
+         .withAggregate(statementsSum).withAggregate(totalMsSum)
+         .withLimit(MAX_ATTRIBUTION_GROUPS)
+         .withTransaction(QContext.getQBackendTransaction())).getResults();
      Map<String, Map<String, Object>> grouped = new LinkedHashMap<>();
      Map<String, Map<String, Double>> tableTimes = new LinkedHashMap<>();
-      double total = records.stream().mapToDouble(row -> number(row, "totalMs")).sum();
-      for(QRecord row : records)
+      double total = results.stream().mapToDouble(row -> numeric(row.getAggregateValue(totalMsSum))).sum();
+      for(AggregateResult row : results)
      {
-         String key = row.getValueString("unitKind") + "\n" + row.getValueString("unitName");
-         Map<String, Object> group = grouped.computeIfAbsent(key, ignored -> map("Kind", row.getValueString("unitKind"), "Unit", row.getValueString("unitName"), "Statements", 0.0, "Total ms", 0.0));
-         group.put("Statements", numeric(group.get("Statements")) + number(row, "statements"));
-         group.put("Total ms", numeric(group.get("Total ms")) + number(row, "totalMs"));
-         tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getValueString("tableName"), "Unknown"), number(row, "totalMs"), Double::sum);
+         String kind = (String) row.getGroupByValue(kindGroup);
+         String name = (String) row.getGroupByValue(nameGroup);
+         String key = kind + "\n" + name;
+         Map<String, Object> group = grouped.computeIfAbsent(key, ignored -> map("Kind", kind, "Name", name, "Statements", 0.0, "Total ms", 0.0));
+         group.put("Statements", numeric(group.get("Statements")) + numeric(row.getAggregateValue(statementsSum)));
+         group.put("Total ms", numeric(group.get("Total ms")) + numeric(row.getAggregateValue(totalMsSum)));
+         tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getGroupByValue(tableGroup), "Unknown"), numeric(row.getAggregateValue(totalMsSum)), Double::sum);
      }
      grouped.forEach((key, group) ->
      {
         group.put("Share %", total == 0 ? 0 : 100 * numeric(group.get("Total ms")) / total);
         group.put("Top table", tableTimes.get(key).entrySet().stream().max(Map.Entry.comparingByValue()).orElseThrow().getKey());
      });
-      List<Map<String, Object>> units = grouped.values().stream().sorted(Comparator.comparingDouble((Map<String, Object> row) -> numeric(row.get("Total ms"))).reversed()).toList();
+      List<Map<String, Object>> units = grouped.values().stream().sorted(Comparator.comparingDouble((Map<String, Object> row) -> numeric(row.get("Total ms"))).reversed())
+         .map(row -> map("Unit of work", stack(Objects.toString(row.get("Name"), ""), null, kindLabel(row.get("Kind")) + ", top table " + row.get("Top table")),
+            "Statements", row.get("Statements"), "Total ms (24h)", row.get("Total ms"), "Share %", row.get("Share %"))).toList();
      List<Map<String, Object>> repeats = DbPerfStore.query("db_perf_repeat_pattern", since("windowEnd").withOrderBy(new QFilterOrderBy("maxRepeats", false)).withLimit(100)).stream()
-         .map(row -> fields(row, "unitKind", "unitName", "fingerprint", "unitsAffected", "maxRepeats", "totalRepeats")).toList();
-      return (new MultiTableData(List.of(table("Attribution starts after the next flush", units, "Kind", "Unit", "Statements", "Total ms", "Share %", "Top table"),
-         table("Repeated statements appear after the next attribution flush when the repetition threshold is reached.", repeats, "unitKind", "unitName", "fingerprint", "unitsAffected", "maxRepeats", "totalRepeats"))));
+         .map(row -> map("Unit of work", stack(Objects.toString(row.getValueString("unitName"), ""), null,
+               kindLabel(row.getValueString("unitKind")) + ": " + truncate(row.getValueString("sqlText"), 160)),
+            "Most repeats in one unit", row.getValue("maxRepeats"), "Units affected", row.getValue("unitsAffected"), "Total repeats", row.getValue("totalRepeats"))).toList();
+      TableData unitTable = table("Attribution starts after the next flush", units, "Unit of work", "Statements", "Total ms (24h)", "Share %");
+      unitTable.getColumns().getFirst().setType("composite");
+      TableData repeatTable = table("Repeated statements appear after the next attribution flush when the repetition threshold is reached.", repeats,
+         "Unit of work", "Most repeats in one unit", "Units affected", "Total repeats");
+      repeatTable.getColumns().getFirst().setType("composite");
+      return (new MultiTableData(List.of(unitTable, repeatTable)));
   }


@@ -303,10 +357,14 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
   {
      List<QRecord> scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100));
      List<QRecord> growing = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("growthBytes7d", false)).withLimit(100));
-      return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream().map(row -> fields(row, "schemaName", "tableName", "seqScan", "idxScan", "seqScanShare")).toList(),
-         "schemaName", "tableName", "seqScan", "idxScan", "seqScanShare"),
-         table("Growth appears after table history has accumulated.", growing.stream().map(row -> fields(row, "schemaName", "tableName", "totalBytes", "growthBytes7d")).toList(),
-            "schemaName", "tableName", "totalBytes", "growthBytes7d"))));
+      return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream()
+            .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Sequential scans", row.getValue("seqScan"),
+               "Index scans", row.getValue("idxScan"), "Sequential share %", row.getValue("seqScanShare"))).toList(),
+            "Table", "Sequential scans", "Index scans", "Sequential share %"),
+         table("Growth appears after table history has accumulated.", growing.stream()
+            .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")),
+               "Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(),
+            "Table", "Size MB", "Growth MB (7 days)"))));
   }


@@ -333,6 +391,79 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer



+   /*******************************************************************************
+    ** A stacked cell: the main text (a record link when a path is given) over a
+    ** smaller secondary line, so tables keep at most four columns.
+    *******************************************************************************/
+   private static CompositeWidgetData stack(String main, String path, String secondary)
+   {
+      TextBlockData first = new TextBlockData().withValues(new TextValues(truncate(main, 300)));
+      if(path != null)
+      {
+         first.withLink(new BlockLink(path));
+      }
+      CompositeWidgetData cell = new CompositeWidgetData().withLayout(CompositeWidgetData.Layout.FLEX_COLUMN).withBlock(first);
+      if(secondary != null && !secondary.isBlank())
+      {
+         cell.withBlock(new TextBlockData().withValues(new TextValues(truncate(secondary, 300))).withStyles(new TextStyles().withColor("MUTED")));
+      }
+      return (cell);
+   }
+
+
+
+   /*******************************************************************************
+    ** Plain label for a unit-of-work kind.
+    *******************************************************************************/
+   private static String kindLabel(Object kind)
+   {
+      return (switch(Objects.toString(kind, ""))
+      {
+         case "HTTP_REQUEST" -> "Request";
+         case "SCHEDULED_JOB" -> "Scheduled job";
+         case "PROCESS" -> "Process";
+         default -> "Other";
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Text capped with an ellipsis.
+    *******************************************************************************/
+   private static String truncate(String value, int max)
+   {
+      String text = value == null ? "" : value;
+      return (text.length() <= max ? text : text.substring(0, max - 1) + "…");
+   }
+
+
+
+   /*******************************************************************************
+    ** A number for a secondary line: whole numbers without decimals.
+    *******************************************************************************/
+   private static String format(Object value)
+   {
+      if(value == null)
+      {
+         return ("0");
+      }
+      double number = ((Number) value).doubleValue();
+      return (number == Math.rint(number) ? String.valueOf((long) number) : String.format(Locale.ROOT, "%.1f", number));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bytes as whole megabytes.
+    *******************************************************************************/
+   private static Long mb(Object bytes)
+   {
+      return (bytes == null ? null : Math.round(((Number) bytes).doubleValue() / 1048576));
+   }
+
+
+
   /*******************************************************************************
    ** Trailing-day filter for time-window records.
    *******************************************************************************/
@@ -605,3 +605,7 @@ INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,th
 SELECT 'TEMP_SPILL', 'Temporary disk spill', 'Statements that write temporary files because sorts or hashes do not fit in work_mem.', TRUE, 'WARNING', 100, 'Temporary data written per day (MB)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'TEMP_SPILL');
 INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
 SELECT 'DEADLOCKS', 'Deadlocks', 'Transactions that deadlocked and were cancelled by PostgreSQL. Usually two code paths that lock rows in a different order.', TRUE, 'WARNING', 1, 'Deadlocks per day', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'DEADLOCKS');
+
+-- changeset qbit-db-performance:34
+-- comment: Repeated Statements: the sampling rate of each window, so repeat counts can be read at scale.
+ALTER TABLE db_perf_repeat_pattern ADD COLUMN IF NOT EXISTS sample_rate DECIMAL(5,4);
@@ -388,6 +388,8 @@ entries:
  summary: "Largest repetition count for one fingerprint in a single sampled unit of work."
 - key: table:db_perf_repeat_pattern;field:totalRepeats
  summary: "Total repeats reported for this repeat pattern record."
+- key: table:db_perf_repeat_pattern;field:sampleRate
+  summary: "Share of units of work that were measured in this window (1 means all)."
 - key: table:db_perf_rule;slot:help
  summary: "Editable thresholds and severity for each suggestion rule."
  content: |
@@ -0,0 +1,239 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.dbperformance;
+
+
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
+import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
+import com.kingsrook.qbits.dbperformance.collect.RollupService;
+import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Attribution detail grows by hundreds of thousands of rows a day. On
+ ** 2026-10-16 the home widget loaded a whole day of it into the heap and ran
+ ** the kof22 web pod out of memory, and the hourly analysis filled the worker
+ ** heap the same way. These tests count every attribution detail record QQQ
+ ** materializes, so the widget, the analysis and the daily rollup cannot go
+ ** back to reading the table in one unbounded query.
+ *******************************************************************************/
+class AttributionBoundedReadTest
+{
+   private static final AtomicInteger LOADED   = new AtomicInteger();
+   private static final AtomicInteger MAX_PAGE = new AtomicInteger();
+
+   private DbPerfRuntime runtime;
+
+
+
+   /*******************************************************************************
+    ** Counts the detail records each QueryAction hands back.
+    *******************************************************************************/
+   public static class CountingCustomizer implements TableCustomizerInterface
+   {
+      /***************************************************************************
+       ** Record the page size, change nothing.
+       ***************************************************************************/
+      @Override
+      public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records)
+      {
+         LOADED.addAndGet(records.size());
+         MAX_PAGE.accumulateAndGet(records.size(), Math::max);
+         return (records);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Fresh H2 instance with the counter on the detail table.
+    *******************************************************************************/
+   @BeforeEach
+   void setup() throws Exception
+   {
+      runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
+      runtime.getInstance().getTable("db_perf_attribution").withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CountingCustomizer.class));
+      LOADED.set(0);
+      MAX_PAGE.set(0);
+   }
+
+
+
+   /*******************************************************************************
+    ** The widget sums in the database: no detail rows reach the heap, and the
+    ** totals still match the detail.
+    *******************************************************************************/
+   @Test
+   void widgetSumsInTheDatabase() throws Exception
+   {
+      Instant now = Instant.now();
+      insert(3000, i -> row(now.minusSeconds(60), now, i));
+      LOADED.set(0);
+
+      MultiTableData attribution = new DbPerfWidgetRenderer().attribution();
+
+      assertThat(LOADED.get()).as("attribution detail rows loaded by the widget").isZero();
+      List<Map<String, Object>> units = attribution.getTableDataList().getFirst().getRows();
+      assertThat(units).hasSize(3);
+      // unit-0 holds i = 0, 3, 6 ... (1000 rows, 2 statements each), mostly against "patient"
+      assertThat(((Number) units.getFirst().get("Statements")).doubleValue() + ((Number) units.get(1).get("Statements")).doubleValue()
+         + ((Number) units.get(2).get("Statements")).doubleValue()).isEqualTo(6000.0);
+      assertThat(units).allSatisfy(unit -> assertThat(((Number) unit.get("Statements")).doubleValue()).isEqualTo(2000.0));
+      assertThat(JsonUtils.toJson(units.getFirst())).contains("top table patient");
+      assertThat(units.stream().mapToDouble(unit -> ((Number) unit.get("Share %")).doubleValue()).sum()).isCloseTo(100.0, org.assertj.core.data.Offset.offset(0.001));
+   }
+
+
+
+   /*******************************************************************************
+    ** Detail older than 24 hours is left out of the sums.
+    *******************************************************************************/
+   @Test
+   void widgetKeepsTheDayWindow() throws Exception
+   {
+      Instant now = Instant.now();
+      insert(3, i -> row(now.minusSeconds(60), now, i));
+      insert(3, i -> row(now.minusSeconds(3 * 86400), now.minusSeconds(2 * 86400), i));
+      List<Map<String, Object>> units = new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getRows();
+      assertThat(units).hasSize(3).allSatisfy(unit -> assertThat(((Number) unit.get("Statements")).doubleValue()).isEqualTo(2.0));
+   }
+
+
+
+   /*******************************************************************************
+    ** The hourly analysis reads attribution as database sums per unit, table
+    ** and operation: no detail rows, same totals.
+    *******************************************************************************/
+   @Test
+   void analysisSumsInTheDatabase() throws Exception
+   {
+      Instant now = Instant.now();
+      insert(3000, i -> row(now.minusSeconds(60), now, i));
+      LOADED.set(0);
+      RuleContext context = new RuleContext(runtime, runtime.getConfig().getBackendName(), new QRecord(), new QRecord(), now.plusSeconds(1));
+
+      List<QRecord> rows = context.window("attribution", 1);
+
+      assertThat(LOADED.get()).as("attribution detail rows loaded by the analysis").isZero();
+      assertThat(rows).hasSize(6);
+      assertThat(rows.stream().mapToDouble(row -> RuleContext.number(row, "statements")).sum()).isEqualTo(6000.0);
+      assertThat(rows.stream().mapToDouble(row -> RuleContext.number(row, "totalMs")).sum()).isEqualTo(1500 * 10.0 + 1500 * 1.0);
+      assertThat(rows).allSatisfy(row -> assertThat(row.getValueString("operation")).isEqualTo("SELECT"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Keyset pages never exceed the page size, and every row is seen once.
+    *******************************************************************************/
+   @Test
+   void forEachPageIsBounded() throws Exception
+   {
+      Instant now = Instant.now();
+      insert(1050, i -> row(now.minusSeconds(60), now, i));
+      LOADED.set(0);
+      List<Integer> sizes = new ArrayList<>();
+      List<Object> ids = new ArrayList<>();
+      DbPerfStore.forEachPage("db_perf_attribution", new QQueryFilter(), 100, page ->
+      {
+         sizes.add(page.size());
+         page.forEach(row -> ids.add(row.getValue("id")));
+      });
+      assertThat(sizes).hasSize(11).allSatisfy(size -> assertThat(size).isLessThanOrEqualTo(100));
+      assertThat(ids).hasSize(1050).doesNotHaveDuplicates();
+      assertThat(MAX_PAGE.get()).isLessThanOrEqualTo(100);
+   }
+
+
+
+   /*******************************************************************************
+    ** The daily rollup reads the day in BATCH_SIZE pages and its sums still
+    ** cover every detail row across page boundaries.
+    *******************************************************************************/
+   @Test
+   void rollupReadsTheDayInPages() throws Exception
+   {
+      ZoneId zone = ZoneId.of(runtime.getConfig().getTimeZone());
+      LocalDate day = LocalDate.now(zone).minusDays(1);
+      Instant start = day.atStartOfDay(zone).toInstant().plusSeconds(3600);
+      int rows = DbPerfStore.BATCH_SIZE * 2 + 500;
+      insert(rows, i -> row(start, start.plusSeconds(60), i));
+      MAX_PAGE.set(0);
+      LOADED.set(0);
+
+      new RollupService(runtime).rollupDay(day);
+
+      assertThat(MAX_PAGE.get()).as("largest page of attribution detail held at once").isLessThanOrEqualTo(DbPerfStore.BATCH_SIZE);
+      assertThat(LOADED.get()).isEqualTo(rows);
+      List<QRecord> daily = DbPerfStore.query("db_perf_attribution_daily", new QQueryFilter());
+      assertThat(daily).hasSize(6);
+      assertThat(daily.stream().mapToLong(row -> row.getValueLong("statements")).sum()).isEqualTo(2L * rows);
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert generated rows in batches.
+    *******************************************************************************/
+   private static void insert(int count, java.util.function.IntFunction<QRecord> factory) throws Exception
+   {
+      List<QRecord> rows = new ArrayList<>();
+      for(int i = 0; i < count; i++)
+      {
+         rows.add(factory.apply(i));
+      }
+      DbPerfStore.insert("db_perf_attribution", rows);
+   }
+
+
+
+   /*******************************************************************************
+    ** Three units; each unit spends most of its time on "patient".
+    *******************************************************************************/
+   private static QRecord row(Instant windowStart, Instant windowEnd, int i)
+   {
+      boolean patient = i % 6 < 3;
+      return (new QRecord().withValue("windowStart", windowStart).withValue("windowEnd", windowEnd).withValue("podName", "pod-" + (i % 2))
+         .withValue("unitKind", "PROCESS").withValue("unitName", "unit-" + (i % 3)).withValue("tableName", patient ? "patient" : "visit")
+         .withValue("operation", "SELECT").withValue("fingerprint", "f".repeat(64)).withValue("sqlText", "select 1")
+         .withValue("units", 1L).withValue("statements", 2L).withValue("rowsCount", 1L).withValue("errors", 0L)
+         .withValue("totalMs", patient ? BigDecimal.TEN : BigDecimal.ONE).withValue("maxMs", BigDecimal.ONE));
+   }
+}
@@ -47,6 +47,7 @@ public class DbPerfFixtureProvider extends UnsupportedStatsProvider
   public Instant reset = START.minusSeconds(3600);
   public boolean limited;
   public boolean fail;
+   public Error   error;


   /*******************************************************************************
@@ -85,6 +86,10 @@ public class DbPerfFixtureProvider extends UnsupportedStatsProvider
   @Override
   public List<StatementStats> readStatements(Connection connection)
   {
+      if(error != null)
+      {
+         throw (error);
+      }
      return (List.of(new StatementStats(1, "app", true, "select * from public.accounts where id=$1", calls, BigDecimal.valueOf(calls * 10), BigDecimal.TEN, BigDecimal.valueOf(15), calls, calls * 9, calls, 0, reset, reset),
         new StatementStats(2, "app", true, "select * from public.accounts where name='private'", calls, BigDecimal.valueOf(calls * 5), BigDecimal.valueOf(5), BigDecimal.TEN, calls, calls, calls, 2, reset, reset)));
   }
@@ -129,7 +129,7 @@ class DbPerfPresentationTest
      assertThat((Map<String, Integer>) overview.get("findingsBySeverity")).containsEntry("HIGH", 1);
      TableData queries = new DbPerfWidgetRenderer().topQueries(data);
      assertThat(queries.getRows()).hasSize(1);
-      assertThat((BigDecimal) queries.getRows().getFirst().get("% DB time")).isEqualByComparingTo("25");
+      assertThat((BigDecimal) queries.getRows().getFirst().get("Share of DB time %")).isEqualByComparingTo("25");
      assertThat(JsonUtils.toJson(queries)).contains("/app/dbPerformance/db_perf_statement/", "composite").doesNotContain("private-value");
      ChartData chart = new DbPerfWidgetRenderer().databaseTime(data);
      assertThat(chart.getChartData().getDatasets()).hasSize(2);
@@ -137,7 +137,7 @@ class DbPerfPresentationTest
      assertThat(new DbPerfWidgetRenderer().findings(data).getRows()).hasSize(1);
      assertThat(new DbPerfWidgetRenderer().tables(data).getTableDataList().getFirst().getRows()).hasSize(1);
      MultiTableData attribution = new DbPerfWidgetRenderer().attribution();
-      assertThat(attribution.getTableDataList().getFirst().getRows().getFirst()).containsEntry("Top table", "patient");
+      assertThat(JsonUtils.toJson(attribution.getTableDataList().getFirst().getRows().getFirst())).contains("top table patient");
      assertThat(attribution.getTableDataList().get(1).getRows()).hasSize(1);
      assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().connectionStatistics(data))).contains("Deadlocks", "Cache hit");
      JsonInput lookup = new JsonInput();
@@ -194,7 +194,7 @@ class DbPerfPresentationTest
      QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW));
      TableData table = new DbPerfWidgetRenderer().liveActivity(fixtureData);
      assertThat(table.getRows()).hasSize(2);
-      assertThat(JsonUtils.toJson(table)).doesNotContain("private-value").contains("normalizedText", "blockingCount");
+      assertThat(JsonUtils.toJson(table)).doesNotContain("private-value").contains("select * from patient where name = ?", "blocks 1", "blocked by");
      QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.VIEW_LIVE_TEXT));
      assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().liveActivity(fixtureData))).doesNotContain("private-value");
   }
@@ -112,6 +112,9 @@ class RollupRetentionTest
      }
      rows.add(new QRecord().withValue("windowStart", now.minusSeconds(2 * 86400)).withValue("unitName", "boundary"));
      DbPerfStore.insert("db_perf_attribution", rows);
+      DbPerfStore.insert("db_perf_repeat_pattern", List.of(
+         new QRecord().withValue("windowStart", now.minusSeconds(3 * 86400)),
+         new QRecord().withValue("windowStart", now.minusSeconds(2 * 86400))));
      DbPerfStore.insert("db_perf_attribution_daily", List.of(new QRecord().withValue("day", LocalDate.of(2026, 9, 20)).withValue("rollupKey", "old")));
      int rule = DbPerfStore.query("db_perf_rule", new QQueryFilter()).getFirst().getValueInteger("id");
      List<QRecord> findings = new ArrayList<>();
@@ -121,8 +124,9 @@ class RollupRetentionTest
      }
      DbPerfStore.insert("db_perf_finding", findings);
      RetentionService retention = new RetentionService(runtime, Clock.fixed(now, ZoneOffset.UTC));
-      assertThat(retention.purge()).isEqualTo(1008);
+      assertThat(retention.purge()).isEqualTo(1009);
      assertThat(retention.purge()).isZero();
+      assertThat(DbPerfStore.query("db_perf_repeat_pattern", new QQueryFilter())).hasSize(1);
      assertThat(DbPerfStore.query("db_perf_attribution", new QQueryFilter())).singleElement().satisfies(row -> assertThat(row.getValueString("unitName")).isEqualTo("boundary"));
      assertThat(DbPerfStore.query("db_perf_finding", new QQueryFilter())).extracting(row -> row.getValueString("status")).containsExactlyInAnyOrder("OPEN", "ACKNOWLEDGED");
   }
@@ -123,6 +123,37 @@ class SnapshotCollectorTest
   }


+   /*******************************************************************************
+    ** An Error (not just an Exception) still closes the run as FAILED, and runs
+    ** left RUNNING by an earlier crash are closed once they are stale.
+    *******************************************************************************/
+   @Test
+   void errorsAndAbandonedRuns() throws Exception
+   {
+      QInstance             instance = DbPerfTestSupport.instance();
+      DbPerfRuntime         runtime  = DbPerfRuntime.of(instance);
+      DbPerfFixtureProvider provider = new DbPerfFixtureProvider();
+      runtime.asSystem(() -> DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName())
+         .withValue("capturedAt", DbPerfFixtureProvider.START.minusSeconds(3600)).withValue("status", "RUNNING"))));
+
+      provider.error = new StackOverflowError();
+      QRecord failed = collect(runtime, provider, 0, true);
+      assertThat(failed.getValueString("status")).isEqualTo("FAILED");
+      assertThat(failed.getValueString("errorMessage")).contains("StackOverflowError");
+      runtime.asSystem(() ->
+      {
+         List<QRecord> rows = DbPerfStore.query("db_perf_snapshot", new QQueryFilter());
+         assertThat(rows).hasSize(2).allSatisfy(row -> assertThat(row.getValueString("status")).isEqualTo("FAILED"));
+         assertThat(rows).anySatisfy(row -> assertThat(row.getValueString("errorMessage")).contains("did not finish"));
+         return (null);
+      });
+
+      provider.error = null;
+      assertThat(collect(runtime, provider, 300, true).getValueString("status")).isEqualTo("OK");
+   }
+
+
+
   /*******************************************************************************
    ** H2 remains usable and explicitly unsupported by the production selector.
    *******************************************************************************/
@@ -86,6 +86,14 @@ class SqlNormalizerTest
      assertThat(SqlNormalizer.mainTable("select 1")).isNull();
      assertThat(SqlNormalizer.normalize("select \"in ( ? , ? )\" from t")).isEqualTo("select \"in ( ? , ? )\" from t");
      assertThat(SqlNormalizer.normalize(null)).isEmpty();
+      ///////////////////////////////////////////////////////////////////////
+      // huge lists (production had 60 KB statements) must not overflow    //
+      ///////////////////////////////////////////////////////////////////////
+      String hugeIn = "select * from t where id in (" + "$1, ".repeat(20000) + "$1)";
+      assertThat(SqlNormalizer.normalize(hugeIn)).isEqualTo("select * from t where id in ( ? )");
+      String hugeValues = "insert into t (a, b) values " + "($1, $2), ".repeat(20000) + "($1, $2)";
+      assertThat(SqlNormalizer.normalize(hugeValues)).isEqualTo("insert into t ( a , b ) values ( ? , ? )");
+      assertThat(SqlNormalizer.normalize("select * from t where id in (1, x)")).isEqualTo("select * from t where id in ( ? , x )");
      assertThat(SqlNormalizer.normalize(" ")).isEmpty();
      assertThat(SqlNormalizer.normalize("select " + "field, ".repeat(5000))).hasSize(10000);
   }
@@ -0,0 +1,135 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.dbperformance.alerts;
+
+
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
+import com.kingsrook.qbits.dbperformance.DbPerfTestSupport;
+import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
+import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Spike detection reads seven days of query snapshots (~230,000 rows in kof22)
+ ** on every evaluation. It must page them and keep only per-statement numbers.
+ *******************************************************************************/
+class SpikeHistoryBoundedTest
+{
+   private static final AtomicInteger MAX_PAGE = new AtomicInteger();
+   private static final AtomicInteger LOADED   = new AtomicInteger();
+
+
+
+   /*******************************************************************************
+    ** Records the largest page QQQ hands back.
+    *******************************************************************************/
+   public static class CountingCustomizer implements TableCustomizerInterface
+   {
+      /***************************************************************************
+       ** Count, change nothing.
+       ***************************************************************************/
+      @Override
+      public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records)
+      {
+         LOADED.addAndGet(records.size());
+         MAX_PAGE.accumulateAndGet(records.size(), Math::max);
+         return (records);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** 2,500 samples across page boundaries: pages stay at BATCH_SIZE, invalid
+    ** snapshots and the window are honored, the current interval is found.
+    *******************************************************************************/
+   @Test
+   void historyIsPagedAndExact() throws Exception
+   {
+      DbPerfRuntime runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
+      String backend = runtime.getConfig().getBackendName();
+      runtime.getInstance().getTable("db_perf_query_snapshot").withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CountingCustomizer.class));
+      Instant now = Instant.parse("2026-10-16T20:00:00Z");
+
+      QRecord ok = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now.minusSeconds(3600)).withValue("status", "OK"))).getFirst();
+      QRecord failed = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now.minusSeconds(1800)).withValue("status", "FAILED"))).getFirst();
+      QRecord current = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now).withValue("status", "OK"))).getFirst();
+
+      int one = statement(backend, 1L);
+      int two = statement(backend, 2L);
+      List<QRecord> rows = new ArrayList<>();
+      for(int i = 0; i < 2500; i++)
+      {
+         rows.add(sample(ok, i % 2 == 0 ? one : two, now.minusSeconds(3600 + i), i));
+      }
+      rows.add(sample(failed, one, now.minusSeconds(1800), 999999));
+      rows.add(sample(ok, one, now.minusSeconds(8 * 86400L), 999999));
+      rows.add(sample(current, one, now, 42));
+      DbPerfStore.insert("db_perf_query_snapshot", rows);
+      MAX_PAGE.set(0);
+      LOADED.set(0);
+
+      RuleContext ctx = new RuleContext(runtime, backend, new QRecord(), DbPerfStore.first("db_perf_settings", new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()), now);
+      Map<Integer, SnapshotEvaluator.StatementHistory> history = SnapshotEvaluator.history(ctx, current.getValueInteger("id"));
+
+      assertThat(MAX_PAGE.get()).as("largest page of query snapshots held at once").isLessThanOrEqualTo(DbPerfStore.BATCH_SIZE);
+      assertThat(history.get(one).sortedPrevious()).hasSize(1250).doesNotContain(999999.0);
+      assertThat(history.get(two).sortedPrevious()).hasSize(1250);
+      assertThat(history.get(one).sortedPrevious()[0]).isEqualTo(0.0);
+      assertThat(history.get(one)).extracting("current").isEqualTo(42.0);
+      assertThat(history.get(two)).extracting("current").isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** One statement identity; returns its id.
+    *******************************************************************************/
+   private static int statement(String backend, long queryId) throws Exception
+   {
+      return (DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", backend).withValue("queryId", queryId)
+         .withValue("queryText", "select " + queryId).withValue("fingerprint", String.valueOf(queryId).repeat(64).substring(0, 64)))).getFirst().getValueInteger("id"));
+   }
+
+
+
+   /*******************************************************************************
+    ** One query snapshot row.
+    *******************************************************************************/
+   private static QRecord sample(QRecord snapshot, int statementId, Instant at, int totalMs)
+   {
+      return (new QRecord().withValue("snapshotId", snapshot.getValueInteger("id")).withValue("statementId", statementId)
+         .withValue("capturedAt", at).withValue("totalMs", BigDecimal.valueOf(totalMs)));
+   }
+}
@@ -0,0 +1,664 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.dbperformance.attribution;
+
+
+import java.math.BigDecimal;
+import java.sql.DriverManager;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
+import com.kingsrook.qbits.dbperformance.DbPerfTestSupport;
+import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
+import com.kingsrook.qbits.dbperformance.collect.RollupService;
+import com.kingsrook.qbits.dbperformance.metadata.DbPerfWriteGuard;
+import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementExecution;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListener;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListeners;
+import com.kingsrook.qqq.backend.core.instrumentation.StatementOperation;
+import com.kingsrook.qqq.backend.core.instrumentation.UnitOfWork;
+import com.kingsrook.qqq.backend.core.instrumentation.UnitOfWorkKind;
+import com.kingsrook.qqq.backend.core.instrumentation.UnitsOfWork;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Real QQQ RDBMS hooks, persistence, bounded counters, and lifecycle contracts.
+ *******************************************************************************/
+class AttributionTest
+{
+   private DbPerfRuntime runtime;
+   private AttributionFlusher flusher;
+   private AttributionListener listener;
+   private QRecord settings;
+
+
+   /*******************************************************************************
+    ** A real business table, separate from the self-suppressed db_perf tables.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      QInstance instance = DbPerfTestSupport.instance();
+      runtime = DbPerfRuntime.of(instance);
+      RDBMSBackendMetaData backend = (RDBMSBackendMetaData) instance.getBackend(runtime.getConfig().getBackendName());
+      instance.addTable(new QTableMetaData().withName("business").withBackendName(backend.getName()).withPrimaryKeyField("id")
+         .withBackendDetails(new RDBMSTableBackendDetails().withTableName("business"))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("name", QFieldType.STRING)));
+      try(var connection = DriverManager.getConnection(backend.getJdbcUrl(), backend.getUsername(), backend.getPassword()); var statement = connection.createStatement())
+      {
+         statement.execute("CREATE TABLE business (id INT PRIMARY KEY, name VARCHAR(100))");
+         statement.execute("INSERT INTO business VALUES (1, 'example')");
+      }
+      flusher = new AttributionFlusher(runtime);
+      flusher.refreshSettings();
+      listener = flusher.getListener();
+      settings = DbPerfStore.query("db_perf_settings", new QQueryFilter()).getFirst();
+      StatementExecutionListeners.register(listener);
+   }
+
+
+   /*******************************************************************************
+    ** No listener, scope, suppression or runtime may escape a test.
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      StatementExecutionListeners.unregister(listener);
+      flusher.close();
+      runtime.stopAttribution();
+      UnitsOfWork.reset();
+      AttributionListener.suppress(false);
+      QContext.clear();
+   }
+
+
+   /*******************************************************************************
+    ** The core hook attributes real queries and reads final HTTP route names.
+    *******************************************************************************/
+   @Test
+   void realQueriesAndFinalNames() throws Exception
+   {
+      try(var scope = UnitsOfWork.begin(UnitOfWorkKind.SCHEDULED_JOB, "job"))
+      {
+         query(2);
+      }
+      try(var scope = UnitsOfWork.begin(UnitOfWorkKind.HTTP_REQUEST, "unresolved"))
+      {
+         query(1);
+         scope.getUnitOfWork().setName("GET /things/{id}");
+      }
+      query(1);
+      flusher.flushNow();
+      List<QRecord> rows = rows("db_perf_attribution");
+      assertThat(rows).hasSize(3);
+      assertThat(rows).extracting(row -> row.getValueString("unitName")).containsExactlyInAnyOrder("job", "GET /things/{id}", "(none)");
+      QRecord job = named(rows, "job");
+      assertThat(job.getValueString("unitKind")).isEqualTo("SCHEDULED_JOB");
+      assertThat(job.getValueLong("units")).isEqualTo(1);
+      assertThat(job.getValueLong("statements")).isEqualTo(2);
+      assertThat(job.getValueLong("rowsCount")).isEqualTo(2);
+      assertThat(job.getValueString("operation")).isEqualTo("QUERY");
+      assertThat(job.getValueString("tableName")).isEqualTo("business");
+      assertThat(job.getValueString("fingerprint")).isEqualTo(SqlNormalizer.fingerprint(job.getValueString("sqlText")));
+      assertThat(named(rows, "(none)").getValueString("unitKind")).isEqualTo("OTHER");
+   }
+
+
+   /*******************************************************************************
+    ** The threshold applies to each unit, and each window uses one batch/table.
+    *******************************************************************************/
+   @Test
+   void repeatPatternsBatchesAndSelfSuppression() throws Exception
+   {
+      AtomicInteger attributionInserts = new AtomicInteger();
+      AtomicInteger repeatInserts = new AtomicInteger();
+      StatementExecutionListener observer = execution ->
+      {
+         if(execution.operation() == StatementOperation.INSERT)
+         {
+            if("db_perf_attribution".equals(execution.tableName()))
+            {
+               attributionInserts.incrementAndGet();
+            }
+            if("db_perf_repeat_pattern".equals(execution.tableName()))
+            {
+               repeatInserts.incrementAndGet();
+            }
+         }
+      };
+      StatementExecutionListeners.register(observer);
+      try
+      {
+         for(int count : List.of(12, 5))
+         {
+            try(var scope = UnitsOfWork.begin(UnitOfWorkKind.SCHEDULED_JOB, "job"))
+            {
+               query(count);
+            }
+         }
+         rows("db_perf_settings");
+         flusher.flushNow();
+         QRecord repeated = rows("db_perf_repeat_pattern").getFirst();
+         assertThat(repeated.getValueLong("unitsAffected")).isEqualTo(1);
+         assertThat(repeated.getValueInteger("maxRepeats")).isEqualTo(12);
+         assertThat(repeated.getValueLong("totalRepeats")).isEqualTo(12);
+         assertThat(repeated.getValueBigDecimal("sampleRate")).isEqualByComparingTo("1");
+         assertThat(rows("db_perf_repeat_pattern")).hasSize(1);
+         assertThat(rows("db_perf_attribution")).singleElement().satisfies(row ->
+         {
+            assertThat(row.getValueLong("units")).isEqualTo(2);
+            assertThat(row.getValueLong("statements")).isEqualTo(17);
+         });
+         flusher.flushNow();
+         assertThat(attributionInserts).hasValue(1);
+         assertThat(repeatInserts).hasValue(1);
+         assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
+      }
+      finally
+      {
+         StatementExecutionListeners.unregister(observer);
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Both runtime switches and sample zero take effect through cached settings.
+    *******************************************************************************/
+   @Test
+   void disabledSamplingAndFilters() throws Exception
+   {
+      settings.withValue("attributionSampleRate", BigDecimal.ZERO);
+      refresh();
+      query(1);
+      try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "zero"))
+      {
+         query(1);
+      }
+      assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
+      settings.withValue("attributionSampleRate", BigDecimal.ONE).withValue("attributionEnabled", false);
+      refresh();
+      query(1);
+      assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
+      settings.withValue("attributionEnabled", true);
+      refresh();
+      runtime.getConfig().setAttributionEnabled(false);
+      query(1);
+      runtime.getConfig().setAttributionEnabled(true);
+      listener.onStatement(new StatementExecution("wrong", "business", StatementOperation.QUERY, "select ?", 1, 1, true, null, null));
+      AttributionListener.suppress(true);
+      query(1);
+      AttributionListener.suppress(false);
+      rows("db_perf_settings");
+      assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
+      query(1);
+      assertThat(listener.drain(Instant.now()).attribution()).hasSize(1);
+   }
+
+
+   /*******************************************************************************
+    ** The cap includes the overflow bucket; a unit counts only once in it.
+    *******************************************************************************/
+   @Test
+   void overflowAndRepeatBounds() throws Exception
+   {
+      settings.withValue("attributionMaxKeys", 2).withValue("nPlusOneThreshold", 2);
+      refresh();
+      try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "many-shapes"))
+      {
+         for(int i = 0; i < 8; i++)
+         {
+            synthetic("select column_" + i + " from business", 1, true, scope.getUnitOfWork());
+         }
+      }
+      for(int i = 0; i < 8; i++)
+      {
+         try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "job_" + i))
+         {
+            synthetic("select column_" + i + " from business", 1, true, scope.getUnitOfWork());
+            synthetic("select column_" + i + " from business", 1, true, scope.getUnitOfWork());
+         }
+      }
+      flusher.flushNow();
+      assertThat(rows("db_perf_attribution")).hasSize(2);
+      assertThat(named(rows("db_perf_attribution"), "(other)").getValueLong("units")).isEqualTo(9);
+      assertThat(rows("db_perf_attribution").stream().mapToLong(row -> row.getValueLong("statements")).sum()).isEqualTo(24);
+      assertThat(rows("db_perf_repeat_pattern")).hasSize(2);
+      assertThat(named(rows("db_perf_repeat_pattern"), "(other)").getValueString("fingerprint")).isEqualTo(SqlNormalizer.fingerprint("(other)"));
+   }
+
+
+   /*******************************************************************************
+    ** Counters, normalization, null rows, process fallback, and text bounds.
+    *******************************************************************************/
+   @Test
+   void syntheticCountersAndSqlSafety()
+   {
+      String sql = "select 'secret', 123 from business where id in (?, ?, ?)";
+      synthetic(sql, 2000000, true, null);
+      synthetic(sql, 5000000, false, null);
+      listener.onStatement(new StatementExecution(runtime.getConfig().getBackendName(), null, null, null, 10, null, true, null, "process"));
+      flusher.flushNow();
+      List<QRecord> rows = rows("db_perf_attribution");
+      QRecord row = named(rows, "(none)");
+      assertThat(row.getValueBigDecimal("totalMs")).isEqualByComparingTo("7");
+      assertThat(row.getValueBigDecimal("maxMs")).isEqualByComparingTo("5");
+      assertThat(row.getValueLong("errors")).isEqualTo(1);
+      assertThat(row.getValueLong("rowsCount")).isEqualTo(2);
+      assertThat(row.getValueString("sqlText")).doesNotContain("secret", "123");
+      assertThat(named(rows, "process").getValueString("unitKind")).isEqualTo("OTHER");
+      synthetic("select " + "identifier".repeat(600) + " from business", 1, true, null);
+      flusher.flushNow();
+      assertThat(rows("db_perf_attribution")).anySatisfy(value -> assertThat(value.getValueString("sqlText")).hasSize(4000));
+   }
+
+
+   /*******************************************************************************
+    ** An abandoned unit is merged when another unit arrives on that worker.
+    *******************************************************************************/
+   @Test
+   void defensiveUnitTransitionAndWindowBoundary() throws Exception
+   {
+      var abandoned = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "abandoned");
+      query(1);
+      UnitsOfWork.reset();
+      try(var scope = UnitsOfWork.begin(UnitOfWorkKind.HTTP_REQUEST, "initial"))
+      {
+         query(1);
+         flusher.flushNow();
+         assertThat(rows("db_perf_attribution")).singleElement().satisfies(row -> assertThat(row.getValueString("unitName")).isEqualTo("abandoned"));
+         scope.getUnitOfWork().setName("final");
+         query(1);
+      }
+      abandoned.close();
+      flusher.flushNow();
+      assertThat(named(rows("db_perf_attribution"), "final").getValueLong("statements")).isEqualTo(2);
+   }
+
+
+   /*******************************************************************************
+    ** Each sampled unit contributes all statements, including across a refresh.
+    *******************************************************************************/
+   @Test
+   void samplingIsPerUnit() throws Exception
+   {
+      settings.withValue("attributionSampleRate", new BigDecimal("0.5"));
+      refresh();
+      for(int i = 0; i < 100; i++)
+      {
+         try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "sample_" + i))
+         {
+            for(int j = 0; j < 3; j++)
+            {
+               synthetic("select ?", 1, true, scope.getUnitOfWork());
+            }
+         }
+      }
+      var window = listener.drain(Instant.now());
+      assertThat(window.attribution().size()).isBetween(20, 80);
+      assertThat(window.attribution().values()).allSatisfy(counts -> assertThat(counts.statements.sum()).isEqualTo(3));
+   }
+
+
+   /*******************************************************************************
+    ** An in-flight unit keeps its sampling decision and original rate annotation.
+    *******************************************************************************/
+   @Test
+   void samplingRefreshDuringUnit() throws Exception
+   {
+      try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "in-flight"))
+      {
+         query(1);
+         settings.withValue("attributionSampleRate", BigDecimal.ZERO);
+         refresh();
+         query(1);
+      }
+      query(1);
+      flusher.flushNow();
+      assertThat(rows("db_perf_attribution")).singleElement().satisfies(row ->
+      {
+         assertThat(row.getValueLong("statements")).isEqualTo(2);
+         assertThat(row.getValueBigDecimal("sampleRate")).isEqualByComparingTo("1");
+      });
+   }
+
+
+   /*******************************************************************************
+    ** Atomic drains cannot lose or double-count concurrent statement delivery.
+    *******************************************************************************/
+   @Test
+   void concurrentDrains() throws Exception
+   {
+      var pool = Executors.newFixedThreadPool(4);
+      try
+      {
+         List<java.util.concurrent.Future<?>> tasks = new ArrayList<>();
+         for(int i = 0; i < 4; i++)
+         {
+            tasks.add(pool.submit(() ->
+            {
+               for(int j = 0; j < 5000; j++)
+               {
+                  synthetic("select ?", 1, true, null);
+               }
+            }));
+         }
+         long total = 0;
+         while(tasks.stream().anyMatch(task -> !task.isDone()))
+         {
+            total += listener.drain(Instant.now()).attribution().values().stream().mapToLong(counts -> counts.statements.sum()).sum();
+            Thread.yield();
+         }
+         for(var task : tasks)
+         {
+            task.get();
+         }
+         total += listener.drain(Instant.now()).attribution().values().stream().mapToLong(counts -> counts.statements.sum()).sum();
+         assertThat(total).isEqualTo(20000);
+      }
+      finally
+      {
+         pool.shutdownNow();
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Cached hot-path overhead stays below a generous two-second budget.
+    *******************************************************************************/
+   @Test
+   void overhead()
+   {
+      StatementExecution execution = new StatementExecution(runtime.getConfig().getBackendName(), "business", StatementOperation.QUERY, "select ?", 1, 1, true, null, null);
+      listener.onStatement(execution);
+      long start = System.nanoTime();
+      for(int i = 0; i < 100000; i++)
+      {
+         listener.onStatement(execution);
+      }
+      assertThat(System.nanoTime() - start).isLessThan(TimeUnit.SECONDS.toNanos(2));
+      assertThat(listener.drain(Instant.now()).attribution().values().iterator().next().statements.sum()).isEqualTo(100001);
+   }
+
+
+   /*******************************************************************************
+    ** Launcher lifecycle registers once, flushes on stop, and respects config.
+    *******************************************************************************/
+   @Test
+   void runtimeLifecycle() throws Exception
+   {
+      flusher.close();
+      assertThat(StatementExecutionListeners.isActive()).isFalse();
+      AttributionRuntimeService service = new AttributionRuntimeService();
+      assertThat(service.getName()).isEqualTo("dbPerformanceAttribution");
+      service.stop();
+      service.start(runtime.getInstance());
+      runtime.startAttribution();
+      assertThat(StatementExecutionListeners.isActive()).isTrue();
+      query(1);
+      service.stop();
+      service.stop();
+      assertThat(StatementExecutionListeners.isActive()).isFalse();
+      assertThat(rows("db_perf_attribution")).hasSize(1);
+      runtime.getConfig().setAttributionEnabled(false);
+      service.start(runtime.getInstance());
+      assertThat(StatementExecutionListeners.isActive()).isFalse();
+      service.stop();
+   }
+
+
+   /*******************************************************************************
+    ** Real flusher rows roll up across pods, preserving max and additive totals.
+    *******************************************************************************/
+   @Test
+   void rollupMergesPods() throws Exception
+   {
+      for(int i = 1; i <= 2; i++)
+      {
+         runtime.getConfig().setPodName("pod_" + i);
+         try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "job"))
+         {
+            synthetic("select ?", i * 1000000L, true, scope.getUnitOfWork());
+            synthetic("select ?", i * 1000000L, false, scope.getUnitOfWork());
+         }
+         flusher.flushNow();
+      }
+      new RollupService(runtime).rollupDay(LocalDate.now(ZoneOffset.UTC));
+      assertThat(rows("db_perf_attribution_daily")).singleElement().satisfies(row ->
+      {
+         assertThat(row.getValueLong("units")).isEqualTo(2);
+         assertThat(row.getValueLong("statements")).isEqualTo(4);
+         assertThat(row.getValueBigDecimal("totalMs")).isEqualByComparingTo("6");
+         assertThat(row.getValueBigDecimal("maxMs")).isEqualByComparingTo("2");
+         assertThat(row.getValueLong("rowsCount")).isEqualTo(4);
+         assertThat(row.getValueLong("errors")).isEqualTo(2);
+      });
+   }
+
+
+   /*******************************************************************************
+    ** Large windows use one QQQ insert action even when the backend pages SQL.
+    *******************************************************************************/
+   @Test
+   void largeWindowUsesOneInsertAction() throws Exception
+   {
+      CountingInsert.inputs.clear();
+      for(String table : List.of("db_perf_attribution", "db_perf_repeat_pattern"))
+      {
+         runtime.getInstance().getTable(table).getCustomizers().put(TableCustomizers.PRE_INSERT_RECORD.getRole(), new QCodeReference(CountingInsert.class));
+      }
+      settings.withValue("nPlusOneThreshold", 2);
+      refresh();
+      for(int i = 0; i < 1005; i++)
+      {
+         try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "job_" + i))
+         {
+            synthetic("select ?", 1, true, scope.getUnitOfWork());
+            synthetic("select ?", 1, true, scope.getUnitOfWork());
+         }
+      }
+      flusher.flushNow();
+      assertThat(rows("db_perf_attribution")).hasSize(1005);
+      assertThat(rows("db_perf_repeat_pattern")).hasSize(1005);
+      assertThat(CountingInsert.inputs).extracting(InsertInput::getTableName).containsExactlyInAnyOrder("db_perf_attribution", "db_perf_repeat_pattern");
+      assertThat(CountingInsert.inputs).allSatisfy(input -> assertThat(input.getRecords()).hasSize(1005));
+   }
+
+
+   /*******************************************************************************
+    ** An insert failure drops the window and never poisons a later good window.
+    *******************************************************************************/
+   @Test
+   void failedWindowIsDropped() throws Exception
+   {
+      RDBMSTableBackendDetails details = (RDBMSTableBackendDetails) runtime.getInstance().getTable("db_perf_attribution").getBackendDetails();
+      query(1);
+      details.setTableName("missing_attribution_table");
+      flusher.flushNow();
+      details.setTableName("db_perf_attribution");
+      flusher.flushNow();
+      assertThat(rows("db_perf_attribution")).isEmpty();
+      query(1);
+      flusher.flushNow();
+      assertThat(rows("db_perf_attribution")).singleElement().satisfies(row -> assertThat(row.getValueLong("statements")).isEqualTo(1));
+   }
+
+
+   /*******************************************************************************
+    ** Cold-cache overflow remains bounded and later hits keep the right shape.
+    *******************************************************************************/
+   @Test
+   void normalizationCacheOverflow()
+   {
+      for(int i = 0; i < 5010; i++)
+      {
+         synthetic("select column_" + i + " from business", 1, true, null);
+      }
+      synthetic("select column_0 from business", 1, true, null);
+      var window = listener.drain(Instant.now());
+      assertThat(window.attribution()).hasSize(5010);
+      String fingerprint = SqlNormalizer.fingerprint(SqlNormalizer.normalize("select column_0 from business"));
+      assertThat(window.attribution().entrySet().stream().filter(entry -> entry.getKey().fingerprint().equals(fingerprint)).findFirst().orElseThrow().getValue().statements.sum()).isEqualTo(2);
+   }
+
+
+   /*******************************************************************************
+    ** The daemon executes on its own system context and honors a short interval.
+    *******************************************************************************/
+   @Test
+   void scheduledFlush() throws Exception
+   {
+      settings.withValue("attributionFlushSeconds", 1);
+      refresh();
+      flusher.start();
+      flusher.start();
+      query(1);
+      long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+      while(rows("db_perf_attribution").isEmpty() && System.nanoTime() < deadline)
+      {
+         Thread.sleep(25);
+      }
+      assertThat(rows("db_perf_attribution")).hasSize(1);
+      assertThat(QContext.getQInstance()).isSameAs(runtime.getInstance());
+      flusher.close();
+      flusher.start();
+      assertThat(StatementExecutionListeners.isActive()).isFalse();
+   }
+
+
+   /*******************************************************************************
+    ** Only one runtime attributes per JVM: starting another stops the first, so
+    ** repeated server starts (tests, hot swaps) never pile up listeners.
+    *******************************************************************************/
+   @Test
+   void oneActiveRuntime() throws Exception
+   {
+      StatementExecutionListeners.unregister(listener);
+      runtime.startAttribution();
+      DbPerfRuntime other = new DbPerfRuntime(runtime.getInstance(), runtime.getConfig());
+      other.startAttribution();
+      runtime.stopAttribution();
+      assertThat(StatementExecutionListeners.isActive()).isTrue();
+      other.stopAttribution();
+      assertThat(StatementExecutionListeners.isActive()).isFalse();
+   }
+
+
+
+   /*******************************************************************************
+    ** Deliver a statement with exact counter values.
+    *******************************************************************************/
+   private void synthetic(String sql, long nanos, boolean succeeded, UnitOfWork unit)
+   {
+      listener.onStatement(new StatementExecution(runtime.getConfig().getBackendName(), "business", StatementOperation.QUERY, sql, nanos, 1, succeeded, unit, null));
+   }
+
+
+   /*******************************************************************************
+    ** Execute real RDBMS-backed QQQ queries.
+    *******************************************************************************/
+   private void query(int count) throws Exception
+   {
+      for(int i = 0; i < count; i++)
+      {
+         new QueryAction().execute(new QueryInput("business").withFilter(new QQueryFilter()));
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Persist administrator settings, then explicitly refresh the cached copy.
+    *******************************************************************************/
+   private void refresh() throws Exception
+   {
+      DbPerfStore.update("db_perf_settings", List.of(settings));
+      flusher.refreshSettings();
+   }
+
+
+   /*******************************************************************************
+    ** Read persisted evidence with a concise test failure on action errors.
+    *******************************************************************************/
+   private List<QRecord> rows(String table)
+   {
+      try
+      {
+         return (DbPerfStore.query(table, new QQueryFilter()));
+      }
+      catch(Exception e)
+      {
+         throw (new AssertionError(e));
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Find one named aggregate.
+    *******************************************************************************/
+   private static QRecord named(List<QRecord> rows, String name)
+   {
+      return (rows.stream().filter(row -> name.equals(row.getValueString("unitName"))).findFirst().orElseThrow());
+   }
+
+   /*******************************************************************************
+    ** Observe whole QQQ actions while preserving the production write guard.
+    *******************************************************************************/
+   public static class CountingInsert extends DbPerfWriteGuard
+   {
+      private static final List<InsertInput> inputs = new ArrayList<>();
+
+
+      /*******************************************************************************
+       ** Capture each action's original input before the backend pages its SQL.
+       *******************************************************************************/
+      @Override
+      public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
+      {
+         inputs.add(input);
+         return (super.preInsert(input, records, preview));
+      }
+   }
+}
Author	SHA1	Message	Date
james.maesandCodex	3dfaf0401e	ci: centralize branch verification and source scanning Co-Authored-By: Codex <noreply@openai.com>	2026-10-21 12:10:54 -05:00
james.maesandCodex	681598be57	ci: adopt central orb 0.7.12 and stable gates (#21 ) Co-authored-by: Codex <noreply@openai.com>	2026-10-21 08:56:36 -05:00
james.maesandCodex	ea9036509c	ci: centralize rc release completion in qqq orb (#20 ) Co-authored-by: Codex <noreply@openai.com>	2026-10-21 02:25:42 -05:00
james.maesandCodex	2a2ba58b06	ci: adopt shared orb 0.7.11 (#19 ) Co-authored-by: Codex <noreply@openai.com>	2026-10-21 00:57:50 -05:00
james.maes	61e9c22563	Merge pull request #18 from QRun-IO/feature/central-ci ci: adopt shared QQQ orb standard	2026-10-20 18:57:48 -05:00
james.maesandCodex	65922dd413	ci: adopt verified central QQQ orb 0.7.3 Co-Authored-By: Codex <noreply@openai.com>	2026-10-20 18:52:28 -05:00
james.maes	bdaf35fc3e	Merge pull request #17 from QRun-IO/feature/bounded-spike-history fix(alerts): page spike history instead of holding 7 days of query snapshots	2026-10-16 15:54:56 -05:00
james.maesandClaude Opus 5.5	c49b479620	fix(alerts): page spike history instead of holding 7 days of query snapshots SnapshotEvaluator.spikes loaded RuleContext.window("query_snapshot", 7) as QRecords on every snapshot evaluation (every collection interval on the worker). kof22 writes ~33,000 query snapshots a day, so by day 7 that is ~230,000 records per evaluation. history() now reads the window in BATCH_SIZE keyset pages (only OK and PARTIAL snapshots, as before) and keeps per statement only the current interval time, the latest capture time and earlier interval times in a primitive double array. Median, twelve-sample baseline and the one-hour absence rule are unchanged (SnapshotEvaluatorTest still passes). SpikeHistoryBoundedTest counts every query snapshot QQQ materializes: 2,500 samples never exceed one 1,000-row page, failed snapshots and samples outside the window are excluded, the current interval is found. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-16 15:24:44 -05:00
james.maes	de991509ca	Merge pull request #16 from QRun-IO/feature/bounded-attribution-reads fix(attribution): sum attribution in the database, page the daily rollup (prod OOM)	2026-10-16 15:15:33 -05:00
james.maesandClaude Opus 5.5	fd0116fbd0	fix(analyze): read attribution as database sums in the hourly analysis dbPerfAnalyze (hourly, on the worker) loaded the full day of db_perf_attribution through RuleContext.window and cached it for the rules: ~570,000 rows. kof22 worker working set jumped from ~660 MiB to ~1.84 GiB at every :07-:08 run on 2026-10-16 (2 Gi limit, 1.5 GB heap). RuleContext.window("attribution", days) now returns AggregateAction sums (units, statements, rowsCount, errors, totalMs; max maxMs) grouped by unitKind, unitName, tableName, operation: the only fields the rules (OVER_AUDITED, HOT_JOB) read. AttributionBoundedReadTest proves no detail rows are loaded and the totals are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-16 15:04:23 -05:00
james.maesandClaude Opus 5.5	744ff04ade	fix(attribution): sum attribution in the database, page the daily rollup The Attribution home widget queried every db_perf_attribution row of the last 24 hours into QRecords and summed them in Java. kof22 writes about 30,000 attribution rows an hour, so one render loaded ~570,000 rows (540 MB in PostgreSQL). On 2026-10-16 a single admin request logged "Large number of records in QueryOutputList" at 50k, 100k and 250k rows and the web pod died with OutOfMemoryError: Java heap space about 50 s later, after slowing every other request with GC thrash. - DbPerfWidgetRenderer.attribution() now uses AggregateAction: SUM of statements and totalMs grouped by unitKind, unitName, tableName, ordered by time and capped at 5,000 groups (prod has ~1,000). - RollupService reads the day's attribution detail in 1,000-row keyset pages through the new DbPerfStore.forEachPage, so the nightly rollup on the worker is bounded by distinct keys instead of the day's rows. - AttributionBoundedReadTest counts every detail record QQQ materializes (post-query customizer): the widget loads none, the rollup never holds more than BATCH_SIZE at once, and sums stay correct across pages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-16 14:34:58 -05:00
james.maes	fdcad364f1	Merge pull request #10 from QRun-IO/ci/release-rc-secrets-inherit ci(release-rc): pass secrets so the shared workflow can sign the RC tag	2026-10-16 08:04:25 -05:00
james.maes	7c71e384c9	Merge pull request #12 from QRun-IO/fix/collector-failures fix(collect): no stack overflow on huge statements; every failure closes the run as FAILED	2026-10-15 17:43:04 -05:00
james.maesandClaude Opus 5.5	b882e3d0f0	fix(collect): no stack overflow on huge statements; every failure closes the run The normalizer folded IN and VALUES lists with nested-repetition regular expressions, which threw StackOverflowError on kof22's 60 KB statements. That Error escaped catch(Exception), so runs stayed RUNNING, nothing was logged and no alert fired. List folding is now a linear token scan, the collector catches Throwable (rethrowing OutOfMemoryError), logs class and frame at ERROR, marks FAILED and alerts, and stale RUNNING runs are closed. Verified against kof22's 4,462 real statement texts (under 1 s). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 17:40:01 -05:00
james.maes	bb7e852a35	Merge pull request #11 from QRun-IO/fix/single-attribution-runtime fix(attribution): one active attribution runtime per JVM	2026-10-15 17:09:55 -05:00
james.maesandClaude Opus 5.5	8a34ac4404	ci(release-rc): pass secrets to the shared workflow so it can sign the RC tag QRun-IO/.github#2 adds a tag-and-release job that signs vX with the QRUN_TAG_SIGNING_KEY secret after Maven Central confirms the artifact. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 16:19:54 -05:00
james.maesandClaude Opus 5.5	63d4c9669f	fix(attribution): one active attribution runtime per JVM Statement listeners are JVM-wide. Starting attribution on another instance now stops the previous runtime's listener and flusher, so repeated server starts (tests, hot swaps) never pin old instances. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 16:11:08 -05:00
james.maes	1663f48073	Merge pull request #9 from QRun-IO/fix/table-icons fix(nav): distinct icon per table; cuts 0.1.0-RC.3	2026-10-15 14:15:35 -05:00
james.maesandClaude Opus 5.5	0b517d7e16	fix(nav): a distinct icon per table and a storage app icon Every db_perf_ table used the speed icon, which repeats in the nav and clashes with web performance (kof22's nav icon uniqueness check). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 14:12:56 -05:00
james.maes	1bc250c1b8	Merge pull request #8 from QRun-IO/fix/widget-columns fix(home): at most four columns per widget table; cuts 0.1.0-RC.2	2026-10-15 13:25:03 -05:00
james.maesandClaude Opus 5.5	c9652fd541	fix(home): at most four columns per widget table, plain headers Top queries, live activity, attribution, repeated statements and tables fold secondary facts into a muted second line of the first cell, so the home stays readable at every width (kof22's widget readability guard). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 13:22:12 -05:00
james.maes	8446ab1f0b	Merge pull request #7 from QRun-IO/feature/attribution feat: QQQ attribution and N+1 detection (QQQ 4.1.0-RC.13); cuts 0.1.0-RC.1	2026-10-15 12:55:23 -05:00
james.maesandClaude Opus 5.5	fe41490559	Merge develop (rule seed text) into feature/attribution; number the sample_rate changeset 34 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 12:52:02 -05:00
james.maesandClaude Opus 5.5	34db3cdff6	Merge develop into feature/attribution; help for sampleRate, README and CHANGELOG Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 12:21:26 -05:00
james.maes	d61684b15e	Merge pull request #6 from QRun-IO/fix/rule-seed-text fix(rules): helpful rule descriptions and threshold labels; spec alert names	2026-10-15 12:18:59 -05:00
james.maesandClaude Opus 5.5	564ebf8d97	feat(attribution): QQQ statement attribution and N+1 detection Registers a core StatementExecutionListener (QQQ 4.1.0-RC.13) that aggregates statements per unit of work (route, scheduled job, process), table, operation and fingerprint in bounded memory, detects repeated statement shapes per unit, and flushes each window with one batched insert per table from a daemon thread that never measures itself. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 11:44:26 -05:00