From 744ff04adecfab7d38405f98a20319d24b71f682 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 16 Oct 2026 14:34:58 -0500 Subject: [PATCH 1/2] fix(attribution): sum attribution in the database, page the daily rollup The Attribution home widget queried every db_perf_attribution row of the last 24 hours into QRecords and summed them in Java. kof22 writes about 30,000 attribution rows an hour, so one render loaded ~570,000 rows (540 MB in PostgreSQL). On 2026-10-16 a single admin request logged "Large number of records in QueryOutputList" at 50k, 100k and 250k rows and the web pod died with OutOfMemoryError: Java heap space about 50 s later, after slowing every other request with GC thrash. - DbPerfWidgetRenderer.attribution() now uses AggregateAction: SUM of statements and totalMs grouped by unitKind, unitName, tableName, ordered by time and capped at 5,000 groups (prod has ~1,000). - RollupService reads the day's attribution detail in 1,000-row keyset pages through the new DbPerfStore.forEachPage, so the nightly rollup on the worker is bounded by distinct keys instead of the day's rows. - AttributionBoundedReadTest counts every detail record QQQ materializes (post-query customizer): the widget loads none, the rollup never holds more than BATCH_SIZE at once, and sums stay correct across pages. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + .../dbperformance/collect/DbPerfStore.java | 42 ++++ .../dbperformance/collect/RollupService.java | 15 +- .../widgets/DbPerfWidgetRenderer.java | 47 +++- .../AttributionBoundedReadTest.java | 214 ++++++++++++++++++ 5 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e53fd1a..104dd8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes are documented here. This project follows Keep a Changelog a ## [Unreleased] ### Fixed +- The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. - Collection no longer stops at RUNNING on large production statement sets: list folding in the SQL normalizer is a linear token scan (the nested-repetition regular expressions overflowed the stack on 60 KB statements with long IN/VALUES lists). Any failure, including an `Error`, now closes the run as FAILED with the exception class, logs it at ERROR (class and frame only, never SQL text) and raises the collection alert; runs left RUNNING by a crash are closed as FAILED after three intervals. - Only one runtime attributes per JVM: starting attribution on a newer instance (a hot swap, or repeated server starts in tests) stops the previous one, so old instances are never pinned by a forgotten listener and flusher. - Each table has its own icon (the app icon is now `storage`), so nav entries are distinguishable and do not repeat icons used elsewhere. diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/collect/DbPerfStore.java b/src/main/java/com/kingsrook/qbits/dbperformance/collect/DbPerfStore.java index 4088834..3e2a865 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/collect/DbPerfStore.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/collect/DbPerfStore.java @@ -66,6 +66,48 @@ public final class DbPerfStore } + /******************************************************************************* + ** Keyset-paged read for tables that can hold more rows than fit in the heap + ** (attribution writes hundreds of thousands per day). At most pageSize + ** records are held at once; the consumer must not keep the page. + *******************************************************************************/ + public static void forEachPage(String table, QQueryFilter filter, int pageSize, PageConsumer consumer) throws Exception + { + Serializable lastId = 0; + while(true) + { + QQueryFilter page = new QQueryFilter().withSubFilter(filter).withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, lastId)) + .withOrderBy(new QFilterOrderBy("id", true)).withLimit(pageSize); + List rows = query(table, page); + if(rows.isEmpty()) + { + return; + } + consumer.accept(rows); + if(rows.size() < pageSize) + { + return; + } + lastId = rows.getLast().getValue("id"); + } + } + + + + /******************************************************************************* + ** Receives one bounded page from forEachPage. + *******************************************************************************/ + @FunctionalInterface + public interface PageConsumer + { + /*************************************************************************** + ** Handle one page. + ***************************************************************************/ + void accept(List page) throws Exception; + } + + + /******************************************************************************* ** First matching record. *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/collect/RollupService.java b/src/main/java/com/kingsrook/qbits/dbperformance/collect/RollupService.java index c6e2a0d..19b9c87 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/collect/RollupService.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/collect/RollupService.java @@ -168,7 +168,19 @@ public class RollupService { Map groups = new LinkedHashMap<>(); List dimensions = List.of("unitKind", "unitName", "processName", "tableName", "operation", "fingerprint"); - for(QRecord row : DbPerfStore.query("db_perf_attribution", range("windowStart", start, end))) + DbPerfStore.forEachPage("db_perf_attribution", range("windowStart", start, end), DbPerfStore.BATCH_SIZE, page -> mergeAttribution(day, page, dimensions, groups)); + replace("db_perf_attribution_daily", day, new ArrayList<>(groups.values())); + } + + + + /******************************************************************************* + ** Fold one page of attribution detail into the day's groups. Memory is + ** bounded by distinct dimension keys, never by the day's detail row count. + *******************************************************************************/ + private static void mergeAttribution(LocalDate day, List page, List dimensions, Map groups) + { + for(QRecord row : page) { StringBuilder key = new StringBuilder(); for(String field : dimensions) @@ -187,7 +199,6 @@ public class RollupService sumDecimal(aggregate, row, "totalMs"); aggregate.setValue("maxMs", SnapshotCollector.decimal(aggregate, "maxMs").max(SnapshotCollector.decimal(row, "maxMs"))); } - replace("db_perf_attribution_daily", day, new ArrayList<>(groups.values())); } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java index f71d5b6..9c61e4a 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java @@ -39,8 +39,15 @@ import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; import com.kingsrook.qbits.dbperformance.provider.ConnectionCount; import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -59,6 +66,7 @@ import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBl import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextStyles; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.map; @@ -280,22 +288,45 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer + /******************************************************************************* + ** Most (unit, table) groups summed for the attribution panel. Production + ** shows about a thousand; the cap only guards pathological unit names. + *******************************************************************************/ + public static final int MAX_ATTRIBUTION_GROUPS = 5000; + + + /******************************************************************************* ** Aggregate units over 24 hours and show repeated-statement patterns below them. + ** The sums run in the database: a day of attribution detail is hundreds of + ** thousands of rows (about 540 MB), and loading it into the heap on a widget + ** request ran the web pod out of memory (2026-10-16). *******************************************************************************/ public MultiTableData attribution() throws QException { - List records = DbPerfStore.query("db_perf_attribution", since("windowEnd")); + GroupBy kindGroup = new GroupBy(QFieldType.STRING, "unitKind"); + GroupBy nameGroup = new GroupBy(QFieldType.STRING, "unitName"); + GroupBy tableGroup = new GroupBy(QFieldType.STRING, "tableName"); + Aggregate statementsSum = new Aggregate("statements", AggregateOperator.SUM); + Aggregate totalMsSum = new Aggregate("totalMs", AggregateOperator.SUM); + List results = new AggregateAction().execute(new AggregateInput("db_perf_attribution") + .withFilter(since("windowEnd").withOrderBy(new QFilterOrderByAggregate(totalMsSum, false))) + .withGroupBy(kindGroup).withGroupBy(nameGroup).withGroupBy(tableGroup) + .withAggregate(statementsSum).withAggregate(totalMsSum) + .withLimit(MAX_ATTRIBUTION_GROUPS) + .withTransaction(QContext.getQBackendTransaction())).getResults(); Map> grouped = new LinkedHashMap<>(); Map> tableTimes = new LinkedHashMap<>(); - double total = records.stream().mapToDouble(row -> number(row, "totalMs")).sum(); - for(QRecord row : records) + double total = results.stream().mapToDouble(row -> numeric(row.getAggregateValue(totalMsSum))).sum(); + for(AggregateResult row : results) { - String key = row.getValueString("unitKind") + "\n" + row.getValueString("unitName"); - Map group = grouped.computeIfAbsent(key, ignored -> map("Kind", row.getValueString("unitKind"), "Name", row.getValueString("unitName"), "Statements", 0.0, "Total ms", 0.0)); - group.put("Statements", numeric(group.get("Statements")) + number(row, "statements")); - group.put("Total ms", numeric(group.get("Total ms")) + number(row, "totalMs")); - tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getValueString("tableName"), "Unknown"), number(row, "totalMs"), Double::sum); + String kind = (String) row.getGroupByValue(kindGroup); + String name = (String) row.getGroupByValue(nameGroup); + String key = kind + "\n" + name; + Map group = grouped.computeIfAbsent(key, ignored -> map("Kind", kind, "Name", name, "Statements", 0.0, "Total ms", 0.0)); + group.put("Statements", numeric(group.get("Statements")) + numeric(row.getAggregateValue(statementsSum))); + group.put("Total ms", numeric(group.get("Total ms")) + numeric(row.getAggregateValue(totalMsSum))); + tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getGroupByValue(tableGroup), "Unknown"), numeric(row.getAggregateValue(totalMsSum)), Double::sum); } grouped.forEach((key, group) -> { diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java new file mode 100644 index 0000000..ecf5efd --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java @@ -0,0 +1,214 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.collect.RollupService; +import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Attribution detail grows by hundreds of thousands of rows a day. On + ** 2026-10-16 the home widget loaded a whole day of it into the heap and ran + ** the kof22 web pod out of memory. These tests count every attribution + ** detail record QQQ materializes, so neither the widget nor the daily rollup + ** can go back to reading the table in one unbounded query. + *******************************************************************************/ +class AttributionBoundedReadTest +{ + private static final AtomicInteger LOADED = new AtomicInteger(); + private static final AtomicInteger MAX_PAGE = new AtomicInteger(); + + private DbPerfRuntime runtime; + + + + /******************************************************************************* + ** Counts the detail records each QueryAction hands back. + *******************************************************************************/ + public static class CountingCustomizer implements TableCustomizerInterface + { + /*************************************************************************** + ** Record the page size, change nothing. + ***************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface queryInput, List records) + { + LOADED.addAndGet(records.size()); + MAX_PAGE.accumulateAndGet(records.size(), Math::max); + return (records); + } + } + + + + /******************************************************************************* + ** Fresh H2 instance with the counter on the detail table. + *******************************************************************************/ + @BeforeEach + void setup() throws Exception + { + runtime = DbPerfRuntime.of(DbPerfTestSupport.instance()); + runtime.getInstance().getTable("db_perf_attribution").withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CountingCustomizer.class)); + LOADED.set(0); + MAX_PAGE.set(0); + } + + + + /******************************************************************************* + ** The widget sums in the database: no detail rows reach the heap, and the + ** totals still match the detail. + *******************************************************************************/ + @Test + void widgetSumsInTheDatabase() throws Exception + { + Instant now = Instant.now(); + insert(3000, i -> row(now.minusSeconds(60), now, i)); + LOADED.set(0); + + MultiTableData attribution = new DbPerfWidgetRenderer().attribution(); + + assertThat(LOADED.get()).as("attribution detail rows loaded by the widget").isZero(); + List> units = attribution.getTableDataList().getFirst().getRows(); + assertThat(units).hasSize(3); + // unit-0 holds i = 0, 3, 6 ... (1000 rows, 2 statements each), mostly against "patient" + assertThat(((Number) units.getFirst().get("Statements")).doubleValue() + ((Number) units.get(1).get("Statements")).doubleValue() + + ((Number) units.get(2).get("Statements")).doubleValue()).isEqualTo(6000.0); + assertThat(units).allSatisfy(unit -> assertThat(((Number) unit.get("Statements")).doubleValue()).isEqualTo(2000.0)); + assertThat(JsonUtils.toJson(units.getFirst())).contains("top table patient"); + assertThat(units.stream().mapToDouble(unit -> ((Number) unit.get("Share %")).doubleValue()).sum()).isCloseTo(100.0, org.assertj.core.data.Offset.offset(0.001)); + } + + + + /******************************************************************************* + ** Detail older than 24 hours is left out of the sums. + *******************************************************************************/ + @Test + void widgetKeepsTheDayWindow() throws Exception + { + Instant now = Instant.now(); + insert(3, i -> row(now.minusSeconds(60), now, i)); + insert(3, i -> row(now.minusSeconds(3 * 86400), now.minusSeconds(2 * 86400), i)); + List> units = new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getRows(); + assertThat(units).hasSize(3).allSatisfy(unit -> assertThat(((Number) unit.get("Statements")).doubleValue()).isEqualTo(2.0)); + } + + + + /******************************************************************************* + ** Keyset pages never exceed the page size, and every row is seen once. + *******************************************************************************/ + @Test + void forEachPageIsBounded() throws Exception + { + Instant now = Instant.now(); + insert(1050, i -> row(now.minusSeconds(60), now, i)); + LOADED.set(0); + List sizes = new ArrayList<>(); + List ids = new ArrayList<>(); + DbPerfStore.forEachPage("db_perf_attribution", new QQueryFilter(), 100, page -> + { + sizes.add(page.size()); + page.forEach(row -> ids.add(row.getValue("id"))); + }); + assertThat(sizes).hasSize(11).allSatisfy(size -> assertThat(size).isLessThanOrEqualTo(100)); + assertThat(ids).hasSize(1050).doesNotHaveDuplicates(); + assertThat(MAX_PAGE.get()).isLessThanOrEqualTo(100); + } + + + + /******************************************************************************* + ** The daily rollup reads the day in BATCH_SIZE pages and its sums still + ** cover every detail row across page boundaries. + *******************************************************************************/ + @Test + void rollupReadsTheDayInPages() throws Exception + { + ZoneId zone = ZoneId.of(runtime.getConfig().getTimeZone()); + LocalDate day = LocalDate.now(zone).minusDays(1); + Instant start = day.atStartOfDay(zone).toInstant().plusSeconds(3600); + int rows = DbPerfStore.BATCH_SIZE * 2 + 500; + insert(rows, i -> row(start, start.plusSeconds(60), i)); + MAX_PAGE.set(0); + LOADED.set(0); + + new RollupService(runtime).rollupDay(day); + + assertThat(MAX_PAGE.get()).as("largest page of attribution detail held at once").isLessThanOrEqualTo(DbPerfStore.BATCH_SIZE); + assertThat(LOADED.get()).isEqualTo(rows); + List daily = DbPerfStore.query("db_perf_attribution_daily", new QQueryFilter()); + assertThat(daily).hasSize(6); + assertThat(daily.stream().mapToLong(row -> row.getValueLong("statements")).sum()).isEqualTo(2L * rows); + } + + + + /******************************************************************************* + ** Insert generated rows in batches. + *******************************************************************************/ + private static void insert(int count, java.util.function.IntFunction factory) throws Exception + { + List rows = new ArrayList<>(); + for(int i = 0; i < count; i++) + { + rows.add(factory.apply(i)); + } + DbPerfStore.insert("db_perf_attribution", rows); + } + + + + /******************************************************************************* + ** Three units; each unit spends most of its time on "patient". + *******************************************************************************/ + private static QRecord row(Instant windowStart, Instant windowEnd, int i) + { + boolean patient = i % 6 < 3; + return (new QRecord().withValue("windowStart", windowStart).withValue("windowEnd", windowEnd).withValue("podName", "pod-" + (i % 2)) + .withValue("unitKind", "PROCESS").withValue("unitName", "unit-" + (i % 3)).withValue("tableName", patient ? "patient" : "visit") + .withValue("operation", "SELECT").withValue("fingerprint", "f".repeat(64)).withValue("sqlText", "select 1") + .withValue("units", 1L).withValue("statements", 2L).withValue("rowsCount", 1L).withValue("errors", 0L) + .withValue("totalMs", patient ? BigDecimal.TEN : BigDecimal.ONE).withValue("maxMs", BigDecimal.ONE)); + } +} -- 2.54.0 From fd0116fbd03b90dd429558de8bace875fb38de96 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 16 Oct 2026 15:04:23 -0500 Subject: [PATCH 2/2] fix(analyze): read attribution as database sums in the hourly analysis dbPerfAnalyze (hourly, on the worker) loaded the full day of db_perf_attribution through RuleContext.window and cached it for the rules: ~570,000 rows. kof22 worker working set jumped from ~660 MiB to ~1.84 GiB at every :07-:08 run on 2026-10-16 (2 Gi limit, 1.5 GB heap). RuleContext.window("attribution", days) now returns AggregateAction sums (units, statements, rowsCount, errors, totalMs; max maxMs) grouped by unitKind, unitName, tableName, operation: the only fields the rules (OVER_AUDITED, HOT_JOB) read. AttributionBoundedReadTest proves no detail rows are loaded and the totals are unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- .../dbperformance/analyze/RuleContext.java | 58 ++++++++++++++++++- .../AttributionBoundedReadTest.java | 31 +++++++++- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 104dd8d..14725c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes are documented here. This project follows Keep a Changelog a ## [Unreleased] ### Fixed -- The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. +- The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. The hourly analysis (`dbPerfAnalyze`) reads attribution the same way: `RuleContext.window("attribution", ...)` returns database sums per unit, table and operation instead of every detail row (it filled the kof22 worker heap to ~1.8 GB every hour). - Collection no longer stops at RUNNING on large production statement sets: list folding in the SQL normalizer is a linear token scan (the nested-repetition regular expressions overflowed the stack on 60 KB statements with long IN/VALUES lists). Any failure, including an `Error`, now closes the run as FAILED with the exception class, logs it at ERROR (class and frame only, never SQL text) and raises the collection alert; runs left RUNNING by a crash are closed as FAILED after three intervals. - Only one runtime attributes per JVM: starting attribution on a newer instance (a hot swap, or repeated server starts in tests) stops the previous one, so old instances are never pinned by a forgotten listener and flusher. - Each table has its own icon (the app icon is now `storage`), so nav entries are distinguishable and do not repeat icons used elsewhere. diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/RuleContext.java b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/RuleContext.java index b8f4ad7..5e373e0 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/RuleContext.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/RuleContext.java @@ -25,17 +25,27 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qbits.dbperformance.DbPerfRuntime; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* @@ -130,7 +140,7 @@ public class RuleContext { filter.withCriteria(new QFilterCriteria("backendName", QCriteriaOperator.EQUALS, backend)); } - List rows = read("db_perf_" + suffix, filter); + List rows = suffix.equals("attribution") ? attributionTotals(filter) : read("db_perf_" + suffix, filter); if(!Set.of("snapshot", "attribution", "repeat_pattern").contains(suffix)) { Set ids = window("snapshot", days).stream() @@ -144,6 +154,52 @@ public class RuleContext } + /******************************************************************************* + ** Most attribution groups the rules receive (prod: ~2,000 per day). + *******************************************************************************/ + public static final int MAX_ATTRIBUTION_GROUPS = 50_000; + + + + /******************************************************************************* + ** Attribution summed in the database per unit, table and operation: the + ** fields the rules read (statements, totalMs and the other counters), never + ** the day's detail rows. A day is ~570,000 rows in kof22 and loading it made + ** every hourly dbPerfAnalyze fill the worker heap (2026-10-16). + *******************************************************************************/ + private static List attributionTotals(QQueryFilter filter) throws QException + { + List dimensions = List.of("unitKind", "unitName", "tableName", "operation"); + AggregateInput input = new AggregateInput("db_perf_attribution").withFilter(filter).withLimit(MAX_ATTRIBUTION_GROUPS) + .withTransaction(QContext.getQBackendTransaction()); + Map groupBys = new LinkedHashMap<>(); + for(String dimension : dimensions) + { + GroupBy groupBy = new GroupBy(QFieldType.STRING, dimension); + groupBys.put(dimension, groupBy); + input.withGroupBy(groupBy); + } + Map aggregates = new LinkedHashMap<>(); + for(String counter : List.of("units", "statements", "rowsCount", "errors", "totalMs")) + { + aggregates.put(counter, new Aggregate(counter, AggregateOperator.SUM)); + } + aggregates.put("maxMs", new Aggregate("maxMs", AggregateOperator.MAX)); + aggregates.values().forEach(input::withAggregate); + + List rows = new ArrayList<>(); + for(AggregateResult result : new AggregateAction().execute(input).getResults()) + { + QRecord row = new QRecord(); + groupBys.forEach((field, groupBy) -> row.setValue(field, result.getGroupByValue(groupBy))); + aggregates.forEach((field, aggregate) -> row.setValue(field, result.getAggregateValue(aggregate))); + rows.add(row); + } + return (rows); + } + + + /******************************************************************************* ** Latest inventory only: retired tables and dropped indexes cannot recur. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java index ecf5efd..89b263e 100644 --- a/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java +++ b/src/test/java/com/kingsrook/qbits/dbperformance/AttributionBoundedReadTest.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.dbperformance.analyze.RuleContext; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; import com.kingsrook.qbits.dbperformance.collect.RollupService; import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer; @@ -48,9 +49,10 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* ** Attribution detail grows by hundreds of thousands of rows a day. On ** 2026-10-16 the home widget loaded a whole day of it into the heap and ran - ** the kof22 web pod out of memory. These tests count every attribution - ** detail record QQQ materializes, so neither the widget nor the daily rollup - ** can go back to reading the table in one unbounded query. + ** the kof22 web pod out of memory, and the hourly analysis filled the worker + ** heap the same way. These tests count every attribution detail record QQQ + ** materializes, so the widget, the analysis and the daily rollup cannot go + ** back to reading the table in one unbounded query. *******************************************************************************/ class AttributionBoundedReadTest { @@ -135,6 +137,29 @@ class AttributionBoundedReadTest + /******************************************************************************* + ** The hourly analysis reads attribution as database sums per unit, table + ** and operation: no detail rows, same totals. + *******************************************************************************/ + @Test + void analysisSumsInTheDatabase() throws Exception + { + Instant now = Instant.now(); + insert(3000, i -> row(now.minusSeconds(60), now, i)); + LOADED.set(0); + RuleContext context = new RuleContext(runtime, runtime.getConfig().getBackendName(), new QRecord(), new QRecord(), now.plusSeconds(1)); + + List rows = context.window("attribution", 1); + + assertThat(LOADED.get()).as("attribution detail rows loaded by the analysis").isZero(); + assertThat(rows).hasSize(6); + assertThat(rows.stream().mapToDouble(row -> RuleContext.number(row, "statements")).sum()).isEqualTo(6000.0); + assertThat(rows.stream().mapToDouble(row -> RuleContext.number(row, "totalMs")).sum()).isEqualTo(1500 * 10.0 + 1500 * 1.0); + assertThat(rows).allSatisfy(row -> assertThat(row.getValueString("operation")).isEqualTo("SELECT")); + } + + + /******************************************************************************* ** Keyset pages never exceed the page size, and every row is seen once. *******************************************************************************/ -- 2.54.0