fix(collect): no stack overflow on huge statements; every failure closes the run

The normalizer folded IN and VALUES lists with nested-repetition regular
expressions, which threw StackOverflowError on kof22's 60 KB statements.
That Error escaped catch(Exception), so runs stayed RUNNING, nothing was
logged and no alert fired. List folding is now a linear token scan, the
collector catches Throwable (rethrowing OutOfMemoryError), logs class and
frame at ERROR, marks FAILED and alerts, and stale RUNNING runs are closed.
Verified against kof22's 4,462 real statement texts (under 1 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes are documented here. This project follows Keep a Changelog a
 
 ## [Unreleased]
 
+### Fixed
+- Collection no longer stops at RUNNING on large production statement sets: list folding in the SQL normalizer is a linear token scan (the nested-repetition regular expressions overflowed the stack on 60 KB statements with long IN/VALUES lists). Any failure, including an `Error`, now closes the run as FAILED with the exception class, logs it at ERROR (class and frame only, never SQL text) and raises the collection alert; runs left RUNNING by a crash are closed as FAILED after three intervals.
+- Only one runtime attributes per JVM: starting attribution on a newer instance (a hot swap, or repeated server starts in tests) stops the previous one, so old instances are never pinned by a forgotten listener and flusher.
+- Each table has its own icon (the app icon is now `storage`), so nav entries are distinguishable and do not repeat icons used elsewhere.
+- Home tables keep at most four columns with plain headers (readable at every width): secondary facts such as calls and mean time, the session's query text, and a unit's kind and top table move into a second line of the first cell.
+
 ### Added
 - QQQ attribution and N+1 detection: a core statement execution listener (QQQ 4.1.0-RC.13) aggregates database time per route, scheduled job, process, table and statement shape in bounded memory, records repeated statement shapes per unit of work, and each pod flushes one batched insert per table every `attribution_flush_seconds`. Hosts without QQQ's application launcher call `DbPerfRuntime.of(instance).startAttribution()` after startup.
 

src/main/java/com/kingsrook/qbits/dbperformance/DbPerfRuntime.java

@@ -42,6 +42,12 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
    private final DbPerfQBitConfig config;
    private AttributionFlusher attributionFlusher;
 
+   /////////////////////////////////////////////////////////////////////////////
+   // statement listeners are JVM-wide: only one runtime attributes at a time //
+   // (a newer instance, e.g. after a hot swap or in tests, replaces the old) //
+   /////////////////////////////////////////////////////////////////////////////
+   private static final AtomicReference<DbPerfRuntime> ACTIVE_ATTRIBUTION = new AtomicReference<>();
+
 
    /*******************************************************************************
     ** Runtime bound to a single host instance.
@@ -149,6 +155,11 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
    {
       if(attributionFlusher == null && Boolean.TRUE.equals(config.getAttributionEnabled()))
       {
+         DbPerfRuntime previous = ACTIVE_ATTRIBUTION.getAndSet(this);
+         if(previous != null && previous != this)
+         {
+            previous.stopAttribution();
+         }
          attributionFlusher = new AttributionFlusher(this);
          attributionFlusher.start();
       }
@@ -160,6 +171,7 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
     *******************************************************************************/
    public synchronized void stopAttribution()
    {
+      ACTIVE_ATTRIBUTION.compareAndSet(this, null);
       if(attributionFlusher != null)
       {
          attributionFlusher.close();

src/main/java/com/kingsrook/qbits/dbperformance/collect/SnapshotCollector.java

@@ -48,6 +48,7 @@ import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
 import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
 import com.kingsrook.qbits.dbperformance.provider.StatementStats;
 import com.kingsrook.qbits.dbperformance.provider.TableStats;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -62,6 +64,8 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat
  *******************************************************************************/
 public class SnapshotCollector
 {
+   private static final QLogger LOG = QLogger.getLogger(SnapshotCollector.class);
+
    private final DbPerfRuntime runtime;
    private final Clock clock;
    private final DbStatsProvider providerOverride;
@@ -104,6 +108,7 @@ public class SnapshotCollector
                throw (new IllegalArgumentException("Backend is not configured for this runtime"));
             }
             QRecord settings = new DbPerfSettingsService(runtime).ensure(backendName);
+            failAbandonedRuns(backendName, settings);
             QRecord inserted = runtime.asSystem(() -> DbPerfStore.insert("db_perf_snapshot", List.of(snapshot)).getFirst());
             snapshot.setValue("id", inserted.getValueInteger("id"));
             runtime.transaction(() ->
@@ -121,9 +126,19 @@ public class SnapshotCollector
                return (null);
             });
          }
-         catch(Exception e)
+         catch(Throwable e)
          {
-            // Never persist a JDBC exception message: it may contain SQL literals.
+            //////////////////////////////////////////////////////////////////////////
+            // Errors too (a StackOverflowError once left runs RUNNING with no log). //
+            // Never persist or log a JDBC exception message: it may contain SQL    //
+            // literals. The class and the throwing frame identify the cause.       //
+            //////////////////////////////////////////////////////////////////////////
+            if(e instanceof OutOfMemoryError outOfMemory)
+            {
+               throw (outOfMemory);
+            }
+            LOG.error("Database performance snapshot failed", logPair("backend", backendName), logPair("snapshotId", snapshot.getValueInteger("id")),
+               logPair("exception", e.getClass().getName()), logPair("at", e.getStackTrace().length == 0 ? null : e.getStackTrace()[0].toString()));
             snapshot.withValue("status", "FAILED").withValue("errorMessage", "Collection failed (" + e.getClass().getSimpleName() + "). Check database connectivity, schema and statistics access.")
                .withValue("durationMs", (int) Math.min(Integer.MAX_VALUE, (System.nanoTime() - started) / 1000000));
             if(snapshot.getValueInteger("id") != null)
@@ -149,6 +164,33 @@ public class SnapshotCollector
    }
 
 
+   /*******************************************************************************
+    ** Runs left RUNNING by a crashed pod or an earlier failure are closed as
+    ** FAILED once they are older than three collection intervals.
+    *******************************************************************************/
+   private void failAbandonedRuns(String backendName, QRecord settings) throws Exception
+   {
+      int     interval = positive(settings, "snapshotIntervalMinutes", 5);
+      Instant cutoff   = clock.instant().minus(Duration.ofMinutes(3L * interval));
+      List<QRecord> abandoned = runtime.asSystem(() -> DbPerfStore.query("db_perf_snapshot", DbPerfStore.eq("backendName", backendName)
+         .withCriteria(new QFilterCriteria("status", QCriteriaOperator.EQUALS, "RUNNING"))
+         .withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.LESS_THAN, cutoff))));
+      if(abandoned.isEmpty())
+      {
+         return;
+      }
+      List<QRecord> updates = abandoned.stream().map(row -> new QRecord().withValue("id", row.getValueInteger("id")).withValue("status", "FAILED")
+         .withValue("errorMessage", "Collection did not finish (the run was interrupted or its pod stopped).")).toList();
+      runtime.asSystem(() ->
+      {
+         DbPerfStore.update("db_perf_snapshot", updates);
+         return (null);
+      });
+      LOG.warn("Closed abandoned database performance snapshots", logPair("backend", backendName), logPair("count", updates.size()));
+   }
+
+
+
    /*******************************************************************************
     ** Statistics read and writes under the ambient system transaction.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfMetaData.java

@@ -23,6 +23,7 @@ package com.kingsrook.qbits.dbperformance.metadata;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qbits.dbperformance.DbPerfPermissions;
 import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
 import com.kingsrook.qbits.dbperformance.model.DbPerfActivitySample;
@@ -669,7 +670,7 @@ public final class DbPerfMetaData
       child(instance, "db_perf_statement", "db_perf_finding", "statementId");
       child(instance, "db_perf_table_stat", "db_perf_finding", "tableStatId");
       child(instance, "db_perf_statement", "db_perf_explain", "statementId");
-      QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel("Database Performance").withIcon(new QIcon("speed"))
+      QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel("Database Performance").withIcon(new QIcon("storage"))
          .withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"))
          .withSection(new QAppSection().withName("data").withLabel("Data").withIcon(new QIcon("table_rows"))
             .withTables(List.of("db_perf_finding", "db_perf_statement", "db_perf_table_stat", "db_perf_index_stat", "db_perf_snapshot", "db_perf_settings", "db_perf_rule")));
@@ -681,12 +682,36 @@ public final class DbPerfMetaData
    }
 
 
+   /*******************************************************************************
+    ** A distinct icon per table, so nav entries are told apart at a glance.
+    *******************************************************************************/
+   static final Map<String, String> TABLE_ICONS = Map.ofEntries(
+      Map.entry("db_perf_finding", "tips_and_updates"),
+      Map.entry("db_perf_statement", "query_stats"),
+      Map.entry("db_perf_table_stat", "grid_on"),
+      Map.entry("db_perf_index_stat", "bookmarks"),
+      Map.entry("db_perf_snapshot", "photo_camera"),
+      Map.entry("db_perf_settings", "settings_suggest"),
+      Map.entry("db_perf_rule", "playlist_add_check"),
+      Map.entry("db_perf_explain", "device_hub"),
+      Map.entry("db_perf_query_snapshot", "timeline"),
+      Map.entry("db_perf_query_daily", "calendar_month"),
+      Map.entry("db_perf_table_snapshot", "stacked_line_chart"),
+      Map.entry("db_perf_table_daily", "date_range"),
+      Map.entry("db_perf_activity_sample", "monitor_heart"),
+      Map.entry("db_perf_connection_sample", "lan"),
+      Map.entry("db_perf_attribution", "hub"),
+      Map.entry("db_perf_attribution_daily", "event_note"),
+      Map.entry("db_perf_repeat_pattern", "repeat"));
+
+
+
    /*******************************************************************************
     ** Common storage, permissions and write guard.
     *******************************************************************************/
    private static QTableMetaData base(DbPerfQBitConfig config, String name, String label, Class<? extends QRecordEntity> entity) throws QException
    {
-      QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withIcon(new QIcon("speed"))
+      QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withIcon(new QIcon(TABLE_ICONS.getOrDefault(name, "table_rows")))
          .withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
          .withPrimaryKeyField("id").withFieldsFromEntity(entity)
          .withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"));

src/main/java/com/kingsrook/qbits/dbperformance/normalize/SqlNormalizer.java

@@ -24,7 +24,10 @@ package com.kingsrook.qbits.dbperformance.normalize;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HexFormat;
+import java.util.List;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -200,12 +203,86 @@ public final class SqlNormalizer
     *******************************************************************************/
    private static String collapseLists(String normalized)
    {
-      normalized = normalized.replaceAll("\\bin \\( \\?(?: , \\?)* \\)", "in ( ? )");
-      normalized = normalized.replaceAll("\\bvalues (\\( \\?(?: , \\?)* \\))(?: , \\( \\?(?: , \\?)* \\))+", "values $1");
-      return (normalized);
+      ///////////////////////////////////////////////////////////////////////////
+      // a linear scan over the space-separated tokens: regular expressions    //
+      // with nested repetition overflow the stack on very long IN and VALUES  //
+      // lists (seen with 60 KB statements in production)                      //
+      ///////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////////////////////
+      // segments between quoted identifiers keep their edge spaces          //
+      ////////////////////////////////////////////////////////////////////////
+      String trimmed = normalized.strip();
+      if(trimmed.isEmpty())
+      {
+         return (normalized);
+      }
+      String       leading  = normalized.substring(0, normalized.indexOf(trimmed.charAt(0)));
+      String       trailing = normalized.substring(leading.length() + trimmed.length());
+      String[]     tokens   = trimmed.split(" ");
+      List<String> out    = new ArrayList<>(tokens.length);
+      int          i      = 0;
+      while(i < tokens.length)
+      {
+         String token = tokens[i];
+         if("in".equals(token))
+         {
+            int end = placeholderGroupEnd(tokens, i + 1);
+            if(end > 0)
+            {
+               out.addAll(List.of("in", "(", "?", ")"));
+               i = end;
+               continue;
+            }
+         }
+         if("values".equals(token))
+         {
+            int first = placeholderGroupEnd(tokens, i + 1);
+            if(first > 0)
+            {
+               int end = first;
+               while(end + 1 < tokens.length && ",".equals(tokens[end]))
+               {
+                  int next = placeholderGroupEnd(tokens, end + 1);
+                  if(next < 0)
+                  {
+                     break;
+                  }
+                  end = next;
+               }
+               out.add("values");
+               out.addAll(Arrays.asList(tokens).subList(i + 1, first));
+               i = end;
+               continue;
+            }
+         }
+         out.add(token);
+         i++;
+      }
+      return (leading + String.join(" ", out) + trailing);
    }
 
 
+
+   /*******************************************************************************
+    ** If tokens[start] opens "( ? , ? ... )" made only of placeholders, the index
+    ** just after its ")"; otherwise -1.
+    *******************************************************************************/
+   private static int placeholderGroupEnd(String[] tokens, int start)
+   {
+      if(start + 2 >= tokens.length || !"(".equals(tokens[start]) || !"?".equals(tokens[start + 1]))
+      {
+         return (-1);
+      }
+      int i = start + 2;
+      while(i + 1 < tokens.length && ",".equals(tokens[i]) && "?".equals(tokens[i + 1]))
+      {
+         i += 2;
+      }
+      return (i < tokens.length && ")".equals(tokens[i]) ? i + 1 : -1);
+   }
+
+
+
    /*******************************************************************************
     ** Consume a quote with SQL doubled-quote and optional escape handling.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java

@@ -29,6 +29,7 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
@@ -55,11 +56,11 @@ import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockLink;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextStyles;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
-import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.fields;
 import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.map;
 import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.number;
 
@@ -169,10 +170,11 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
       List<Map<String, Object>> rows = new ArrayList<>();
       for(Map<String, Object> statement : data.statements(100, null))
       {
-         rows.add(map("Query", link((String) statement.get("queryText"), (String) statement.get("historyPath")), "Calls", statement.get("calls24h"),
-            "Mean ms", statement.get("meanMs24h"), "Total ms", statement.get("totalMs24h"), "% DB time", statement.get("shareOfDbTime24h"), "Hit ratio", statement.get("hitRatio")));
+         rows.add(map("Query", stack((String) statement.get("queryText"), (String) statement.get("historyPath"),
+               format(statement.get("calls24h")) + " calls, mean " + format(statement.get("meanMs24h")) + " ms"),
+            "Total ms (24h)", statement.get("totalMs24h"), "Share of DB time %", statement.get("shareOfDbTime24h"), "Cache hit ratio", statement.get("hitRatio")));
       }
-      TableData table = table("Top queries appear after two snapshots establish interval deltas.", rows, "Query", "Calls", "Mean ms", "Total ms", "% DB time", "Hit ratio");
+      TableData table = table("Top queries appear after two snapshots establish interval deltas.", rows, "Query", "Total ms (24h)", "Share of DB time %", "Cache hit ratio");
       table.getColumns().getFirst().setType("composite");
       return (table);
    }
@@ -210,7 +212,20 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
       List<Map<String, Object>> rows = sessions.stream().filter(row -> !((List<?>) row.get("blockedBy")).isEmpty() || ((Number) row.get("blockingCount")).intValue() > 0
          || "active".equals(row.get("state")) && numeric(row.get("querySeconds")) >= longRunning
          || Objects.toString(row.get("state"), "").startsWith("idle in transaction") && numeric(row.get("xactSeconds")) >= idle).toList();
-      return (table("No long-running or blocked sessions right now", rows, "pid", "user", "application", "state", "querySeconds", "xactSeconds", "blockedBy", "blockingCount", "normalizedText"));
+      List<Map<String, Object>> cells = new ArrayList<>();
+      for(Map<String, Object> row : rows)
+      {
+         boolean blocked = !((List<?>) row.get("blockedBy")).isEmpty();
+         int     blocking = ((Number) row.get("blockingCount")).intValue();
+         cells.add(map("Session", stack("pid " + row.get("pid") + ", " + Objects.toString(row.get("user"), "?") + ", " + Objects.toString(row.get("application"), ""), null,
+               Objects.toString(row.get("normalizedText"), "")),
+            "State", Objects.toString(row.get("state"), "") + (row.get("waitEvent") == null ? "" : " (" + row.get("waitEvent") + ")"),
+            "Query / transaction s", format(row.get("querySeconds")) + " / " + format(row.get("xactSeconds")),
+            "Blocking", blocked ? "blocked by " + row.get("blockedBy") : (blocking > 0 ? "blocks " + blocking : "")));
+      }
+      TableData table = table("No long-running or blocked sessions right now", cells, "Session", "State", "Query / transaction s", "Blocking");
+      table.getColumns().getFirst().setType("composite");
+      return (table);
    }
 
 
@@ -277,7 +292,7 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
       for(QRecord row : records)
       {
          String key = row.getValueString("unitKind") + "\n" + row.getValueString("unitName");
-         Map<String, Object> group = grouped.computeIfAbsent(key, ignored -> map("Kind", row.getValueString("unitKind"), "Unit", row.getValueString("unitName"), "Statements", 0.0, "Total ms", 0.0));
+         Map<String, Object> group = grouped.computeIfAbsent(key, ignored -> map("Kind", row.getValueString("unitKind"), "Name", row.getValueString("unitName"), "Statements", 0.0, "Total ms", 0.0));
          group.put("Statements", numeric(group.get("Statements")) + number(row, "statements"));
          group.put("Total ms", numeric(group.get("Total ms")) + number(row, "totalMs"));
          tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getValueString("tableName"), "Unknown"), number(row, "totalMs"), Double::sum);
@@ -287,11 +302,19 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
          group.put("Share %", total == 0 ? 0 : 100 * numeric(group.get("Total ms")) / total);
          group.put("Top table", tableTimes.get(key).entrySet().stream().max(Map.Entry.comparingByValue()).orElseThrow().getKey());
       });
-      List<Map<String, Object>> units = grouped.values().stream().sorted(Comparator.comparingDouble((Map<String, Object> row) -> numeric(row.get("Total ms"))).reversed()).toList();
+      List<Map<String, Object>> units = grouped.values().stream().sorted(Comparator.comparingDouble((Map<String, Object> row) -> numeric(row.get("Total ms"))).reversed())
+         .map(row -> map("Unit of work", stack(Objects.toString(row.get("Name"), ""), null, kindLabel(row.get("Kind")) + ", top table " + row.get("Top table")),
+            "Statements", row.get("Statements"), "Total ms (24h)", row.get("Total ms"), "Share %", row.get("Share %"))).toList();
       List<Map<String, Object>> repeats = DbPerfStore.query("db_perf_repeat_pattern", since("windowEnd").withOrderBy(new QFilterOrderBy("maxRepeats", false)).withLimit(100)).stream()
-         .map(row -> fields(row, "unitKind", "unitName", "fingerprint", "unitsAffected", "maxRepeats", "totalRepeats")).toList();
-      return (new MultiTableData(List.of(table("Attribution starts after the next flush", units, "Kind", "Unit", "Statements", "Total ms", "Share %", "Top table"),
-         table("Repeated statements appear after the next attribution flush when the repetition threshold is reached.", repeats, "unitKind", "unitName", "fingerprint", "unitsAffected", "maxRepeats", "totalRepeats"))));
+         .map(row -> map("Unit of work", stack(Objects.toString(row.getValueString("unitName"), ""), null,
+               kindLabel(row.getValueString("unitKind")) + ": " + truncate(row.getValueString("sqlText"), 160)),
+            "Most repeats in one unit", row.getValue("maxRepeats"), "Units affected", row.getValue("unitsAffected"), "Total repeats", row.getValue("totalRepeats"))).toList();
+      TableData unitTable = table("Attribution starts after the next flush", units, "Unit of work", "Statements", "Total ms (24h)", "Share %");
+      unitTable.getColumns().getFirst().setType("composite");
+      TableData repeatTable = table("Repeated statements appear after the next attribution flush when the repetition threshold is reached.", repeats,
+         "Unit of work", "Most repeats in one unit", "Units affected", "Total repeats");
+      repeatTable.getColumns().getFirst().setType("composite");
+      return (new MultiTableData(List.of(unitTable, repeatTable)));
    }
 
 
@@ -303,10 +326,14 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
    {
       List<QRecord> scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100));
       List<QRecord> growing = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("growthBytes7d", false)).withLimit(100));
-      return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream().map(row -> fields(row, "schemaName", "tableName", "seqScan", "idxScan", "seqScanShare")).toList(),
-         "schemaName", "tableName", "seqScan", "idxScan", "seqScanShare"),
-         table("Growth appears after table history has accumulated.", growing.stream().map(row -> fields(row, "schemaName", "tableName", "totalBytes", "growthBytes7d")).toList(),
-            "schemaName", "tableName", "totalBytes", "growthBytes7d"))));
+      return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream()
+            .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Sequential scans", row.getValue("seqScan"),
+               "Index scans", row.getValue("idxScan"), "Sequential share %", row.getValue("seqScanShare"))).toList(),
+            "Table", "Sequential scans", "Index scans", "Sequential share %"),
+         table("Growth appears after table history has accumulated.", growing.stream()
+            .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")),
+               "Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(),
+            "Table", "Size MB", "Growth MB (7 days)"))));
    }
 
 
@@ -333,6 +360,79 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
 
 
 
+   /*******************************************************************************
+    ** A stacked cell: the main text (a record link when a path is given) over a
+    ** smaller secondary line, so tables keep at most four columns.
+    *******************************************************************************/
+   private static CompositeWidgetData stack(String main, String path, String secondary)
+   {
+      TextBlockData first = new TextBlockData().withValues(new TextValues(truncate(main, 300)));
+      if(path != null)
+      {
+         first.withLink(new BlockLink(path));
+      }
+      CompositeWidgetData cell = new CompositeWidgetData().withLayout(CompositeWidgetData.Layout.FLEX_COLUMN).withBlock(first);
+      if(secondary != null && !secondary.isBlank())
+      {
+         cell.withBlock(new TextBlockData().withValues(new TextValues(truncate(secondary, 300))).withStyles(new TextStyles().withColor("MUTED")));
+      }
+      return (cell);
+   }
+
+
+
+   /*******************************************************************************
+    ** Plain label for a unit-of-work kind.
+    *******************************************************************************/
+   private static String kindLabel(Object kind)
+   {
+      return (switch(Objects.toString(kind, ""))
+      {
+         case "HTTP_REQUEST" -> "Request";
+         case "SCHEDULED_JOB" -> "Scheduled job";
+         case "PROCESS" -> "Process";
+         default -> "Other";
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Text capped with an ellipsis.
+    *******************************************************************************/
+   private static String truncate(String value, int max)
+   {
+      String text = value == null ? "" : value;
+      return (text.length() <= max ? text : text.substring(0, max - 1) + "…");
+   }
+
+
+
+   /*******************************************************************************
+    ** A number for a secondary line: whole numbers without decimals.
+    *******************************************************************************/
+   private static String format(Object value)
+   {
+      if(value == null)
+      {
+         return ("0");
+      }
+      double number = ((Number) value).doubleValue();
+      return (number == Math.rint(number) ? String.valueOf((long) number) : String.format(Locale.ROOT, "%.1f", number));
+   }
+
+
+
+   /*******************************************************************************
+    ** Bytes as whole megabytes.
+    *******************************************************************************/
+   private static Long mb(Object bytes)
+   {
+      return (bytes == null ? null : Math.round(((Number) bytes).doubleValue() / 1048576));
+   }
+
+
+
    /*******************************************************************************
     ** Trailing-day filter for time-window records.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/dbperformance/DbPerfFixtureProvider.java

@@ -47,6 +47,7 @@ public class DbPerfFixtureProvider extends UnsupportedStatsProvider
    public Instant reset = START.minusSeconds(3600);
    public boolean limited;
    public boolean fail;
+   public Error   error;
 
 
    /*******************************************************************************
@@ -85,6 +86,10 @@ public class DbPerfFixtureProvider extends UnsupportedStatsProvider
    @Override
    public List<StatementStats> readStatements(Connection connection)
    {
+      if(error != null)
+      {
+         throw (error);
+      }
       return (List.of(new StatementStats(1, "app", true, "select * from public.accounts where id=$1", calls, BigDecimal.valueOf(calls * 10), BigDecimal.TEN, BigDecimal.valueOf(15), calls, calls * 9, calls, 0, reset, reset),
          new StatementStats(2, "app", true, "select * from public.accounts where name='private'", calls, BigDecimal.valueOf(calls * 5), BigDecimal.valueOf(5), BigDecimal.TEN, calls, calls, calls, 2, reset, reset)));
    }

src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java

@@ -129,7 +129,7 @@ class DbPerfPresentationTest
       assertThat((Map<String, Integer>) overview.get("findingsBySeverity")).containsEntry("HIGH", 1);
       TableData queries = new DbPerfWidgetRenderer().topQueries(data);
       assertThat(queries.getRows()).hasSize(1);
-      assertThat((BigDecimal) queries.getRows().getFirst().get("% DB time")).isEqualByComparingTo("25");
+      assertThat((BigDecimal) queries.getRows().getFirst().get("Share of DB time %")).isEqualByComparingTo("25");
       assertThat(JsonUtils.toJson(queries)).contains("/app/dbPerformance/db_perf_statement/", "composite").doesNotContain("private-value");
       ChartData chart = new DbPerfWidgetRenderer().databaseTime(data);
       assertThat(chart.getChartData().getDatasets()).hasSize(2);
@@ -137,7 +137,7 @@ class DbPerfPresentationTest
       assertThat(new DbPerfWidgetRenderer().findings(data).getRows()).hasSize(1);
       assertThat(new DbPerfWidgetRenderer().tables(data).getTableDataList().getFirst().getRows()).hasSize(1);
       MultiTableData attribution = new DbPerfWidgetRenderer().attribution();
-      assertThat(attribution.getTableDataList().getFirst().getRows().getFirst()).containsEntry("Top table", "patient");
+      assertThat(JsonUtils.toJson(attribution.getTableDataList().getFirst().getRows().getFirst())).contains("top table patient");
       assertThat(attribution.getTableDataList().get(1).getRows()).hasSize(1);
       assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().connectionStatistics(data))).contains("Deadlocks", "Cache hit");
       JsonInput lookup = new JsonInput();
@@ -194,7 +194,7 @@ class DbPerfPresentationTest
       QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW));
       TableData table = new DbPerfWidgetRenderer().liveActivity(fixtureData);
       assertThat(table.getRows()).hasSize(2);
-      assertThat(JsonUtils.toJson(table)).doesNotContain("private-value").contains("normalizedText", "blockingCount");
+      assertThat(JsonUtils.toJson(table)).doesNotContain("private-value").contains("select * from patient where name = ?", "blocks 1", "blocked by");
       QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.VIEW_LIVE_TEXT));
       assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().liveActivity(fixtureData))).doesNotContain("private-value");
    }

src/test/java/com/kingsrook/qbits/dbperformance/SnapshotCollectorTest.java

@@ -123,6 +123,37 @@ class SnapshotCollectorTest
    }
 
 
+   /*******************************************************************************
+    ** An Error (not just an Exception) still closes the run as FAILED, and runs
+    ** left RUNNING by an earlier crash are closed once they are stale.
+    *******************************************************************************/
+   @Test
+   void errorsAndAbandonedRuns() throws Exception
+   {
+      QInstance             instance = DbPerfTestSupport.instance();
+      DbPerfRuntime         runtime  = DbPerfRuntime.of(instance);
+      DbPerfFixtureProvider provider = new DbPerfFixtureProvider();
+      runtime.asSystem(() -> DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName())
+         .withValue("capturedAt", DbPerfFixtureProvider.START.minusSeconds(3600)).withValue("status", "RUNNING"))));
+
+      provider.error = new StackOverflowError();
+      QRecord failed = collect(runtime, provider, 0, true);
+      assertThat(failed.getValueString("status")).isEqualTo("FAILED");
+      assertThat(failed.getValueString("errorMessage")).contains("StackOverflowError");
+      runtime.asSystem(() ->
+      {
+         List<QRecord> rows = DbPerfStore.query("db_perf_snapshot", new QQueryFilter());
+         assertThat(rows).hasSize(2).allSatisfy(row -> assertThat(row.getValueString("status")).isEqualTo("FAILED"));
+         assertThat(rows).anySatisfy(row -> assertThat(row.getValueString("errorMessage")).contains("did not finish"));
+         return (null);
+      });
+
+      provider.error = null;
+      assertThat(collect(runtime, provider, 300, true).getValueString("status")).isEqualTo("OK");
+   }
+
+
+
    /*******************************************************************************
     ** H2 remains usable and explicitly unsupported by the production selector.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/dbperformance/SqlNormalizerTest.java

@@ -86,6 +86,14 @@ class SqlNormalizerTest
       assertThat(SqlNormalizer.mainTable("select 1")).isNull();
       assertThat(SqlNormalizer.normalize("select \"in ( ? , ? )\" from t")).isEqualTo("select \"in ( ? , ? )\" from t");
       assertThat(SqlNormalizer.normalize(null)).isEmpty();
+      ///////////////////////////////////////////////////////////////////////
+      // huge lists (production had 60 KB statements) must not overflow    //
+      ///////////////////////////////////////////////////////////////////////
+      String hugeIn = "select * from t where id in (" + "$1, ".repeat(20000) + "$1)";
+      assertThat(SqlNormalizer.normalize(hugeIn)).isEqualTo("select * from t where id in ( ? )");
+      String hugeValues = "insert into t (a, b) values " + "($1, $2), ".repeat(20000) + "($1, $2)";
+      assertThat(SqlNormalizer.normalize(hugeValues)).isEqualTo("insert into t ( a , b ) values ( ? , ? )");
+      assertThat(SqlNormalizer.normalize("select * from t where id in (1, x)")).isEqualTo("select * from t where id in ( ? , x )");
       assertThat(SqlNormalizer.normalize(" ")).isEmpty();
       assertThat(SqlNormalizer.normalize("select " + "field, ".repeat(5000))).hasSize(10000);
    }

src/test/java/com/kingsrook/qbits/dbperformance/attribution/AttributionTest.java

@@ -569,6 +569,25 @@ class AttributionTest
    }
 
 
+   /*******************************************************************************
+    ** Only one runtime attributes per JVM: starting another stops the first, so
+    ** repeated server starts (tests, hot swaps) never pile up listeners.
+    *******************************************************************************/
+   @Test
+   void oneActiveRuntime() throws Exception
+   {
+      StatementExecutionListeners.unregister(listener);
+      runtime.startAttribution();
+      DbPerfRuntime other = new DbPerfRuntime(runtime.getInstance(), runtime.getConfig());
+      other.startAttribution();
+      runtime.stopAttribution();
+      assertThat(StatementExecutionListeners.isActive()).isTrue();
+      other.stopAttribution();
+      assertThat(StatementExecutionListeners.isActive()).isFalse();
+   }
+
+
+
    /*******************************************************************************
     ** Deliver a statement with exact counter values.
     *******************************************************************************/