Compare commits

...
Author SHA1 Message Date
james.maes 7c5e7a9320 Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#39) from deps/align-gitea into develop
build / maven (push) Successful in 5m20s
maven / build (push) Successful in 5m20s
2026-11-07 18:37:05 +00:00
james.maesandClaude Opus 5.5 f0ecacc3e9 test: regenerate OpenAPI contract for QQQ 4.1.0-RC.26
build / maven (push) Successful in 4m8s
maven / build (push) Successful in 4m8s
QQQ RC.26 adds the shared BasicErrorResponseV1 schema to extension models; regenerate the committed fragment and allow that core schema.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 12:30:25 -06:00
james.maesandClaude Opus 5.5 725446709b build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)
build / maven (push) Canceled after 0s
maven / build (push) Canceled after 0s
pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.13 -> 4.1.0-RC.26
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 17:59:16 +00:00
james.maes d578b5180a Merge pull request 'ci: pin shared workflow to v1' (#38) from ci/pin-v1-develop into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (push) Successful in 16s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 12s
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Successful in 12s
build / maven (push) Successful in 4m53s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 4m53s
2026-11-06 16:06:22 +00:00
james.maesandClaude Opus 5.5 cf106d4301 ci: pin shared workflow to v1
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 7s
build / maven (push) Successful in 5m7s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 5m7s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 14:08:36 +00:00
james.maes a26a175aa8 Merge pull request 'ci: use shared ci-workflows Maven workflow' (#37) from ci/ci-workflows into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 7s
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Successful in 5s
build / maven (push) Successful in 3m50s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow d20a578c7d818bc7ca32a2ac2c675da4e4f27385] (push) Successful in 10s
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 3m50s
2026-11-05 23:16:16 +00:00
CI aff84d5add ci: rerun shared workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Failing after 3s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 2s
build / maven (push) Successful in 3m43s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Failing after 2s
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 3m44s
2026-11-05 16:24:46 -06:00
james.maesandClaude Opus 5.5 171b0d3fad ci: use shared ci-workflows Maven workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Successful in 8s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
build / maven (push) Failing after 3m54s
QRun-IO/ci-policy: ci / ci-result (pull_request) Canceled after 0s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Failing after 3m55s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 3m42s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Failing after 1s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Canceled after 12m41s
Replace the platform-ci release callers with one caller of
KofTwentyTwo/ci-workflows maven.yaml (qqq-orb port).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:04:26 +00:00
forge-recovery b70c234c79 ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / ci-result (push) Successful in 4s
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 25m25s
qrun-release / postpublish (push) Failing after 10m34s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 14s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 13s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 6s
Shared Java Maven CI with Docker / verify (push) Successful in 3m56s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 22s
Shared Java Maven CI with Docker / gate (push) Successful in 20s
qrun-release / release (push) Successful in 25m48s
Shared QRun source version / version (push) Failing after 10m34s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 3m18s
2026-11-02 22:15:48 +00:00
forge-recovery ef58b9d8ad ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 40s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 12s
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 11m7s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 7s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 3m30s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 20s
2026-11-02 21:33:08 +00:00
forge-recoveryandCodex 5d82358d2b ci: enroll shared Gitea release-candidate lifecycle
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 4s
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 12s
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 2m49s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 17s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 59m32s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 20:16:11 +00:00
3468453335 ci: add the isolated postpublish job to the QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 11s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
Shared Java Maven CI with Docker / verify (push) Successful in 3m2s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
qrun-release / postpublish (push) Skipped
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 11s
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 2m22s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 9s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 2h51m11s
Shared Java Maven CI with Docker / gate (push) Successful in 2m25s
qrun-release / release (push) Successful in 2h52m39s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 15:48:51 +00:00
612f474e91 ci: move the QRun release caller to the thin v2 contract
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-9 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 5s
Shared Java Maven CI with Docker / secrets (push) Successful in 22s
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Canceled after 1h7m43s
Shared Java Maven CI with Docker / verify (push) Canceled after 0s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 21s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Canceled after 0s
Shared Java Maven CI with Docker / gate (push) Canceled after 0s
qrun-release / release (push) Canceled after 1h7m51s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 06:17:47 +00:00
forge-recoveryandCodex f005c40083 ci: add the identical QRun release caller
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Successful in 52m15s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 3s
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-7 (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 2m39s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 10s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 16s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 04:10:32 +00:00
james.maesandClaude Fable 5.1 d46b6ca222 feat(provider): H2 statistics provider and per-area capabilities (#34)
QRun-IO/ci-policy: ci / gradle-0 (push) Skipped
QRun-IO/ci-policy: ci / node-3 (push) Skipped
QBit CodeQL / Analyze branch source (push) Canceled after 0s
CodeQL / analyze (push) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 11s
QRun-IO/ci-policy: ci / java-docker-1 (push) Successful in 40m46s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 4s
QRun-IO/ci-policy: ci / source-config-5 (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / java-maven-2 (push) Skipped
QRun-IO/ci-policy: ci / node-4 (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 3m47s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 9s
H2StatsProvider over INFORMATION_SCHEMA (statements via QUERY_STATISTICS, sessions, blocking, connections, indexes, size, settings, approximated table rows and sizes, text plans); per-area provider capabilities with rules skipped where a provider lacks the data; a vendor registry with a seam for MySQL/MariaDB; provider-specific setup and health checks; an Enable statement statistics action; and the H2 plan guard (single SELECT/WITH only, side-effecting functions refused, EXPLAIN ANALYZE only for non-admin users, read-only connection, always rolled back).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 06:14:44 -06:00
james.maesandCodex 4fd2c30b07 ci: adopt managed orb 0.8 patch track (#28)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 15:26:36 -05:00
james.maesandCodex 65e63551f9 ci: adopt shared orb 0.7.18 (#27)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:43:20 -05:00
james.maesandCodex ccc0cc2842 ci: adopt shared orb 0.7.17 (#26)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:15:18 -05:00
james.maesandCodex 9178646423 ci: centralize branch verification and source scanning (#23)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 12:17:56 -05:00
james.maesandCodex 681598be57 ci: adopt central orb 0.7.12 and stable gates (#21)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 08:56:36 -05:00
james.maesandCodex ea9036509c ci: centralize rc release completion in qqq orb (#20)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 02:25:42 -05:00
james.maesandCodex 2a2ba58b06 ci: adopt shared orb 0.7.11 (#19)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 00:57:50 -05:00
james.maes 61e9c22563 Merge pull request #18 from QRun-IO/feature/central-ci
ci: adopt shared QQQ orb standard
2026-10-20 18:57:48 -05:00
james.maesandCodex 65922dd413 ci: adopt verified central QQQ orb 0.7.3
Co-Authored-By: Codex <noreply@openai.com>
2026-10-20 18:52:28 -05:00
james.maes bdaf35fc3e Merge pull request #17 from QRun-IO/feature/bounded-spike-history
fix(alerts): page spike history instead of holding 7 days of query snapshots
2026-10-16 15:54:56 -05:00
james.maesandClaude Opus 5.5 c49b479620 fix(alerts): page spike history instead of holding 7 days of query snapshots
SnapshotEvaluator.spikes loaded RuleContext.window("query_snapshot", 7)
as QRecords on every snapshot evaluation (every collection interval on
the worker). kof22 writes ~33,000 query snapshots a day, so by day 7 that
is ~230,000 records per evaluation.

history() now reads the window in BATCH_SIZE keyset pages (only OK and
PARTIAL snapshots, as before) and keeps per statement only the current
interval time, the latest capture time and earlier interval times in a
primitive double array. Median, twelve-sample baseline and the one-hour
absence rule are unchanged (SnapshotEvaluatorTest still passes).

SpikeHistoryBoundedTest counts every query snapshot QQQ materializes:
2,500 samples never exceed one 1,000-row page, failed snapshots and
samples outside the window are excluded, the current interval is found.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 15:24:44 -05:00
james.maes de991509ca Merge pull request #16 from QRun-IO/feature/bounded-attribution-reads
fix(attribution): sum attribution in the database, page the daily rollup (prod OOM)
2026-10-16 15:15:33 -05:00
james.maesandClaude Opus 5.5 fd0116fbd0 fix(analyze): read attribution as database sums in the hourly analysis
dbPerfAnalyze (hourly, on the worker) loaded the full day of
db_perf_attribution through RuleContext.window and cached it for the
rules: ~570,000 rows. kof22 worker working set jumped from ~660 MiB to
~1.84 GiB at every :07-:08 run on 2026-10-16 (2 Gi limit, 1.5 GB heap).

RuleContext.window("attribution", days) now returns AggregateAction sums
(units, statements, rowsCount, errors, totalMs; max maxMs) grouped by
unitKind, unitName, tableName, operation: the only fields the rules
(OVER_AUDITED, HOT_JOB) read. AttributionBoundedReadTest proves no detail
rows are loaded and the totals are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 15:04:23 -05:00
james.maesandClaude Opus 5.5 744ff04ade fix(attribution): sum attribution in the database, page the daily rollup
The Attribution home widget queried every db_perf_attribution row of the
last 24 hours into QRecords and summed them in Java. kof22 writes about
30,000 attribution rows an hour, so one render loaded ~570,000 rows
(540 MB in PostgreSQL). On 2026-10-16 a single admin request logged
"Large number of records in QueryOutputList" at 50k, 100k and 250k rows
and the web pod died with OutOfMemoryError: Java heap space about 50 s
later, after slowing every other request with GC thrash.

- DbPerfWidgetRenderer.attribution() now uses AggregateAction: SUM of
  statements and totalMs grouped by unitKind, unitName, tableName,
  ordered by time and capped at 5,000 groups (prod has ~1,000).
- RollupService reads the day's attribution detail in 1,000-row keyset
  pages through the new DbPerfStore.forEachPage, so the nightly rollup
  on the worker is bounded by distinct keys instead of the day's rows.
- AttributionBoundedReadTest counts every detail record QQQ materializes
  (post-query customizer): the widget loads none, the rollup never holds
  more than BATCH_SIZE at once, and sums stay correct across pages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 14:34:58 -05:00
james.maes fdcad364f1 Merge pull request #10 from QRun-IO/ci/release-rc-secrets-inherit
ci(release-rc): pass secrets so the shared workflow can sign the RC tag
2026-10-16 08:04:25 -05:00
james.maes 7c71e384c9 Merge pull request #12 from QRun-IO/fix/collector-failures
fix(collect): no stack overflow on huge statements; every failure closes the run as FAILED
2026-10-15 17:43:04 -05:00
james.maesandClaude Opus 5.5 b882e3d0f0 fix(collect): no stack overflow on huge statements; every failure closes the run
The normalizer folded IN and VALUES lists with nested-repetition regular
expressions, which threw StackOverflowError on kof22's 60 KB statements.
That Error escaped catch(Exception), so runs stayed RUNNING, nothing was
logged and no alert fired. List folding is now a linear token scan, the
collector catches Throwable (rethrowing OutOfMemoryError), logs class and
frame at ERROR, marks FAILED and alerts, and stale RUNNING runs are closed.
Verified against kof22's 4,462 real statement texts (under 1 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 17:40:01 -05:00
james.maes bb7e852a35 Merge pull request #11 from QRun-IO/fix/single-attribution-runtime
fix(attribution): one active attribution runtime per JVM
2026-10-15 17:09:55 -05:00
james.maesandClaude Opus 5.5 8a34ac4404 ci(release-rc): pass secrets to the shared workflow so it can sign the RC tag
QRun-IO/.github#2 adds a tag-and-release job that signs vX with the
QRUN_TAG_SIGNING_KEY secret after Maven Central confirms the artifact.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 16:19:54 -05:00
james.maesandClaude Opus 5.5 63d4c9669f fix(attribution): one active attribution runtime per JVM
Statement listeners are JVM-wide. Starting attribution on another
instance now stops the previous runtime's listener and flusher, so
repeated server starts (tests, hot swaps) never pin old instances.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 16:11:08 -05:00
james.maes 1663f48073 Merge pull request #9 from QRun-IO/fix/table-icons
fix(nav): distinct icon per table; cuts 0.1.0-RC.3
2026-10-15 14:15:35 -05:00
james.maesandClaude Opus 5.5 0b517d7e16 fix(nav): a distinct icon per table and a storage app icon
Every db_perf_ table used the speed icon, which repeats in the nav and
clashes with web performance (kof22's nav icon uniqueness check).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 14:12:56 -05:00
james.maes 1bc250c1b8 Merge pull request #8 from QRun-IO/fix/widget-columns
fix(home): at most four columns per widget table; cuts 0.1.0-RC.2
2026-10-15 13:25:03 -05:00
james.maesandClaude Opus 5.5 c9652fd541 fix(home): at most four columns per widget table, plain headers
Top queries, live activity, attribution, repeated statements and tables
fold secondary facts into a muted second line of the first cell, so the
home stays readable at every width (kof22's widget readability guard).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 13:22:12 -05:00
james.maes 8446ab1f0b Merge pull request #7 from QRun-IO/feature/attribution
feat: QQQ attribution and N+1 detection (QQQ 4.1.0-RC.13); cuts 0.1.0-RC.1
2026-10-15 12:55:23 -05:00
james.maesandClaude Opus 5.5 fe41490559 Merge develop (rule seed text) into feature/attribution; number the sample_rate changeset 34
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:52:02 -05:00
james.maesandClaude Opus 5.5 34db3cdff6 Merge develop into feature/attribution; help for sampleRate, README and CHANGELOG
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:21:26 -05:00
james.maes d61684b15e Merge pull request #6 from QRun-IO/fix/rule-seed-text
fix(rules): helpful rule descriptions and threshold labels; spec alert names
2026-10-15 12:18:59 -05:00
james.maesandClaude Opus 5.5 f0b544b074 fix(rules): helpful rule labels, descriptions and threshold labels; spec alert type names
Seeded suggestion rules get plain descriptions of what each finds and
what to do, and threshold labels without stray quotes. The spec now
names the alert types dbperf.* (core rejects underscores).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:15:51 -05:00
james.maes 890df96bc1 Merge pull request #5 from QRun-IO/feature/plans-and-home
feat: plan viewer, UI API extension, app home, help and docs
2026-10-15 12:12:51 -05:00
james.maesandClaude Opus 5.5 a571fe4c3a Merge develop (findings, alerts, health) into feature/plans-and-home; help for finding actions
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:09:57 -05:00
james.maesandClaude Opus 5.5 f8d25d1f7c feat(ui): plan viewer, db-performance UI API extension, app home widgets, help bundle and docs
Query plan record action and POST explain: plain or generic EXPLAIN,
and EXPLAIN (ANALYZE, BUFFERS) for single SELECTs in a rolled-back
READ ONLY transaction with statement_timeout, behind explain_analyze,
always audited, ANALYZE plans and parameters never stored. UI API
extension /qqq/v1/qbit/db-performance (overview, activity with live
text only for view_live_text, explain, statements/lookup) with a
committed OpenAPI fragment and contract test. Eight full-width typed
home widgets. Data-driven help bundle with a coverage test. README,
CHANGELOG and UPGRADE.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:06:37 -05:00
james.maes 1f14317e7e Merge pull request #4 from QRun-IO/feature/findings
feat: suggestions engine, finding actions, alerts and health
2026-10-15 11:54:19 -05:00
james.maesandClaude Opus 5.5 564ebf8d97 feat(attribution): QQQ statement attribution and N+1 detection
Registers a core StatementExecutionListener (QQQ 4.1.0-RC.13) that
aggregates statements per unit of work (route, scheduled job, process),
table, operation and fingerprint in bounded memory, detects repeated
statement shapes per unit, and flushes each window with one batched
insert per table from a daemon thread that never measures itself.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 11:44:26 -05:00
james.maes 4dde72733b Merge pull request #3 from QRun-IO/feature/foundation
feat: foundation (schema, Postgres collector, delta snapshots, rollup and retention)
2026-10-15 11:26:48 -05:00
81 changed files with 10165 additions and 251 deletions

No files matched your search

+11 -57
View File
@@ -1,60 +1,14 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
qqq-orb: kingsrook/qqq-orb@0.8
workflows:
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
+7
View File
@@ -0,0 +1,7 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
+23
View File
@@ -0,0 +1,23 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+62 -13
View File
@@ -1,25 +1,74 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
on:
'on':
workflow_dispatch:
inputs:
dry_run:
description: Print the plan without merging or creating branches
description: Plan a cut without changing release state
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types: [closed]
branches: [develop]
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
+16
View File
@@ -0,0 +1,16 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
+3 -3
View File
@@ -4,7 +4,7 @@ This file provides guidance to Codex when working with code in this repository.
## Project Overview
**QBit DB Performance**: Database performance for QQQ apps: slow and expensive queries from pg_stat_statements with delta snapshots, table scan and index analysis, live activity and blocking chains, connection capacity, maintenance, QQQ query attribution with N+1 detection, and ranked suggestions, all as standard QQQ tables. Design: docs/SPEC.md in this repository. Implementation in progress.
**QBit DB Performance**: Database performance for QQQ apps in standard secured tables: PostgreSQL statement deltas, table and index analysis, live activity and blocking, connection capacity, audited read-only query plans, typed full-width dashboards, a versioned UI API and data-driven help. Collection and presentation are implemented; attribution, findings, alerts and health runtime integrations are developed separately. Design: docs/SPEC.md.
## Build Commands
@@ -21,7 +21,7 @@ This file provides guidance to Codex when working with code in this repository.
## Naming (owner rule)
- Tables are lowercase snake_case prefixed with the full qbit name: `db_perf_<name>`. The QQQ table name and the backend (SQL) name are identical; possible-value sources take the table name.
- Permissions use the full qbit name: `db_perf.<action>` (for example `db_perf.view`, `db_perf.admin`).
- Permissions use the full qbit name: `db_performance.<action>` (for example `db_performance.view`, `db_performance.manage_settings`).
- Qbit data lives in standard, visible QQQ tables, secured by permissions and record security locks, never hidden.
- Processes run as table or record actions, never in the nav.
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
## Dependencies
- Java 21.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.10` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.11` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Tests: JUnit 5, AssertJ.
## Release
+29 -3
View File
@@ -1,8 +1,34 @@
# Changelog
All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning.
All notable changes are documented here. This project follows Keep a Changelog and semantic versioning.
## [0.1.0] - Unreleased
## [Unreleased]
### Added
- Repository scaffold: `DbPerfQBitConfig` and `DbPerfQBitProducer` (table prefix `db_perf_`, permissions `db_perf.*`), build, CI and repository standard files.
- H2 2.x support (`H2StatsProvider`) from `INFORMATION_SCHEMA`: statement statistics from `QUERY_STATISTICS` (when the database runs with `QUERY_STATISTICS=TRUE`), sessions with blocking and idle-in-transaction detection, connection counts, table row estimates and sizes, index definitions, settings, and text query plans (`EXPLAIN`; `EXPLAIN ANALYZE` only for non-admin H2 users, since H2 has no read-only transaction: plans refuse side-effecting functions such as `FILE_WRITE`, DDL/DML keywords, user-defined aliases and multiple statements). The enterprise test server showed UNSUPPORTED on H2; it now collects.
- Per-area capabilities (`StatsFeature`): providers declare which statistics areas they can supply and `ProviderCapabilities.features()` reports what is available on a connection. Snapshot cache-hit ratio and connection limit stay empty when the vendor has none, and analysis rules declare the areas they need (`DbPerfRule.requires()`) and are skipped on providers that cannot supply them, so H2 never gets "unused index" or "low cache hit" findings from missing counters.
- Provider registry keyed by JDBC product name (`DbStatsProviders.register`), with `docs/PROVIDERS.md` describing what MySQL/MariaDB and SQLite would map to.
- **Enable statement statistics** action on Settings (`dbPerfEnableStatementStatistics`, manage settings): on H2 runs `SET QUERY_STATISTICS_MAX_ENTRIES 1000` and `SET QUERY_STATISTICS TRUE` as an admin; on PostgreSQL explains the manual extension setup. It is the qbit's only change to the monitored database and is explicit.
### Changed
- Health and the Setup panel take their prerequisite checks and remedies from the backend's provider: PostgreSQL still names `pg_stat_statements` and `pg_read_all_stats`; H2 names `QUERY_STATISTICS` and the admin role (new access codes `NO_QUERY_STATISTICS` and `NOT_ADMIN`).
- The previous-snapshot baseline no longer depends on a PostgreSQL-only counter (it uses the database size every supported provider records).
### Fixed
- Query-spike alerts no longer hold seven days of query snapshots as records on every snapshot evaluation (~230,000 rows at kof22's rate). `SnapshotEvaluator` reads the window in 1,000-row keyset pages and keeps only each statement's current interval time, latest capture time and earlier interval times as primitive doubles; the spike rule (twelve prior samples, median, absence after an hour) is unchanged.
- The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. The hourly analysis (`dbPerfAnalyze`) reads attribution the same way: `RuleContext.window("attribution", ...)` returns database sums per unit, table and operation instead of every detail row (it filled the kof22 worker heap to ~1.8 GB every hour).
- Collection no longer stops at RUNNING on large production statement sets: list folding in the SQL normalizer is a linear token scan (the nested-repetition regular expressions overflowed the stack on 60 KB statements with long IN/VALUES lists). Any failure, including an `Error`, now closes the run as FAILED with the exception class, logs it at ERROR (class and frame only, never SQL text) and raises the collection alert; runs left RUNNING by a crash are closed as FAILED after three intervals.
- Only one runtime attributes per JVM: starting attribution on a newer instance (a hot swap, or repeated server starts in tests) stops the previous one, so old instances are never pinned by a forgotten listener and flusher.
- Each table has its own icon (the app icon is now `storage`), so nav entries are distinguishable and do not repeat icons used elsewhere.
- Home tables keep at most four columns with plain headers (readable at every width): secondary facts such as calls and mean time, the session's query text, and a unit's kind and top table move into a second line of the first cell.
### Added
- QQQ attribution and N+1 detection: a core statement execution listener (QQQ 4.1.0-RC.13) aggregates database time per route, scheduled job, process, table and statement shape in bounded memory, records repeated statement shapes per unit of work, and each pod flushes one batched insert per table every `attribution_flush_seconds`. Hosts without QQQ's application launcher call `DbPerfRuntime.of(instance).startAttribution()` after startup.
- Secured database-performance tables, PostgreSQL statistics snapshots, daily rollups and configurable retention.
- Audited PLAN/GENERIC and read-only ANALYZE query viewer with SQL validation, timeout, rollback and transient parameter handling.
- Query plan statement action with readable plan text, cost, SQL Workbench link and a sanitized assistant prompt.
- UI API extension 1.0.0 with overview, live activity, explain and statement lookup, plus its committed OpenAPI contract.
- Eight full-width typed home panels with pagination, record links, guided empty states and permission-gated live activity.
- Maven-versioned QBIT help bundle, complete metadata coverage checks, rule threshold guidance and adoption documentation.
- PostgreSQL security acceptance tests and H2 API/widget fixtures.
+3 -3
View File
@@ -4,7 +4,7 @@ This file provides guidance to Claude Code when working with code in this reposi
## Project Overview
**QBit DB Performance**: Database performance for QQQ apps: slow and expensive queries from pg_stat_statements with delta snapshots, table scan and index analysis, live activity and blocking chains, connection capacity, maintenance, QQQ query attribution with N+1 detection, and ranked suggestions, all as standard QQQ tables. Design: docs/SPEC.md in this repository. Implementation in progress.
**QBit DB Performance**: Database performance for QQQ apps in standard secured tables: PostgreSQL statement deltas, table and index analysis, live activity and blocking, connection capacity, audited read-only query plans, typed full-width dashboards, a versioned UI API and data-driven help. Collection and presentation are implemented; attribution, findings, alerts and health runtime integrations are developed separately. Design: docs/SPEC.md.
## Build Commands
@@ -21,7 +21,7 @@ This file provides guidance to Claude Code when working with code in this reposi
## Naming (owner rule)
- Tables are lowercase snake_case prefixed with the full qbit name: `db_perf_<name>`. The QQQ table name and the backend (SQL) name are identical; possible-value sources take the table name.
- Permissions use the full qbit name: `db_perf.<action>` (for example `db_perf.view`, `db_perf.admin`).
- Permissions use the full qbit name: `db_performance.<action>` (for example `db_performance.view`, `db_performance.manage_settings`).
- Qbit data lives in standard, visible QQQ tables, secured by permissions and record security locks, never hidden.
- Processes run as table or record actions, never in the nav.
@@ -35,7 +35,7 @@ This file provides guidance to Claude Code when working with code in this reposi
## Dependencies
- Java 21.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.10` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.11` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Tests: JUnit 5, AssertJ.
## Release
+119 -44
View File
@@ -1,67 +1,146 @@
# QBit DB Performance
[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://www.apache.org/licenses/LICENSE-2.0)
[![Java](https://img.shields.io/badge/java-21+-blue.svg)](https://adoptium.net/)
> Database performance for QQQ apps: slow and expensive queries from pg_stat_statements with delta snapshots, table scan and index analysis, live activity and blocking chains, connection capacity, maintenance, QQQ query attribution with N+1 detection, and ranked suggestions, all as standard QQQ tables.
**Status:** Phase A is implemented: schema and secured metadata, PostgreSQL statistics collection, daily rollups and retention. Attribution listeners, suggestions, explain actions, dashboard/API integration and help are later phases. Nothing is published yet. Approved design: [docs/SPEC.md](docs/SPEC.md).
Database performance for QQQ apps: PostgreSQL and H2 statement deltas, table and index statistics, live sessions and blocking, connection capacity, audited query plans, and typed home dashboards in standard secured QQQ tables. Approved design: [docs/SPEC.md](docs/SPEC.md). This branch implements collection and the presentation/plan-viewer phase; attribution listeners, findings evaluation, alerts and health are developed in a separate phase. No release has been published yet.
## Requirements
- Java 21.
- QQQ 4.1 (built against `4.1.0-RC.11`).
- PostgreSQL 13+ for collection; PostgreSQL 17 and H2 2.2 are migration test targets.
- Java 21 and QQQ **4.1.0-RC.13+** (the statement execution listener SPI used for QQQ attribution).
- PostgreSQL **13+**; **16+** for parameterized generic plans, with the pgJDBC driver (built against 42.7.14).
- `pg_stat_statements` preloaded, created in the monitored database, and readable by the application role.
- `pg_read_all_stats` granted to that role for complete statistics and other sessions' visibility.
- Or H2 **2.x** (embedded or server), with `QUERY_STATISTICS=TRUE` for statement statistics (see [H2](#h2)).
- A host scheduler for automatic collection and rollups. Docker is required for the PostgreSQL acceptance tests.
## Install
Not yet published. Once released, add `com.kingsrook.qbits:qbit-db-performance` and produce the qbit into your `QInstance`:
Add the dependency when the first release is available (use your locally installed snapshot during development):
```xml
<dependency>
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-db-performance</artifactId>
<version>0.1.0-SNAPSHOT</version>
</dependency>
```
Register a QQQ `PostgreSQLBackendMetaData` backend and the scheduler first, then produce the qbit:
```java
new DbPerfQBitProducer()
.withQBitConfig(new DbPerfQBitConfig()
.withBackendName("rdbms")
.withSchedulerName("scheduler")) // omit to register no schedules
.withSchedulerName("scheduler")
.withParentAppName("operations"))
.produce(qInstance);
```
The backend must already exist. For PostgreSQL, use QQQ's
`PostgreSQLBackendMetaData` from `qqq-backend-module-postgres` so table actions
use the PostgreSQL SQL dialect. The qbit reads statistics through the backend's
connection manager and writes its records through QQQ actions as the system user.
The optional parent must already exist. Include the migration in the host's Liquibase master:
Include `db/liquibase/qbit-db-performance/db_performance.changelog.sql` in the
host's Liquibase changelog. It creates 17 tables and seeds 14 suggestion rules
without overwriting administrator edits. The runtime creates its Settings row
on first use. It does not run migrations itself.
```xml
<include file="db/liquibase/qbit-db-performance/db_performance.changelog.sql"/>
```
Enable `pg_stat_statements` in PostgreSQL's `shared_preload_libraries`, create
the extension in the monitored database, and grant `pg_read_all_stats` to the
application database role. Missing statistics access produces a PARTIAL snapshot
and a Settings message; unsupported database vendors remain usable for metadata.
The changelog creates 17 tables and seeds 14 rules without overwriting administrator edits. The qbit never runs migrations itself. Declare the database role grant through your normal database provisioning:
## Naming
```sql
GRANT pg_read_all_stats TO your_application_role;
```
Tables are `db_perf_<name>` (lowercase snake_case, backend name equal to the table name). Permissions are `db_performance.<action>`; the complete names are in `DbPerfPermissions.ALL`.
Configure `shared_preload_libraries` with `pg_stat_statements` and create the extension in the monitored database through your normal deployment process. Missing access produces reduced-capability status and setup guidance.
The app and tables require `db_performance.view`. Settings updates, rule changes
and Collect now require `db_performance.manage_settings`. Other tables are
read-only to users. Settings and rules have no AI exposure.
### H2
Collection defaults to every five minutes, with table/index samples hourly.
New statements establish a baseline instead of booking historical counts into
one interval. Snapshot totals include all statement deltas; only the top 200
details are stored by default. Therefore a statement's 24-hour totals and daily
rollup reflect its retained details, while its share uses the full database total.
The same qbit runs on an H2 backend; the provider is chosen from the JDBC product name, so no configuration is needed. Add query statistics to the H2 JDBC URL (or run **Enable statement statistics** on Settings as an H2 admin user; H2 forgets it when the database closes):
The daily process rolls up yesterday and missing days in the preceding week,
then purges details after 30 days and daily rows after 365 days. These thresholds
are editable in Settings. Days use the configured time zone, UTC by default.
Processes are scheduled or exposed as table actions, never navigation children.
```
jdbc:h2:file:/data/app;MODE=MySQL;DB_CLOSE_DELAY=-1;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000
```
The cap matters: H2 keeps 100 statements by default and evicts a third of them, oldest first, whenever it is passed. Use an admin user (the default `sa` is one) to see other sessions and blocking. H2 keeps no transaction, cache-hit, scan, vacuum or index-usage counters, so those dashboard figures stay empty and the rules that need them (missing/unused index, bloat, vacuum, cache hit, temp spill, deadlocks, pool sizing) are skipped rather than reporting zeros. Plans are H2 text (index choice and scan counts), not a cost tree. [docs/PROVIDERS.md](docs/PROVIDERS.md) has the full capability table and what other vendors would need.
With `schedulerName`, `dbPerfCollectSnapshot` runs every five minutes and `dbPerfRollupAndPurge` daily at 03:15 (UTC by default). Omitting it registers no schedules. Use **Collect now** on Snapshots for the first baseline and a second collection for deltas. Table and index statistics normally update hourly. The scheduled processes are durable-eligible when the host has durable jobs enabled.
UI hosts must include `qqq-middleware-javalin` (an optional dependency of this qbit). The producer registers UI API extension `db-performance` version `1.0.0` under `/qqq/v1/qbit/db-performance`. Core-only hosts can use tables and collection without middleware.
## Tables
All table names and SQL names are identical; all data is visible through standard QQQ tables.
| Tables | Purpose | Retention |
|---|---|---|
| `db_perf_settings`, `db_perf_rule` | Per-backend settings and rule thresholds | Indefinite |
| `db_perf_snapshot` | Collection status, interval totals and capacity | 30 days |
| `db_perf_statement` | Normalized query inventory and retained 24-hour totals | Stale after 365 days |
| `db_perf_query_snapshot`, `db_perf_query_daily` | Statement interval details and daily aggregates | 30 / 365 days |
| `db_perf_table_stat`, `db_perf_index_stat` | Current table/index inventory and maintenance | Stale after 365 days |
| `db_perf_table_snapshot`, `db_perf_table_daily` | Table history and growth | 30 / 365 days |
| `db_perf_activity_sample`, `db_perf_connection_sample` | Notable activity and connection counts | 30 days |
| `db_perf_attribution`, `db_perf_attribution_daily`, `db_perf_repeat_pattern` | QQQ units of work and repeated statements | 30 / 365 / 30 days |
| `db_perf_finding` | Ranked suggestions and evidence | Open retained; stale closed after 365 days |
| `db_perf_explain` | Every plan request and eligible saved plans | 30 days |
The home page stacks Setup, Database time, Top queries, Findings, Live activity, Connections, QQQ attribution and Tables at full width. Tables paginate; widgets refresh every 30–60 seconds. Statement records include Query Plans and snapshot/daily/finding child lists. Attribution and findings panels remain guided empty states until their producing runtime is installed and has collected data.
Snapshot totals include all observed statement deltas. Only the top configured details are retained, so statement totals and rollups can undercount queries outside that selection. The database-time share denominator includes all observed statement deltas. Reset counters start new baselines rather than creating spikes.
## Permissions
| Permission | Access |
|---|---|
| `db_performance.view` | App, read-only tables, widgets, overview, lookup and normalized activity |
| `db_performance.view_live_text` | Raw SQL in the transient activity API only; also requires view |
| `db_performance.explain` | PLAN/GENERIC requests and Query plan record action |
| `db_performance.explain_analyze` | Execute the guarded ANALYZE service; the record action also needs explain |
| `db_performance.manage_settings` | Settings/rule edits, Collect now and Enable statement statistics |
| `db_performance.manage_findings` | Finding status actions supplied by the findings phase |
Hosts can change `permissionBaseName`; grant the corresponding complete names. Persistent collector and explain-audit writes run as the system user, retaining the requesting session's user ID for plans. Dashboard and API reads retain the viewer's session.
## Configuration
| Host option | Default / meaning |
|---|---|
| `backendName` | Required; monitored database and qbit storage backend |
| `schedulerName` | Optional; host-registered scheduler |
| `appName`, `parentAppName` | `dbPerformance`; optional existing parent app |
| `permissionBaseName` | `db_performance` |
| `attributionEnabled` | `true`; attribution listener integration is a separate phase |
| `sqlWorkbenchRunnerPath` | `/app/sqlWorkbenchRunner` |
| `podName` | `HOSTNAME`, or `local` |
| `snapshotCron`, `analyzeCron`, `rollupCron` | `0 */5 * * * ?`, `0 7 * * * ?`, `0 15 3 * * ?`; analysis schedule is owned by the findings phase |
| `timeZone` | `UTC` |
Runtime values live in Settings: 5-minute collection, hourly tables, 200 top statement details, 30-day detail retention, 365-day rollups, 10-second explain timeout, 30-second long-query and 60-second idle-transaction thresholds. Attribution and alert settings are documented in the shipped help bundle, as are every rule's threshold units and seeded values. Existing administrator settings are never overwritten on startup.
## Query plans and UI API
**Query plan** on one statement presents PLAN/ANALYZE and optional parameter values (one per line). The Result screen shows selectable plan text, cost, a SQL Workbench link and an assistant prompt. There is no action on the Query Plans audit table.
- `GET /overview?backend=`: setup status, latest snapshot, top ten queries and open findings.
- `GET /activity?backend=`: current provider sessions and transitive blocking chains; raw text is permission-gated.
- `POST /explain`: `{ "backend": "rdbms", "sql": "select * from orders where id = $1", "mode": "PLAN" }`. Supply either `statementId` or `sql`. For ANALYZE, add `"parameters": ["123"]` and obtain its separate permission.
- `GET /statements/lookup?backend=&sql=`: normalized fingerprint matches with `/app/<appName>/db_perf_statement/<id>` history paths.
Paths above are relative to `/qqq/v1/qbit/db-performance`. The [committed OpenAPI fragment](src/main/resources/openapi/qqq-v1/db-performance.yaml) is generated from endpoint specs and verified by a contract test. The server never calls an AI model.
## Security and PHI
Stored SQL is normalized and fingerprinted. Raw `pg_stat_activity` text is never persisted and appears only in the activity API with `view_live_text`; home widgets always use normalized text. SQL literals and parameter values must not be used as identifiers: normalization preserves quoted identifiers. Settings and rules are excluded from AI exposure.
On H2, PLAN runs `EXPLAIN` (not executed; `?n` slots stay unbound for generic plans) and ANALYZE runs `EXPLAIN ANALYZE` with the values bound. H2 has no READ ONLY transaction and a rollback does not undo file, session, sequence or DDL side effects, so on H2 both PLAN and ANALYZE refuse more than one statement and any SQL that names a side-effecting function (`FILE_WRITE`, `FILE_READ`, `CSVWRITE`, `CSVREAD`, `LINK_SCHEMA`, `SCRIPT`, `RUNSCRIPT`, `DISK_SPACE_USED`, `CANCEL_SESSION`, `ABORT_SESSION`, `SLEEP`, sequences), a DDL/DML or session keyword, or any user-defined alias; quoted names count. ANALYZE executes the query, so it is offered only to non-admin H2 users (an admin can call file functions) and only for a single SELECT/WITH; capabilities report it as unavailable otherwise. The connection is marked read-only (a hint in H2) and the transaction always rolls back.
PLAN executes EXPLAIN in a read-only transaction with a local timeout and rollback. Parameterized PLAN uses PostgreSQL 16+ GENERIC_PLAN. Only eligible literal-free PLAN/GENERIC plans are stored. A user's SQL must equal its own normalized form to qualify; SQL loaded from the normalized statement table also qualifies.
ANALYZE accepts one SELECT or a non-modifying WITH, rejects INTO and row locks, binds supplied strings as unknown JDBC parameters, and executes inside a read-only transaction. Functions are still constrained by PostgreSQL READ ONLY. The EXPLAIN command uses pgJDBC simple-query execution with JDBC-managed parameter quoting, so PostgreSQL can coerce unknown-typed EXECUTE arguments. The transaction always rolls back and the prepared statement is explicitly deallocated before the connection returns to its pool. SQLSTATE 57014 records TIMEOUT. The service stores only a sanitized audit, never ANALYZE plans or parameter values. Returned ANALYZE plans are transient and can contain values; the assistant prompt redacts literals in plan expressions. Review transient plans before copying them elsewhere.
## Adopting in another app
Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) once the first release is published.
1. Add compatible QQQ, PostgreSQL backend and qbit dependencies; include the changelog once.
2. Preload/create pg_stat_statements and grant the database statistics role through your provisioning.
3. Register the backend, optional parent app and scheduler, then produce the qbit.
4. Grant the exact permissions to intended operators; reserve ANALYZE and raw live text for appropriate roles.
5. Run Collect now twice, check Setup, and confirm the API extension in `/qqq/v1/openapi.yaml`.
6. Optionally configure SQL Workbench links; install attribution/findings runtime integrations when available. Review and commit proposed database changes through the host's migration workflow.
## Build
@@ -69,12 +148,8 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/
./mvnw -o verify
```
Always use the Maven wrapper (Maven 3.9.11); Maven 3.10 cannot read the parent POM.
On Windows use `./mvnw.cmd -o verify`. Validation includes main/test checkstyle,
JUnit tests and JaCoCo gates of 70% instructions and 90% classes. PostgreSQL
Testcontainers tests (including `PostgresStatsProviderIT`) run with the suite and
skip when Docker is unavailable.
On Windows use `./mvnw.cmd -o verify`. Always use the wrapper; Maven 3.10 cannot read the parent POM. Offline mode requires dependencies already cached. Verification runs main/test Checkstyle, JUnit including PostgreSQL 17 Testcontainers, and JaCoCo gates of 70% instructions and 90% classes. Docker must be accessible to the build process. Regenerate the API fragment with `./mvnw -o -Dtest=DbPerfApiContractTest -Ddbperf.openapi.update=true test`.
## License
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
[Apache License 2.0](LICENSE).
+13
View File
@@ -0,0 +1,13 @@
# Upgrading QBit DB Performance
## First release
There is no previous published release. See [CHANGELOG.md](CHANGELOG.md) for additions and [README.md](README.md) for complete installation.
Use QQQ 4.1.0-RC.11 or newer, Java 21, and PostgreSQL 13 or newer (16+ for generic plans). Include `db/liquibase/qbit-db-performance/db_performance.changelog.sql` once in your application's Liquibase changelog. Run migrations through the host deployment before collecting statistics; the qbit does not apply them automatically.
Preload and create `pg_stat_statements`, then grant `pg_read_all_stats` to the application's database role through your normal provisioning. Without the role the collector reports reduced visibility. Preserve existing rule and settings edits; seeding is insert-only.
Register the producer and a host scheduler. Grant `db_performance.view` and the narrower operator permissions appropriate to each role; ANALYZE and raw live text require separate grants. Existing deployments using an early scaffold's `db_perf.*` grants must use the actual `db_performance.*` names (or the configured permission base).
Collect twice to establish useful deltas. Inspect Setup for access and schedule status. UI hosts need the optional `qqq-middleware-javalin` dependency so the four endpoints register in the standard UI API. The help bundle version is filtered from the Maven project version during the build.
+58
View File
@@ -0,0 +1,58 @@
# Attribution runtime (phase B2)
The producer registers `AttributionRuntimeService` with the QQQ application
launcher. It starts one daemon per application instance after initialization,
independently of scheduled-job leadership. Hosts without `QApplicationLauncher`
should call `DbPerfRuntime.of(instance).startAttribution()` after migrations and
instance validation, and `stopAttribution()` before disposing of the instance.
`attributionEnabled=false` prevents listener registration and background work.
Metadata production itself starts no threads, so existing metadata tests remain
isolated.
`AttributionFlusher` can also be constructed directly for deterministic tests:
`refreshSettings()`, register `getListener()` with `StatementExecutionListeners`,
and call `flushNow()`. Always close it to unregister. Production `start()` loads
settings, registers the listener, schedules flushing, and installs a JVM shutdown
hook. Closing stops scheduling, unregisters, and flushes once. Shutdown only
persists completed units; hosts should drain application requests first.
Statements never read settings from the database. A successful settings refresh
publishes an immutable cached copy; a failed refresh keeps the previous copy.
The initial copy disables recording until settings can be read. The configured
flush interval takes effect when the next pass is scheduled. Disabled attribution
still refreshes settings so administrators can re-enable it.
Units are sampled once and buffered until completion, allowing HTTP route names
to be finalized. A completed unit belongs to the window in which it ends; an
abandoned unit is merged when that worker sees a different unit. No-unit work is
sampled per statement and counted as one unit per statement. Sampling transitions
retain separate aggregates for each effective rate. Values are observed counts,
not extrapolated estimates. Daily rollups sum those counts across windows and
pods and take the maximum duration.
Each global map includes its overflow bucket within `attribution_max_keys`.
One slot is reserved for a universal `(other)` key (all dimensions collapse,
fingerprint is SHA-256 of `(other)`). Per-unit maps use the same bound with a
minimum of two slots. Collapsed per-unit shapes cannot establish N+1 evidence
and are excluded from repeat detection, avoiding false positives. Global repeat
overflow sums qualifying repetitions but counts each affected unit only once.
A mixed-rate overflow row has a null sample rate because there is no single
valid rate for it. The SQL normalization cache holds at most 5,000 raw SQL keys
in memory and clears on overflow; only normalized SQL is persisted, capped at
4,000 characters.
Window swaps and counter merges synchronize only in memory. Database work runs
under the system user on the flusher thread with attribution suppressed. Each
nonempty table receives one `InsertAction`, even above 1,000 rows; QQQ may page
the underlying SQL. Both tables are written in one transaction. Failed windows
are dropped, with rate-limited fixed-text logging, and never queued for retry.
The repeat-pattern migration adds `sample_rate` without changing prior
changesets. Existing retention already includes repeat patterns.
Build using the locally installed statement-listener API:
```powershell
.\mvnw.cmd -o verify '-Dqqq.snapshot.version=4.1.0-dbperf-SNAPSHOT'
```
The normal POM default is `4.1.0-RC.13`.
+73
View File
@@ -0,0 +1,73 @@
# Statistics providers
The qbit reads database statistics through `DbStatsProvider`. `DbStatsProviders` picks the provider from the JDBC product name (`DatabaseMetaData.getDatabaseProductName()`); anything unregistered gets `UnsupportedStatsProvider`, which shows a setup message instead of failing. Each provider declares the statistics areas (`StatsFeature`) its vendor can supply; `ProviderCapabilities.features()` says which are available on the current connection. Dashboards leave unavailable figures empty and analysis rules that need a missing area (`DbPerfRule.requires()`) are skipped, so a vendor with fewer counters produces fewer findings, never false ones.
| JDBC product name | Provider | Saved name | Status |
|---|---|---|---|
| `PostgreSQL` | `PostgresStatsProvider` | `POSTGRES` | Supported (13+) |
| `H2` | `H2StatsProvider` | `H2` | Supported (2.x) |
| `MySQL`, `MariaDB` | none yet | `MYSQL` (reserved) | Unsupported |
| `SQLite` | none yet | | Unsupported |
## Areas by provider
S = supported, A = approximated, - = not available from the vendor.
| Area (`StatsFeature`) | PostgreSQL | H2 | H2 source and notes |
|---|---|---|---|
| Database counters (commits, rollbacks, temp, deadlocks) | S | - | H2 keeps no transaction or temp-file counters |
| Cache hit ratio | S | - | `info.CACHE_HIT_RATIO` exists but is not a cumulative hit/read pair; shown in settings only |
| Database size | S | S | `SETTINGS` `info.FILE_SIZE` (0 for in-memory databases) |
| Connection limit | S | - | H2 has no connection limit |
| Statement statistics | S | S | `QUERY_STATISTICS` with `QUERY_STATISTICS=TRUE`: calls, total/mean/max ms, rows. No query id or user: the id is a stable SHA-256 hash of the text and the user is empty. The collector's own `INFORMATION_SCHEMA` and `DISK_SPACE_USED` reads and connection setup (`SET ...`, which H2 replays for every JDBC URL setting on each new connection, and `CALL DATABASE()`) are excluded. H2 keeps `QUERY_STATISTICS_MAX_ENTRIES` statements (default 100, set 1000) and evicts the oldest third past the cap |
| Statement I/O (blocks, temp) | S | - | |
| Statement reset time | S (14+) | - | A restart or eviction shows as falling call counts, which the delta logic treats as a reset |
| Table row estimates | S | A | `TABLES.ROW_COUNT_ESTIMATE` |
| Table sizes | S | A | `DISK_SPACE_USED(table)`: table data only; H2 cannot size indexes, so total = table bytes |
| Table access counters (scans, tuple writes) | S | - | |
| Table maintenance (dead rows, vacuum, freeze) | S | - | MVStore has no vacuum |
| Index definitions | S | S | `INDEXES` + `INDEX_COLUMNS`; constraint-generated indexes are marked `USING CONSTRAINT` and never proposed for removal |
| Index usage and size | S | - | |
| Live sessions | S | S | `SESSIONS`: RUNNING = active, SLEEP with uncommitted changes = idle in transaction. Non-admin users see only their own session |
| Blocking | S | S | `SESSIONS.BLOCKER_ID` (admin users) |
| Wait events | S | A | Blocked sessions report wait type `Lock` |
| Transaction age | S | A | No transaction start in H2: an idle transaction's age is the time since it went idle (a lower bound) |
| Connection counts | S | S | `SESSIONS` grouped by user, client info and state |
| Plans | JSON tree with costs | Text | `EXPLAIN` / `EXPLAIN ANALYZE` (index choice, scan counts); generic plans keep `?n` unbound. ANALYZE only for non-admin users (see below) |
Access codes saved on settings: PostgreSQL `FULL`, `NO_READ_ALL_STATS`, `NO_PG_STAT_STATEMENTS`, `UNSUPPORTED`; H2 `FULL`, `NOT_ADMIN`, `NO_QUERY_STATISTICS`, `UNSUPPORTED`. Health and the Setup panel ask the provider for the remedy text (`remedy`, `setupChecks`). **Enable statement statistics** on Settings calls `enableStatementStatistics`: H2 runs `SET QUERY_STATISTICS_MAX_ENTRIES 1000` and `SET QUERY_STATISTICS TRUE` (admin, until the database closes); PostgreSQL cannot do it at runtime.
## H2 query plan rule
H2 has no READ ONLY transaction and a rollback does not undo file, session, sequence or DDL side effects, so on H2 both PLAN and ANALYZE refuse more than one statement and any SQL that names a side-effecting function (`FILE_WRITE`, `FILE_READ`, `CSVWRITE`, `CSVREAD`, `LINK_SCHEMA`, `SCRIPT`, `RUNSCRIPT`, `DISK_SPACE_USED`, `CANCEL_SESSION`, `ABORT_SESSION`, `SLEEP`, sequences), a DDL/DML or session keyword, or any user-defined alias; quoted names count. ANALYZE executes the query, so it is offered only to non-admin H2 users (an admin can call file functions) and only for a single SELECT/WITH; capabilities report it as unavailable otherwise. The connection is marked read-only (a hint in H2) and the transaction always rolls back. A host whose H2 user is `sa` (an admin, as in the enterprise test server) therefore gets PLAN only.
## Adding a vendor
1. Implement `DbStatsProvider`: the reads, `capabilities`, `features`, `remedy`, `setupChecks`, and optionally `enableStatementStatistics` and `explainText`. Map session states to `active` / `idle` / `idle in transaction` and leave unavailable counters out of the maps rather than writing zeros.
2. Register it under the JDBC product name: add it to the table in `DbStatsProviders`, or call `DbStatsProviders.register("ProductName", MyProvider::new)` from the host before collection.
3. Add its saved name to `DbPerfPossibleValues.Provider` and any new access codes to `StatsAccess` and the health check.
4. Make sure the qbit's Liquibase changelog runs on the vendor (the qbit stores its tables in the monitored backend).
5. Test every read against a real database (Testcontainers or embedded), with and without the vendor's statistics prerequisites.
## MySQL and MariaDB
What a provider would map (MySQL 8.0+, MariaDB 10.6+ with `performance_schema=ON`, which needs a server restart):
| Area | Source | Notes |
|---|---|---|
| Statements | `performance_schema.events_statements_summary_by_digest` | Digest id and normalized text, calls, timer (picoseconds), rows examined/sent, temp tables, `FIRST_SEEN`; `TRUNCATE` resets. Needs `SELECT` on `performance_schema` |
| Database counters | `SHOW GLOBAL STATUS` (`Com_commit`, `Com_rollback`, `Created_tmp_disk_tables`, `Innodb_deadlocks`) | Server-wide, not per schema |
| Cache hit ratio | `Innodb_buffer_pool_read_requests` / `Innodb_buffer_pool_reads` | |
| Connection limit and counts | `max_connections`, `information_schema.PROCESSLIST` / `performance_schema.threads` | Needs `PROCESS` to see other users |
| Sessions and blocking | `PROCESSLIST`, `sys.innodb_lock_waits` (MySQL) / `information_schema.INNODB_LOCK_WAITS` (MariaDB) | |
| Tables | `information_schema.TABLES` (`TABLE_ROWS` estimate, `DATA_LENGTH`, `INDEX_LENGTH`) | Sizes are estimates |
| Table access | `performance_schema.table_io_waits_summary_by_table` | Read/write counts, no seq-vs-index split |
| Index usage | `performance_schema.table_io_waits_summary_by_index_usage`, `sys.schema_unused_indexes` | |
| Maintenance | - | InnoDB purge is automatic; no vacuum or dead-row counters |
| Plans | `EXPLAIN FORMAT=JSON`, `EXPLAIN ANALYZE` (MySQL 8.0.18+, tree text) | MariaDB uses `ANALYZE FORMAT=JSON`; a JSON renderer would be needed |
Beyond the provider class, MySQL support also needs: the MySQL/MariaDB Connector/J on the host classpath; a provider-neutral `ExplainService` path for JSON plans (today JSON plans are PostgreSQL's format and the renderer is PostgreSQL-specific); MySQL-compatible SQL in the changelog (verify it runs, since the qbit stores its own tables there) and in the migration snippets the findings suggest (`CREATE INDEX CONCURRENTLY` is PostgreSQL syntax; MySQL needs `ALGORITHM=INPLACE, LOCK=NONE`); help text that currently names PostgreSQL views; and Testcontainers tests for both MySQL and MariaDB, since their lock views differ.
## SQLite
SQLite has no server, sessions, statement statistics or users. A provider could supply only: database size (`PRAGMA page_count` x `page_size`), tables and row counts (`sqlite_schema` plus `COUNT(*)` or `sqlite_stat1` after `ANALYZE`), table and index sizes (the `dbstat` virtual table, when compiled in), index definitions (`sqlite_schema.sql`, `PRAGMA index_list` / `index_info`), and text plans (`EXPLAIN QUERY PLAN`). Statement statistics would have to come from QQQ attribution (the qbit's own statement listener), not the database. It is useful only as an inventory and plan viewer.
+7 -5
View File
@@ -47,7 +47,9 @@ actions └► Attribution aggregator (memory) │ │ Analyze → f
`explain(...)`. `PostgresStatsProvider` is the v1 implementation (PostgreSQL 13+, features that
need 16+/17 such as `EXPLAIN (GENERIC_PLAN)` and per-entry `stats_since` are detected).
The provider is chosen per monitored backend from the JDBC vendor; an unsupported vendor is
reported in the Setup card, never a crash.
reported in the Setup card, never a crash. `H2StatsProvider` (H2 2.x, `INFORMATION_SCHEMA`)
followed; providers declare per-area `StatsFeature`s and rules are skipped where an area is
missing. See [PROVIDERS.md](PROVIDERS.md).
- **Connections** are the backend's own (QQQ `ConnectionManager`), so they run as the app's
database user. That user needs the built-in `pg_read_all_stats` role to see other sessions'
query text and all statements; without it the qbit runs in a reduced mode and the Setup card and
@@ -230,11 +232,11 @@ batches, so a rerun after a crash never double counts. Each pass also writes its
`db_perf_snapshot` row (status, duration, error), which is the report even where the host runs
scheduled processes inline.
**Alerts** (core `QAlerts`; delivered by qbit-alerts when installed, logged otherwise; keyed, and
resolved when clear): `db_perf.collection_failed`, `db_perf.query_spike` (a statement's total time
**Alerts** (core `QAlerts`, type names use `dbperf.` because core rejects underscores in alert type names; delivered by qbit-alerts when installed, logged otherwise; keyed, and
resolved when clear): `dbperf.collection_failed`, `dbperf.query_spike` (a statement's total time
in an interval above `spike_factor` × its 7-day interval median and above a floor),
`db_perf.connection_saturation` (connections ≥ `connection_alert_percent` of `max_connections`),
`db_perf.blocked_chain` (a session blocked longer than `blocked_alert_seconds`).
`dbperf.connection_saturation` (connections ≥ `connection_alert_percent` of `max_connections`),
`dbperf.blocked_chain` (a session blocked longer than `blocked_alert_seconds`).
**Health** (core health SPI, Admin → Health): `db_performance.collector` (last successful snapshot
age vs interval), `db_performance.stats_access` (pg_stat_statements present, `pg_read_all_stats`
+30 -4
View File
@@ -18,16 +18,18 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<revision>0.1.0-SNAPSHOT</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
<mockito.version>5.23.0</mockito.version>
</properties>
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.0.0</version>
<version>2.2.1</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -67,6 +69,17 @@
</scm>
<dependencies>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-javalin</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-module-postgres</artifactId>
@@ -92,7 +105,7 @@
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.14</version>
<scope>test</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
@@ -126,12 +139,25 @@
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
<excludes><exclude>help/**</exclude></excludes>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
<includes><include>help/**</include></includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version>
<configuration>
<argLine>@{argLine} -javaagent:"${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar"</argLine>
<includes>
<include>**/*Test.java</include>
<include>**/*IT.java</include>
@@ -216,14 +242,14 @@
<profiles>
<!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent -->
<!-- imports the released 4.1 BOM, this profile is active by default and -->
<!-- imports qqq-bom-pom 4.1.0-RC.11 ahead of the parent's 4.0.0 BOM. -->
<!-- imports qqq-bom-pom 4.1.0-RC.13 ahead of the parent's 4.0.0 BOM. -->
<profile>
<id>qqq-snapshot</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.11</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -23,10 +23,19 @@ package com.kingsrook.qbits.dbperformance;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.dbperformance.alerts.SnapshotEvaluator;
import com.kingsrook.qbits.dbperformance.api.DbPerformanceUiApi;
import com.kingsrook.qbits.dbperformance.attribution.AttributionRuntimeService;
import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess;
import com.kingsrook.qbits.dbperformance.health.DbPerfHealthChecks;
import com.kingsrook.qbits.dbperformance.metadata.DbPerfHelp;
import com.kingsrook.qbits.dbperformance.metadata.DbPerfMetaData;
import com.kingsrook.qbits.dbperformance.processes.DbPerfProcesses;
import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgets;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
@@ -137,9 +146,30 @@ public class DbPerfQBitProducer implements QBitMetaDataProducer<DbPerfQBitConfig
instance.addQBit(getQBitMetaData());
DbPerfMetaData.addTo(instance, qBitConfig);
new DbPerfRuntime(instance, qBitConfig).addSelfToInstance(instance);
com.kingsrook.qbits.dbperformance.processes.DbPerfProcesses.addTo(instance, qBitConfig);
com.kingsrook.qbits.dbperformance.alerts.SnapshotEvaluator.register(instance, qBitConfig);
com.kingsrook.qbits.dbperformance.health.DbPerfHealthChecks.register(instance);
instance.withRuntimeService(new QCodeReference(AttributionRuntimeService.class));
DbPerfProcesses.addTo(instance, qBitConfig);
SnapshotEvaluator.register(instance, qBitConfig);
DbPerfHealthChecks.register(instance);
QueryPlanProcess.register(instance, qBitConfig);
registerUiApi(instance);
DbPerfWidgets.register(instance, qBitConfig);
DbPerfHelp.register(instance, qBitConfig);
return (new MetaDataProducerMultiOutput());
}
/*******************************************************************************
** Middleware is optional for hosts that use only the core tables and collector.
*******************************************************************************/
private static void registerUiApi(QInstance instance)
{
try
{
Class.forName("com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData");
DbPerformanceUiApi.register(instance);
}
catch(ClassNotFoundException e)
{
// Core-only hosts have no HTTP API to register.
}
}
}
@@ -22,6 +22,7 @@ package com.kingsrook.qbits.dbperformance;
import java.util.concurrent.atomic.AtomicReference;
import com.kingsrook.qbits.dbperformance.attribution.AttributionFlusher;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.context.CapturedContext;
import com.kingsrook.qqq.backend.core.context.QContext;
@@ -39,6 +40,13 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
public static final String NAME = "dbPerformance";
private final QInstance instance;
private final DbPerfQBitConfig config;
private AttributionFlusher attributionFlusher;
/////////////////////////////////////////////////////////////////////////////
// statement listeners are JVM-wide: only one runtime attributes at a time //
// (a newer instance, e.g. after a hot swap or in tests, replaces the old) //
/////////////////////////////////////////////////////////////////////////////
private static final AtomicReference<DbPerfRuntime> ACTIVE_ATTRIBUTION = new AtomicReference<>();
/*******************************************************************************
@@ -140,6 +148,38 @@ public class DbPerfRuntime implements QSupplementalInstanceMetaData
}
/*******************************************************************************
** Called by the launcher, or explicitly by hosts without a launcher.
*******************************************************************************/
public synchronized void startAttribution()
{
if(attributionFlusher == null && Boolean.TRUE.equals(config.getAttributionEnabled()))
{
DbPerfRuntime previous = ACTIVE_ATTRIBUTION.getAndSet(this);
if(previous != null && previous != this)
{
previous.stopAttribution();
}
attributionFlusher = new AttributionFlusher(this);
attributionFlusher.start();
}
}
/*******************************************************************************
** Stop the per-instance listener and its daemon; safe to call repeatedly.
*******************************************************************************/
public synchronized void stopAttribution()
{
ACTIVE_ATTRIBUTION.compareAndSet(this, null);
if(attributionFlusher != null)
{
attributionFlusher.close();
attributionFlusher = null;
}
}
/*******************************************************************************
** Checked service operation.
*******************************************************************************/
@@ -22,9 +22,11 @@ package com.kingsrook.qbits.dbperformance.alerts;
import java.time.Instant;
import java.util.Comparator;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
@@ -38,6 +40,9 @@ import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -146,7 +151,7 @@ public class SnapshotEvaluator
long blocked = activity.stream().filter(r -> "BLOCKED".equals(r.getValueString("reason")))
.filter(r -> Math.max(RuleContext.number(r, "querySeconds"), RuleContext.number(r, "xactSeconds")) >= RuleContext.number(settings, "blockedAlertSeconds")).count();
signal(BLOCKED_CHAIN, backend, null, blocked > 0, "Database blocking chain", blocked + " blocked sessions exceed the duration threshold", snapshot, "db_perf_snapshot");
if("OK".equals(status) || "NO_READ_ALL_STATS".equals(settings.getValueString("statsAccess")))
if("OK".equals(status) || List.of("NO_READ_ALL_STATS", "NOT_ADMIN").contains(settings.getValueString("statsAccess")))
{
spikes(ctx, snapshot);
}
@@ -159,33 +164,31 @@ public class SnapshotEvaluator
*******************************************************************************/
private void spikes(RuleContext ctx, QRecord snapshot) throws Exception
{
Map<Integer, List<QRecord>> history = ctx.window("query_snapshot", 7).stream().collect(Collectors.groupingBy(r -> r.getValueInteger("statementId")));
Map<Integer, StatementHistory> history = history(ctx, snapshot.getValueInteger("id"));
for(QRecord statement : ctx.identities("statement"))
{
List<QRecord> samples = history.getOrDefault(statement.getValueInteger("id"), List.of());
QRecord current = samples.stream().filter(r -> snapshot.getValueInteger("id").equals(r.getValueInteger("snapshotId"))).findFirst().orElse(null);
if(current == null)
StatementHistory samples = history.getOrDefault(statement.getValueInteger("id"), new StatementHistory());
if(samples.current == null)
{
Instant last = samples.stream().map(r -> r.getValueInstant("capturedAt")).max(Comparator.naturalOrder()).orElse(Instant.EPOCH);
Instant last = samples.last == null ? Instant.EPOCH : samples.last;
if(!last.plusSeconds(3600).isAfter(ctx.now))
{
signal(QUERY_SPIKE, ctx.backend, statement.getValueString("id"), false, "Query absent", "Statement absent from snapshots for at least one hour", statement, "db_perf_statement");
}
continue;
}
List<Double> previous = samples.stream().filter(r -> r.getValueInstant("capturedAt").isBefore(ctx.now))
.map(r -> RuleContext.number(r, "totalMs")).sorted().toList();
if(previous.size() < 12)
double[] previous = samples.sortedPrevious();
if(previous.length < 12)
{
if(RuleContext.number(current, "totalMs") < RuleContext.number(ctx.settings, "spikeMinTotalMs"))
if(samples.current < RuleContext.number(ctx.settings, "spikeMinTotalMs"))
{
signal(QUERY_SPIKE, ctx.backend, statement.getValueString("id"), false, "Query recovered", "Interval time is below the configured floor", statement, "db_perf_statement");
}
continue;
}
int middle = previous.size() / 2;
double median = previous.size() % 2 == 0 ? (previous.get(middle - 1) + previous.get(middle)) / 2 : previous.get(middle);
double time = RuleContext.number(current, "totalMs");
int middle = previous.length / 2;
double median = previous.length % 2 == 0 ? (previous[middle - 1] + previous[middle]) / 2 : previous[middle];
double time = samples.current;
boolean spike = time >= RuleContext.number(ctx.settings, "spikeMinTotalMs") && time >= median * RuleContext.number(ctx.settings, "spikeFactor");
String normalized = SqlNormalizer.normalize(statement.getValueString("queryText"));
String text = normalized.substring(0, Math.min(200, normalized.length()));
@@ -195,6 +198,90 @@ public class SnapshotEvaluator
}
/*******************************************************************************
** Seven days of per-statement interval times, read in keyset pages and kept
** as primitive doubles. kof22 writes ~33,000 query snapshots a day, so the
** window is ~230,000 rows; holding them as QRecords on every snapshot
** evaluation cost hundreds of MB of worker heap.
*******************************************************************************/
static Map<Integer, StatementHistory> history(RuleContext ctx, Integer currentSnapshotId) throws Exception
{
Set<Integer> valid = ctx.window("snapshot", 7).stream().filter(r -> List.of("OK", "PARTIAL").contains(r.getValueString("status")))
.map(r -> r.getValueInteger("id")).collect(Collectors.toSet());
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN, ctx.now.minusSeconds(7 * 86400L)),
new QFilterCriteria("capturedAt", QCriteriaOperator.LESS_THAN_OR_EQUALS, ctx.now));
Map<Integer, StatementHistory> history = new HashMap<>();
DbPerfStore.forEachPage("db_perf_query_snapshot", filter, DbPerfStore.BATCH_SIZE, page ->
{
for(QRecord row : page)
{
Integer snapshotId = row.getValueInteger("snapshotId");
if(!valid.contains(snapshotId))
{
continue;
}
StatementHistory statement = history.computeIfAbsent(row.getValueInteger("statementId"), ignored -> new StatementHistory());
Instant at = row.getValueInstant("capturedAt");
double totalMs = RuleContext.number(row, "totalMs");
if(statement.last == null || at.isAfter(statement.last))
{
statement.last = at;
}
if(snapshotId.equals(currentSnapshotId) && statement.current == null)
{
statement.current = totalMs;
}
if(at.isBefore(ctx.now))
{
statement.add(totalMs);
}
}
});
return (history);
}
/*******************************************************************************
** One statement's samples: the current interval, the latest capture time
** and earlier interval times.
*******************************************************************************/
static final class StatementHistory
{
private Double current;
private Instant last;
private double[] previous = new double[16];
private int size;
/***************************************************************************
** Append an earlier interval time.
***************************************************************************/
void add(double value)
{
if(size == previous.length)
{
previous = Arrays.copyOf(previous, size * 2);
}
previous[size++] = value;
}
/***************************************************************************
** Earlier interval times, ascending.
***************************************************************************/
double[] sortedPrevious()
{
double[] sorted = Arrays.copyOf(previous, size);
Arrays.sort(sorted);
return (sorted);
}
}
/*******************************************************************************
** Stable backend/subject keys make raises and resolutions restart-safe.
*******************************************************************************/
@@ -25,9 +25,12 @@ import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -70,7 +73,8 @@ public class AnalyzeService
/*******************************************************************************
** Disabled rules are neither evaluated nor reconciled.
** Disabled rules, and rules whose evidence the backend's provider cannot
** supply (unknown before the first collection), are neither evaluated nor reconciled.
*******************************************************************************/
public List<Result> run(String backend) throws Exception
{
@@ -79,9 +83,15 @@ public class AnalyzeService
throw (new IllegalArgumentException("Backend is not configured for this runtime"));
}
QRecord settings = new DbPerfSettingsService(runtime).ensure(backend);
String provider = settings.getValueString("provider");
Set<StatsFeature> available = provider == null ? null : DbStatsProviders.forProviderName(provider).features();
List<Result> results = new ArrayList<>();
for(DbPerfRule rule : rules)
{
if(available != null && !available.containsAll(rule.requires()))
{
continue;
}
try
{
QRecord row = runtime.asSystem(() -> DbPerfStore.first("db_perf_rule", DbPerfStore.eq("code", rule.code())));
@@ -22,6 +22,8 @@ package com.kingsrook.qbits.dbperformance.analyze;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
/*******************************************************************************
@@ -39,4 +41,15 @@ public interface DbPerfRule
** Evaluate the configured thresholds against stored observations.
*******************************************************************************/
List<FindingDraft> evaluate(RuleContext ctx) throws Exception;
/*******************************************************************************
** Statistics areas the evidence comes from. A rule is skipped (neither
** evaluated nor reconciled) when the backend's provider cannot supply them.
*******************************************************************************/
default Set<StatsFeature> requires()
{
return (Set.of());
}
}
@@ -25,17 +25,27 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
/*******************************************************************************
@@ -130,7 +140,7 @@ public class RuleContext
{
filter.withCriteria(new QFilterCriteria("backendName", QCriteriaOperator.EQUALS, backend));
}
List<QRecord> rows = read("db_perf_" + suffix, filter);
List<QRecord> rows = suffix.equals("attribution") ? attributionTotals(filter) : read("db_perf_" + suffix, filter);
if(!Set.of("snapshot", "attribution", "repeat_pattern").contains(suffix))
{
Set<Integer> ids = window("snapshot", days).stream()
@@ -144,6 +154,52 @@ public class RuleContext
}
/*******************************************************************************
** Most attribution groups the rules receive (prod: ~2,000 per day).
*******************************************************************************/
public static final int MAX_ATTRIBUTION_GROUPS = 50_000;
/*******************************************************************************
** Attribution summed in the database per unit, table and operation: the
** fields the rules read (statements, totalMs and the other counters), never
** the day's detail rows. A day is ~570,000 rows in kof22 and loading it made
** every hourly dbPerfAnalyze fill the worker heap (2026-10-16).
*******************************************************************************/
private static List<QRecord> attributionTotals(QQueryFilter filter) throws QException
{
List<String> dimensions = List.of("unitKind", "unitName", "tableName", "operation");
AggregateInput input = new AggregateInput("db_perf_attribution").withFilter(filter).withLimit(MAX_ATTRIBUTION_GROUPS)
.withTransaction(QContext.getQBackendTransaction());
Map<String, GroupBy> groupBys = new LinkedHashMap<>();
for(String dimension : dimensions)
{
GroupBy groupBy = new GroupBy(QFieldType.STRING, dimension);
groupBys.put(dimension, groupBy);
input.withGroupBy(groupBy);
}
Map<String, Aggregate> aggregates = new LinkedHashMap<>();
for(String counter : List.of("units", "statements", "rowsCount", "errors", "totalMs"))
{
aggregates.put(counter, new Aggregate(counter, AggregateOperator.SUM));
}
aggregates.put("maxMs", new Aggregate("maxMs", AggregateOperator.MAX));
aggregates.values().forEach(input::withAggregate);
List<QRecord> rows = new ArrayList<>();
for(AggregateResult result : new AggregateAction().execute(input).getResults())
{
QRecord row = new QRecord();
groupBys.forEach((field, groupBy) -> row.setValue(field, result.getGroupByValue(groupBy)));
aggregates.forEach((field, aggregate) -> row.setValue(field, result.getAggregateValue(aggregate)));
rows.add(row);
}
return (rows);
}
/*******************************************************************************
** Latest inventory only: retired tables and dropped indexes cannot recur.
*******************************************************************************/
@@ -36,6 +36,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -60,6 +61,30 @@ public enum StoredRule implements DbPerfRule
}
/*******************************************************************************
** Provider areas behind each rule's evidence; attribution rules need none.
*******************************************************************************/
@Override
public Set<StatsFeature> requires()
{
return (switch(this)
{
case SLOW_QUERY -> Set.of(StatsFeature.STATEMENTS);
case MISSING_INDEX, OVER_AUDITED -> Set.of(StatsFeature.TABLE_ACCESS);
case UNUSED_INDEX -> Set.of(StatsFeature.INDEX_USAGE);
case DUPLICATE_INDEX -> Set.of(StatsFeature.INDEX_DEFINITIONS);
case BLOAT, VACUUM_STALE -> Set.of(StatsFeature.TABLE_MAINTENANCE);
case POOL_SIZING -> Set.of(StatsFeature.MAX_CONNECTIONS, StatsFeature.CONNECTIONS);
case IDLE_IN_TRANSACTION -> Set.of(StatsFeature.SESSIONS);
case CACHE_HIT_LOW -> Set.of(StatsFeature.CACHE_COUNTERS);
case TEMP_SPILL -> Set.of(StatsFeature.STATEMENT_IO);
case DEADLOCKS -> Set.of(StatsFeature.DATABASE_COUNTERS);
case N_PLUS_ONE, HOT_JOB -> Set.of();
});
}
/*******************************************************************************
** Disabled evaluation is inert even when called outside AnalyzeService.
*******************************************************************************/
@@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.openapi.model.HttpMethod;
/*******************************************************************************
** Database Performance activity under the authenticated UI API.
*******************************************************************************/
public class ActivitySpec extends DbPerformanceJsonSpec
{
/*******************************************************************************
** Method and path relative to /qqq/v1.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/db-performance/activity")
.withShortSummary("Database Performance activity").withLongDescription("Requires db_performance.view and uses the caller's session."));
}
/*******************************************************************************
** Versioned response shape.
*******************************************************************************/
@Override
protected String getResponseSchemaName()
{
return ("DbPerformanceActivityV1");
}
/*******************************************************************************
** Execute in the caller's context; also directly testable without transport.
*******************************************************************************/
@Override
public Object handle(JsonInput input) throws Exception
{
return (data().activity(input.getQueryParam("backend"), true));
}
}
@@ -0,0 +1,349 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import java.math.BigDecimal;
import java.sql.Connection;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
/*******************************************************************************
** Read models shared by the API and widgets, under the caller's QQQ permissions.
*******************************************************************************/
public class DbPerformanceData
{
private final DbPerfRuntime runtime;
/*******************************************************************************
** Resolve one configured monitored backend, never arbitrary host credentials.
*******************************************************************************/
public DbPerformanceData(DbPerfRuntime runtime)
{
this.runtime = runtime;
}
/*******************************************************************************
** Explicit permission check even for hosts that disable core action enforcement.
*******************************************************************************/
public void requireView(String backend) throws QException
{
if(QContext.getQSession() == null || !QContext.getQSession().hasPermission(runtime.getConfig().getPermissionBaseName() + ".view"))
{
throw (new QPermissionDeniedException("Database Performance view permission is required."));
}
if(backend != null && !runtime.getConfig().getBackendName().equals(backend))
{
throw (new QBadRequestException("Choose the configured monitored backend."));
}
}
/*******************************************************************************
** Backend equality filter, reused by every backend-owned table.
*******************************************************************************/
public QQueryFilter backendFilter()
{
return (DbPerfStore.eq("backendName", runtime.getConfig().getBackendName()));
}
/*******************************************************************************
** Read settings without creating or changing them during a dashboard request.
*******************************************************************************/
public QRecord settings() throws QException
{
requireView(null);
return (DbPerfStore.first("db_perf_settings", backendFilter()));
}
/*******************************************************************************
** Latest collected numbers, including degraded and failed collection status.
*******************************************************************************/
public QRecord latest() throws QException
{
requireView(null);
return (DbPerfStore.first("db_perf_snapshot", backendFilter().withOrderBy(new QFilterOrderBy("capturedAt", false))));
}
/*******************************************************************************
** Snapshot deltas over the trailing day, ordered for charting.
*******************************************************************************/
public List<QRecord> snapshots() throws QException
{
requireView(null);
return (DbPerfStore.query("db_perf_snapshot", backendFilter().withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN_OR_EQUALS, Instant.now().minusSeconds(86400)))
.withOrderBy(new QFilterOrderBy("capturedAt", true))));
}
/*******************************************************************************
** Ranked statements with literal-free text and canonical history links.
*******************************************************************************/
public List<Map<String, Object>> statements(int limit, String fingerprint) throws QException
{
requireView(null);
QQueryFilter filter = backendFilter().withOrderBy(new QFilterOrderBy("totalMs24h", false)).withLimit(limit);
if(fingerprint != null)
{
filter.withCriteria(new QFilterCriteria("fingerprint", QCriteriaOperator.EQUALS, fingerprint));
}
List<Map<String, Object>> results = new ArrayList<>();
for(QRecord record : DbPerfStore.query("db_perf_statement", filter))
{
Map<String, Object> item = fields(record, "id", "calls24h", "meanMs24h", "totalMs24h", "shareOfDbTime24h");
String text = SqlNormalizer.normalize(record.getValueString("queryText"));
item.put("queryText", fingerprint == null ? text.substring(0, Math.min(300, text.length())) : text);
item.put("historyPath", recordPath("db_perf_statement", record.getValueInteger("id")));
item.put("hitRatio", hitRatio(record));
results.add(item);
}
return (results);
}
/*******************************************************************************
** 24-hour cache hit ratio uses deltas rather than cumulative statement counters.
*******************************************************************************/
private BigDecimal hitRatio(QRecord record) throws QException
{
List<QRecord> rows = DbPerfStore.query("db_perf_query_snapshot", DbPerfStore.eq("statementId", record.getValueInteger("id"))
.withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN_OR_EQUALS, Instant.now().minusSeconds(86400))));
double hits = rows.stream().mapToDouble(row -> number(row, "sharedBlksHit")).sum();
double reads = rows.stream().mapToDouble(row -> number(row, "sharedBlksRead")).sum();
return (hits + reads == 0 ? null : BigDecimal.valueOf(hits / (hits + reads)));
}
/*******************************************************************************
** Open and acknowledged findings stay actionable and are ordered by score.
*******************************************************************************/
public List<QRecord> findings(int limit) throws QException
{
requireView(null);
return (DbPerfStore.query("db_perf_finding", backendFilter()
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("OPEN", "ACKNOWLEDGED")))
.withOrderBy(new QFilterOrderBy("score", false)).withLimit(limit)));
}
/*******************************************************************************
** Small overview response; last snapshot metrics are interval deltas.
*******************************************************************************/
public Map<String, Object> overview(String backend) throws QException
{
requireView(backend);
Map<String, Integer> counts = new LinkedHashMap<>();
List<Map<String, Object>> top = new ArrayList<>();
for(QRecord record : findings(Integer.MAX_VALUE))
{
counts.merge(record.getValueString("severity"), 1, Integer::sum);
if(top.size() < 5)
{
top.add(fields(record, "id", "title", "severity", "score"));
}
}
return (map("settings", fields(settings(), "provider", "statsAccess", "statsAccessMessage", "lastSnapshotAt"),
"snapshot", fields(latest(), "id", "capturedAt", "status", "connectionsTotal", "maxConnections", "cacheHitRatio", "tempBytes", "deadlocks", "statementTotalMs", "statementCalls"),
"topStatements", statements(10, null), "findingsBySeverity", counts, "topFindings", top));
}
/*******************************************************************************
** Link arbitrary SQL shapes back to their normalized statement history.
*******************************************************************************/
public Map<String, Object> lookup(String backend, String sql) throws QException
{
requireView(backend);
if(sql == null || sql.isBlank())
{
throw (new QBadRequestException("Supply SQL to look up."));
}
String normalized = SqlNormalizer.normalize(sql);
String fingerprint = SqlNormalizer.fingerprint(normalized);
return (map("normalizedSql", normalized, "fingerprint", fingerprint, "matches", statements(100, fingerprint)));
}
/*******************************************************************************
** Live activity is read now, not from saved samples, and is never persisted.
*******************************************************************************/
public Map<String, Object> activity(String backend, boolean allowRawText) throws Exception
{
requireView(backend);
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName())))
{
DbStatsProvider provider = DbStatsProviders.forConnection(connection);
return (activity(provider.readSessions(connection), allowRawText));
}
catch(java.sql.SQLException e)
{
throw (new QException("Live activity is unavailable. Check the backend and statistics access."));
}
}
/*******************************************************************************
** Pure projection also used by provider fixture tests; widgets always pass false.
*******************************************************************************/
public Map<String, Object> activity(List<SessionInfo> sessions, boolean allowRawText) throws QException
{
requireView(null);
boolean redacted = !allowRawText || !QContext.getQSession().hasPermission(runtime.getConfig().getPermissionBaseName() + ".view_live_text");
Map<Integer, Set<Integer>> blocked = new LinkedHashMap<>();
for(SessionInfo session : sessions)
{
for(Integer blocker : session.blockedBy())
{
blocked.computeIfAbsent(blocker, key -> new HashSet<>()).add(session.pid());
}
}
List<Map<String, Object>> rows = new ArrayList<>();
List<Map<String, Object>> chains = new ArrayList<>();
Set<Integer> allBlocked = new HashSet<>();
blocked.values().forEach(allBlocked::addAll);
for(SessionInfo session : sessions)
{
String normalized = SqlNormalizer.normalize(session.queryText());
rows.add(map("pid", session.pid(), "user", session.dbUser(), "application", session.applicationName(), "state", session.state(),
"waitEvent", session.waitEvent(), "querySeconds", session.querySeconds(), "xactSeconds", session.xactSeconds(),
"blockedBy", session.blockedBy(), "blockingCount", blocked.getOrDefault(session.pid(), Set.of()).size(),
"fingerprint", SqlNormalizer.fingerprint(normalized), "normalizedText", normalized, "query", redacted ? null : session.queryText(), "liveTextRedacted", redacted));
}
for(Integer root : blocked.keySet())
{
if(!allBlocked.contains(root))
{
Set<Integer> descendants = new java.util.TreeSet<>();
descendants(root, blocked, descendants);
descendants.remove(root);
chains.add(map("rootPid", root, "blockedPids", new ArrayList<>(descendants)));
}
}
return (map("sessions", rows, "blockingChains", chains, "liveTextRedacted", redacted));
}
/*******************************************************************************
** Traverse transitive blocking with cycle protection.
*******************************************************************************/
private static void descendants(Integer pid, Map<Integer, Set<Integer>> blocked, Set<Integer> result)
{
for(Integer child : blocked.getOrDefault(pid, Set.of()))
{
if(result.add(child))
{
descendants(child, blocked, result);
}
}
}
/*******************************************************************************
** One record path builder for API and widgets, including custom app names.
*******************************************************************************/
public String recordPath(String table, Integer id)
{
return ("/app/" + runtime.getConfig().getAppName() + "/" + table + "/" + id);
}
/*******************************************************************************
** Explicit allowlist prevents returning raw counters, configuration or future fields.
*******************************************************************************/
public static Map<String, Object> fields(QRecord record, String... names)
{
Map<String, Object> result = new LinkedHashMap<>();
for(String name : names)
{
result.put(name, record == null ? null : record.getValue(name));
}
return (result);
}
/*******************************************************************************
** Nullable JSON object with stable property order.
*******************************************************************************/
public static Map<String, Object> map(Object... pairs)
{
Map<String, Object> result = new LinkedHashMap<>();
for(int i = 0; i < pairs.length; i += 2)
{
result.put((String) pairs[i], pairs[i + 1]);
}
return (result);
}
/*******************************************************************************
** Nullable statistics arithmetic.
*******************************************************************************/
public static double number(QRecord row, String field)
{
return (row == null || row.getValue(field) == null ? 0 : ((Number) row.getValue(field)).doubleValue());
}
}
@@ -0,0 +1,100 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
import com.kingsrook.qqq.openapi.model.In;
import com.kingsrook.qqq.openapi.model.Method;
import com.kingsrook.qqq.openapi.model.Parameter;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
import io.javalin.http.HttpStatus;
/*******************************************************************************
** Common schemas and error contract, with no alternate route or permission pipeline.
*******************************************************************************/
public abstract class DbPerformanceJsonSpec extends AbstractJsonEndpointSpec
{
/*******************************************************************************
** Stable operation identifiers for generated clients.
*******************************************************************************/
@Override
public Method defineMethod()
{
return (super.defineMethod().withOperationId("dbPerformance" + getClass().getSimpleName().replace("Spec", "")));
}
/*******************************************************************************
** Publish all nested response and request component schemas.
*******************************************************************************/
@Override
public Map<String, Schema> defineComponentSchemas()
{
return (DbPerformanceSchemas.all());
}
/*******************************************************************************
** Backend defaults to the qbit's single configured monitored backend for reads.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
return (List.of(new Parameter().withName("backend").withIn(In.QUERY).withRequired(false)
.withSchema(new Schema().withType(Type.STRING)).withDescription("Configured monitored backend; defaults to the qbit backend.")));
}
/*******************************************************************************
** Framework exceptions map to the same status in every UI client.
*******************************************************************************/
@Override
public List<BasicResponse> defineAdditionalBasicResponses()
{
return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Invalid request", "DbPerformanceErrorV1"),
new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", "DbPerformanceErrorV1"),
new BasicResponse(HttpStatus.FORBIDDEN, "Permission denied", "DbPerformanceErrorV1")));
}
/*******************************************************************************
** Resolve per-request runtime from the authenticated QContext.
*******************************************************************************/
protected DbPerformanceData data()
{
return (new DbPerformanceData(DbPerfRuntime.of(QContext.getQInstance())));
}
}
@@ -0,0 +1,151 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
/*******************************************************************************
** Explicit versioned schemas; every public property is described for generated clients.
*******************************************************************************/
public final class DbPerformanceSchemas
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private DbPerformanceSchemas()
{
}
/*******************************************************************************
** Shared schema components referenced by the four independently versioned operations.
*******************************************************************************/
public static Map<String, Schema> all()
{
Map<String, Schema> schemas = new TreeMap<>();
schemas.put("DbPerformanceErrorV1", object("error", text()));
schemas.put("DbPerformanceSettingsV1", object("provider", text(), "statsAccess", text(), "statsAccessMessage", text(), "lastSnapshotAt", text()));
schemas.put("DbPerformanceSnapshotV1", object("id", integer(), "capturedAt", text(), "status", text(), "connectionsTotal", integer(), "maxConnections", integer(),
"cacheHitRatio", number(), "tempBytes", integer(), "deadlocks", integer(), "statementTotalMs", number(), "statementCalls", integer()));
schemas.put("DbPerformanceStatementV1", object("id", integer(), "queryText", text().withDescription("Normalized SQL, truncated to 300 characters."),
"calls24h", integer(), "meanMs24h", number(), "totalMs24h", number(), "shareOfDbTime24h", number(), "hitRatio", number(), "historyPath", text()));
schemas.put("DbPerformanceFindingV1", object("id", integer(), "title", text(), "severity", text(), "score", number()));
schemas.put("DbPerformanceOverviewV1", object("settings", ref("Settings"), "snapshot", ref("Snapshot"), "topStatements", array(ref("Statement")),
"findingsBySeverity", new Schema().withType(Type.OBJECT).withAdditionalProperties(integer()), "topFindings", array(ref("Finding"))));
schemas.put("DbPerformanceSessionV1", object("pid", integer(), "user", text(), "application", text(), "state", text(), "waitEvent", text(),
"querySeconds", number(), "xactSeconds", number(), "blockedBy", array(integer()), "blockingCount", integer(), "fingerprint", text(), "normalizedText", text(),
"query", text().withDescription("Raw live SQL only with view_live_text; never persisted."), "liveTextRedacted", bool()));
schemas.put("DbPerformanceBlockingChainV1", object("rootPid", integer(), "blockedPids", array(integer())));
schemas.put("DbPerformanceActivityV1", object("sessions", array(ref("Session")), "blockingChains", array(ref("BlockingChain")), "liveTextRedacted", bool()));
schemas.put("DbPerformanceLookupV1", object("normalizedSql", text(), "fingerprint", text(), "matches", array(ref("Statement"))));
schemas.put("DbPerformanceExplainRequestV1", object("backend", text(), "statementId", integer(), "sql", text(),
"mode", new Schema().withType(Type.STRING).withEnumValues(List.of("PLAN", "ANALYZE")), "parameters", array(text()))
.withDescription("Supply exactly one of statementId or sql. Parameters are strings, accepted only for ANALYZE and never stored."));
schemas.put("DbPerformanceExplainResultV1", object("mode", text(), "planJson", text(), "planText", text(), "totalCost", number(), "actualTotalMs", number(),
"warnings", array(text()), "explainId", integer(), "sqlWorkbenchUrl", text(), "assistantPrompt", text()));
return (schemas);
}
/*******************************************************************************
** Nullable scalar for metrics that have not yet been collected.
*******************************************************************************/
private static Schema text()
{
return (new Schema().withType(Type.STRING).withNullable(true));
}
/*******************************************************************************
** Integral count or identifier.
*******************************************************************************/
private static Schema integer()
{
return (new Schema().withType(Type.INTEGER).withNullable(true));
}
/*******************************************************************************
** Fractional duration, cost or ratio.
*******************************************************************************/
private static Schema number()
{
return (new Schema().withType(Type.NUMBER).withNullable(true));
}
/*******************************************************************************
** Required boolean flag.
*******************************************************************************/
private static Schema bool()
{
return (new Schema().withType(Type.BOOLEAN));
}
/*******************************************************************************
** Stable component reference.
*******************************************************************************/
public static Schema ref(String suffix)
{
return (new Schema().withRefToSchema("DbPerformance" + suffix + "V1"));
}
/*******************************************************************************
** A list with an explicitly typed item.
*******************************************************************************/
private static Schema array(Schema item)
{
return (new Schema().withType(Type.ARRAY).withItems(item));
}
/*******************************************************************************
** Deterministically ordered properties for stable OpenAPI generation.
*******************************************************************************/
private static Schema object(Object... pairs)
{
Map<String, Schema> properties = new TreeMap<>();
for(int i = 0; i < pairs.length; i += 2)
{
properties.put((String) pairs[i], (Schema) pairs[i + 1]);
}
return (new Schema().withType(Type.OBJECT).withProperties(properties));
}
}
@@ -0,0 +1,65 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData;
/*******************************************************************************
** Discoverable UI API extension, never raw Javalin routes.
*******************************************************************************/
public final class DbPerformanceUiApi
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private DbPerformanceUiApi()
{
}
/*******************************************************************************
** Add the versioned extension once per host instance.
*******************************************************************************/
public static void register(QInstance instance)
{
MiddlewareSpecExtensionsMetaData.ofOrWithNew(instance).withExtension(extension());
}
/*******************************************************************************
** Public contract used for routing, discovery and committed OpenAPI generation.
*******************************************************************************/
public static MiddlewareSpecExtension extension()
{
return (new MiddlewareSpecExtension().withName("db-performance").withSemanticVersion("1.0.0")
.withDescription("Database performance, live activity and audited query plans")
.withSpec(OverviewSpec.class).withSpec(ActivitySpec.class).withSpec(ExplainSpec.class).withSpec(LookupSpec.class));
}
}
@@ -0,0 +1,152 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import com.kingsrook.qqq.openapi.model.Parameter;
import com.kingsrook.qqq.openapi.model.Schema;
/*******************************************************************************
** Audited plan execution. Service denial is a core permission exception (HTTP 403).
*******************************************************************************/
public class ExplainSpec extends DbPerformanceJsonSpec
{
/*******************************************************************************
** JSON-only POST, with stronger permission for ANALYZE.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/db-performance/explain")
.withShortSummary("Explain a query (PostgreSQL JSON or H2 text plan)").withLongDescription("PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze. Values and ANALYZE plans are never persisted."));
}
/*******************************************************************************
** All inputs are in the body.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
return (List.of());
}
/*******************************************************************************
** Versioned response shape.
*******************************************************************************/
@Override
protected String getResponseSchemaName()
{
return ("DbPerformanceExplainResultV1");
}
/*******************************************************************************
** Versioned request shape.
*******************************************************************************/
@Override
protected Schema defineRequestBodySchema()
{
return (DbPerformanceSchemas.ref("ExplainRequest"));
}
/*******************************************************************************
** Validate JSON types without copying arbitrary values to errors or logs.
*******************************************************************************/
@Override
public Object handle(JsonInput input) throws Exception
{
Map<String, Object> body = input.getBody();
ExplainRequest.Mode mode = null;
try
{
mode = ExplainRequest.Mode.valueOf(string(body, "mode"));
}
catch(IllegalArgumentException | NullPointerException e)
{
// The service audits invalid modes before returning a user-facing error.
}
Integer id = null;
if(body.get("statementId") != null)
{
try
{
id = new java.math.BigDecimal(body.get("statementId").toString()).intValueExact();
}
catch(ArithmeticException | NumberFormatException e)
{
throw (new QBadRequestException("statementId must be an integer."));
}
}
List<String> parameters = new ArrayList<>();
if(body.get("parameters") != null)
{
if(!(body.get("parameters") instanceof List<?> values))
{
throw (new QBadRequestException("parameters must be an array of strings."));
}
for(Object value : values)
{
if(!(value instanceof String))
{
throw (new QBadRequestException("parameters must be an array of strings."));
}
parameters.add((String) value);
}
}
return (new ExplainService(DbPerfRuntime.of(QContext.getQInstance())).explain(new ExplainRequest(string(body, "backend"), id, string(body, "sql"), mode, parameters)));
}
/*******************************************************************************
** Accept strings only, rather than coercing objects into executable SQL.
*******************************************************************************/
private static String string(Map<String, Object> body, String key) throws QBadRequestException
{
Object value = body.get(key);
if(value != null && !(value instanceof String))
{
throw (new QBadRequestException(key + " must be a string."));
}
return ((String) value);
}
}
@@ -0,0 +1,86 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import com.kingsrook.qqq.openapi.model.In;
import com.kingsrook.qqq.openapi.model.Parameter;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
/*******************************************************************************
** Database Performance lookup under the authenticated UI API.
*******************************************************************************/
public class LookupSpec extends DbPerformanceJsonSpec
{
/*******************************************************************************
** Method and path relative to /qqq/v1.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/db-performance/statements/lookup")
.withShortSummary("Database Performance lookup").withLongDescription("Requires db_performance.view and uses the caller's session."));
}
/*******************************************************************************
** Versioned response shape.
*******************************************************************************/
@Override
protected String getResponseSchemaName()
{
return ("DbPerformanceLookupV1");
}
/*******************************************************************************
** Execute in the caller's context; also directly testable without transport.
*******************************************************************************/
@Override
public Object handle(JsonInput input) throws Exception
{
return (data().lookup(input.getQueryParam("backend"), input.getQueryParam("sql")));
}
/*******************************************************************************
** Lookup requires a SQL shape in addition to the optional backend.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
List<Parameter> parameters = new ArrayList<>(super.defineRequestParameters());
parameters.add(new Parameter().withName("sql").withIn(In.QUERY).withRequired(true).withSchema(new Schema().withType(Type.STRING)));
return (parameters);
}
}
@@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.api;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.openapi.model.HttpMethod;
/*******************************************************************************
** Database Performance overview under the authenticated UI API.
*******************************************************************************/
public class OverviewSpec extends DbPerformanceJsonSpec
{
/*******************************************************************************
** Method and path relative to /qqq/v1.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/db-performance/overview")
.withShortSummary("Database Performance overview").withLongDescription("Requires db_performance.view and uses the caller's session."));
}
/*******************************************************************************
** Versioned response shape.
*******************************************************************************/
@Override
protected String getResponseSchemaName()
{
return ("DbPerformanceOverviewV1");
}
/*******************************************************************************
** Execute in the caller's context; also directly testable without transport.
*******************************************************************************/
@Override
public Object handle(JsonInput input) throws Exception
{
return (data().overview(input.getQueryParam("backend")));
}
}
@@ -0,0 +1,270 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.attribution;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListeners;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** One daemon per runtime. Windows are detached before any database operation;
** failed windows are dropped and later windows continue without retry queues.
*******************************************************************************/
public class AttributionFlusher implements AutoCloseable
{
private static final QLogger LOG = QLogger.getLogger(AttributionFlusher.class);
private final DbPerfRuntime runtime;
private final AttributionListener listener;
private ScheduledExecutorService executor;
private Thread shutdownHook;
private boolean closed;
private long failures;
/*******************************************************************************
** Construct without starting threads, for hosts and deterministic tests.
*******************************************************************************/
public AttributionFlusher(DbPerfRuntime runtime)
{
this.runtime = runtime;
this.listener = new AttributionListener(runtime);
}
/*******************************************************************************
** Register once after the instance and schema are ready.
*******************************************************************************/
public synchronized void start()
{
if(executor != null || closed || !Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()))
{
return;
}
refreshSettings();
StatementExecutionListeners.register(listener);
executor = Executors.newSingleThreadScheduledExecutor(task ->
{
Thread thread = new Thread(task, "db-perf-attribution");
thread.setDaemon(true);
return (thread);
});
shutdownHook = new Thread(this::close, "db-perf-attribution-shutdown");
Runtime.getRuntime().addShutdownHook(shutdownHook);
schedule();
}
/*******************************************************************************
** Reschedule after each pass so administrative interval changes take effect.
*******************************************************************************/
private synchronized void schedule()
{
if(!closed)
{
executor.schedule(() ->
{
flushNow();
schedule();
}, listener.settings().flushSeconds(), TimeUnit.SECONDS);
}
}
/*******************************************************************************
** Exactly one InsertAction per nonempty table, with all rows in the window.
** Serialize flushers only; statement threads never acquire this monitor.
*******************************************************************************/
public synchronized void flushNow()
{
boolean previous = AttributionListener.suppress(true);
try
{
Instant end = Instant.now();
AttributionListener.Window window = listener.drain(end);
List<QRecord> attribution = records(window.attribution(), window.start(), end, false);
List<QRecord> repeats = records(window.repeats(), window.start(), end, true);
if(!attribution.isEmpty() || !repeats.isEmpty())
{
runtime.transaction(() ->
{
insert("db_perf_attribution", attribution);
insert("db_perf_repeat_pattern", repeats);
return (null);
});
}
}
catch(Exception e)
{
failed();
}
finally
{
refreshSettings();
AttributionListener.suppress(previous);
}
}
/*******************************************************************************
** Refresh even after failed writes, including while attribution is disabled.
*******************************************************************************/
public synchronized void refreshSettings()
{
boolean previous = AttributionListener.suppress(true);
try
{
listener.refresh(new DbPerfSettingsService(runtime).ensure(runtime.getConfig().getBackendName()));
}
catch(Exception e)
{
failed();
}
finally
{
AttributionListener.suppress(previous);
}
}
/*******************************************************************************
** Rate-limited fixed text; database errors may contain sensitive SQL.
*******************************************************************************/
private void failed()
{
failures++;
if(failures == 1 || failures % 1000 == 0)
{
LOG.warn("Database Performance attribution write or settings refresh failed; the failed window is dropped");
}
}
/*******************************************************************************
** Convert nanoseconds to persisted milliseconds without floating point loss.
*******************************************************************************/
private List<QRecord> records(Map<AttributionListener.Key, AttributionListener.Counters> values, Instant start, Instant end, boolean repeat)
{
List<QRecord> records = new ArrayList<>(values.size());
for(Map.Entry<AttributionListener.Key, AttributionListener.Counters> entry : values.entrySet())
{
AttributionListener.Key key = entry.getKey();
AttributionListener.Counters counts = entry.getValue();
QRecord row = new QRecord().withValue("windowStart", start).withValue("windowEnd", end)
.withValue("podName", truncate(runtime.getConfig().getPodName(), 200)).withValue("unitKind", key.kind())
.withValue("unitName", truncate(key.name(), 300)).withValue("processName", truncate(key.process(), 200))
.withValue("tableName", truncate(key.table(), 200)).withValue("fingerprint", key.fingerprint())
.withValue("sqlText", truncate(counts.sql, 4000)).withValue("sampleRate", counts.sampleRate < 0 ? null : BigDecimal.valueOf(counts.sampleRate));
if(repeat)
{
row.withValue("unitsAffected", counts.units.sum()).withValue("totalRepeats", counts.statements.sum())
.withValue("maxRepeats", (int) Math.min(Integer.MAX_VALUE, counts.max.get()));
}
else
{
row.withValue("operation", key.operation()).withValue("units", counts.units.sum()).withValue("statements", counts.statements.sum())
.withValue("totalMs", BigDecimal.valueOf(counts.nanos.sum(), 6)).withValue("maxMs", BigDecimal.valueOf(counts.max.get(), 6))
.withValue("rowsCount", counts.rows.sum()).withValue("errors", counts.errors.sum());
}
records.add(row);
}
return (records);
}
/*******************************************************************************
** Bound text at the persistence boundary after normalization.
*******************************************************************************/
private static String truncate(String value, int length)
{
return (value == null || value.length() <= length ? value : value.substring(0, length));
}
/*******************************************************************************
** Reject record-level failures so the whole window transaction rolls back.
*******************************************************************************/
private static void insert(String table, List<QRecord> rows) throws Exception
{
if(!rows.isEmpty())
{
List<QRecord> inserted = InsertAction.executeForRecords(new InsertInput(table).withRecords(rows).withTransaction(QContext.getQBackendTransaction()));
if(inserted.size() != rows.size() || inserted.stream().anyMatch(row -> row.getErrors() != null && !row.getErrors().isEmpty()))
{
throw (new IllegalStateException("Attribution insert failed"));
}
}
}
/*******************************************************************************
** Listener exposed for deterministic delivery and registration tests.
*******************************************************************************/
public AttributionListener getListener()
{
return (listener);
}
/*******************************************************************************
** Unregister first, stop scheduling, and flush one final detached window.
*******************************************************************************/
@Override
public synchronized void close()
{
if(closed)
{
return;
}
closed = true;
StatementExecutionListeners.unregister(listener);
if(executor != null)
{
executor.shutdownNow();
}
if(shutdownHook != null && Thread.currentThread() != shutdownHook)
{
try
{
Runtime.getRuntime().removeShutdownHook(shutdownHook);
}
catch(IllegalStateException expected)
{
// JVM shutdown is already running the hook.
}
}
flushNow();
}
}
@@ -0,0 +1,425 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.attribution;
import java.time.Instant;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instrumentation.StatementExecution;
import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListener;
import com.kingsrook.qqq.backend.core.instrumentation.UnitOfWork;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Bounded, in-memory attribution. No table actions run on statement threads.
** Completed units belong to the window in which they end, including long units.
*******************************************************************************/
public class AttributionListener implements StatementExecutionListener
{
private static final String OTHER = "(other)";
private static final Key OVERFLOW = new Key("OTHER", OTHER, OTHER, OTHER, "OTHER", SqlNormalizer.fingerprint(OTHER), -1);
private static final ThreadLocal<Boolean> SUPPRESSED = ThreadLocal.withInitial(() -> false);
private final DbPerfRuntime runtime;
private final ConcurrentHashMap<String, Shape> shapes = new ConcurrentHashMap<>();
private final ThreadLocal<Buffer> current = new ThreadLocal<>();
private volatile Settings settings = new Settings(false, 1.0, 60, 10000, 10);
private Window window = new Window(Instant.now(), new HashMap<>(), new HashMap<>());
/*******************************************************************************
** Bind to one instance and monitored backend.
*******************************************************************************/
public AttributionListener(DbPerfRuntime runtime)
{
this.runtime = runtime;
}
/*******************************************************************************
** Ignore self-observation, then sample and count without database access.
*******************************************************************************/
@Override
public void onStatement(StatementExecution execution)
{
Settings copy = settings;
if(!copy.enabled() || !Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()) || SUPPRESSED.get()
|| !runtime.getConfig().getBackendName().equals(execution.backendName())
|| (execution.tableName() != null && execution.tableName().startsWith(DbPerfQBitConfig.TABLE_NAME_PREFIX))
|| (QContext.getQInstance() != null && QContext.getQInstance() != runtime.getInstance()))
{
return;
}
UnitOfWork unit = execution.unitOfWork();
Buffer buffer = current.get();
if(buffer != null && (unit == null || buffer.unit.getId() != unit.getId()))
{
merge(buffer);
current.remove();
buffer = null;
}
if(unit != null && buffer == null)
{
buffer = new Buffer(unit, copy, sampled(unit.getId(), copy.sampleRate()));
current.set(buffer);
}
if(buffer == null ? !sampled(ThreadLocalRandom.current().nextLong(), copy.sampleRate()) : !buffer.sampled)
{
return;
}
Shape shape = shape(execution.sql());
Key key = new Key(unit == null ? "OTHER" : unit.getKind().name(), unit == null ? name(execution.processName()) : "",
execution.processName(), execution.tableName(), execution.operation() == null ? "OTHER" : execution.operation().name(), shape.fingerprint(), buffer == null ? copy.sampleRate() : buffer.settings.sampleRate());
if(buffer == null)
{
synchronized(this)
{
Key target = bounded(window.attribution(), key, copy.maxKeys());
Counters counts = window.attribution().computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : shape.sql(), key.sampleRate()));
counts.sampleRate(key.sampleRate());
counts.add(execution);
counts.units.increment();
}
}
else
{
Key target = bounded(buffer.counts, key, Math.max(2, buffer.settings.maxKeys()));
buffer.counts.computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : shape.sql(), key.sampleRate())).add(execution);
}
}
/*******************************************************************************
** Read the final route name, then release all per-thread counters.
*******************************************************************************/
@Override
public void onUnitOfWorkEnd(UnitOfWork unit)
{
Buffer buffer = current.get();
if(buffer != null && buffer.unit.getId() == unit.getId())
{
current.remove();
if(settings.enabled() && Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()))
{
merge(buffer);
}
}
}
/*******************************************************************************
** Merge each unit once per destination, even when several shapes overflow.
** Repeat counts are combined across operations before threshold evaluation.
*******************************************************************************/
private synchronized void merge(Buffer buffer)
{
Set<Key> touched = new HashSet<>();
Map<Key, Counters> repeats = new HashMap<>();
for(Map.Entry<Key, Counters> entry : buffer.counts.entrySet())
{
Key local = entry.getKey();
Key key = local == OVERFLOW ? OVERFLOW : new Key(local.kind(), name(buffer.unit.getName()), local.process(), local.table(), local.operation(), local.fingerprint(), buffer.settings.sampleRate());
Key target = bounded(window.attribution(), key, settings.maxKeys());
Counters counts = window.attribution().computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : entry.getValue().sql, entry.getValue().sampleRate));
counts.merge(entry.getValue());
if(touched.add(target))
{
counts.units.increment();
}
// Collapsed shapes cannot prove repetition of any one fingerprint.
if(local != OVERFLOW)
{
Key repeat = new Key(key.kind(), key.name(), key.process(), key.table(), null, key.fingerprint(), key.sampleRate());
repeats.computeIfAbsent(repeat, ignored -> new Counters(entry.getValue().sql, entry.getValue().sampleRate)).merge(entry.getValue());
}
}
Map<Key, Long> totals = new HashMap<>();
Map<Key, Long> maxima = new HashMap<>();
for(Map.Entry<Key, Counters> entry : repeats.entrySet())
{
long count = entry.getValue().statements.sum();
if(count >= buffer.settings.threshold())
{
Key target = bounded(window.repeats(), entry.getKey(), settings.maxKeys());
window.repeats().computeIfAbsent(target, ignored -> new Counters(target == OVERFLOW ? OTHER : entry.getValue().sql, entry.getValue().sampleRate));
window.repeats().get(target).sampleRate(entry.getValue().sampleRate);
totals.merge(target, count, Long::sum);
maxima.merge(target, count, Math::max);
}
}
for(Map.Entry<Key, Long> entry : totals.entrySet())
{
Counters counts = window.repeats().get(entry.getKey());
counts.units.increment();
counts.statements.add(entry.getValue());
counts.max.accumulateAndGet(maxima.get(entry.getKey()), Math::max);
}
}
/*******************************************************************************
** Reserve one slot for a universal overflow key, preserving the strict cap.
*******************************************************************************/
private static Key bounded(Map<Key, Counters> counts, Key key, int limit)
{
return (counts.containsKey(key) || (key != OVERFLOW && counts.size() < limit - (counts.containsKey(OVERFLOW) ? 0 : 1)) ? key : OVERFLOW);
}
/*******************************************************************************
** Normalize once per raw SQL shape; cold-cache admission alone is serialized.
*******************************************************************************/
private Shape shape(String sql)
{
String raw = sql == null ? "" : sql;
Shape cached = shapes.get(raw);
if(cached != null)
{
return (cached);
}
String normalized = SqlNormalizer.normalize(raw);
Shape created = new Shape(normalized, SqlNormalizer.fingerprint(normalized));
synchronized(shapes)
{
if(shapes.size() >= 5000)
{
shapes.clear();
}
shapes.put(raw, created);
}
return (created);
}
/*******************************************************************************
** SplitMix64 finalizer distributes sequential unit IDs without allocations.
*******************************************************************************/
private static boolean sampled(long id, double rate)
{
long hash = (id ^ (id >>> 30)) * 0xbf58476d1ce4e5b9L;
hash = (hash ^ (hash >>> 27)) * 0x94d049bb133111ebL;
hash ^= hash >>> 31;
return ((hash >>> 11) * 0x1.0p-53 < rate);
}
/*******************************************************************************
** Use a stable label for units without a name.
*******************************************************************************/
private static String name(String value)
{
return (value == null || value.isBlank() ? "(none)" : value);
}
/*******************************************************************************
** Detach an immutable-in-practice window; no writers retain its counters.
*******************************************************************************/
synchronized Window drain(Instant end)
{
Window previous = window;
window = new Window(end, new HashMap<>(), new HashMap<>());
return (previous);
}
/*******************************************************************************
** Publish a validated immutable settings copy only from the flusher.
*******************************************************************************/
void refresh(QRecord row)
{
double rate = row.getValueBigDecimal("attributionSampleRate") == null ? 1.0 : row.getValueBigDecimal("attributionSampleRate").doubleValue();
settings = new Settings(!Boolean.FALSE.equals(row.getValueBoolean("attributionEnabled")), Math.max(0, Math.min(1, rate)),
positive(row, "attributionFlushSeconds", 60), positive(row, "attributionMaxKeys", 10000), positive(row, "nPlusOneThreshold", 10));
}
/*******************************************************************************
** Invalid administrative values use defaults.
*******************************************************************************/
private static int positive(QRecord row, String field, int fallback)
{
Integer value = row.getValueInteger(field);
return (value == null || value <= 0 ? fallback : value);
}
/*******************************************************************************
** Current settings for scheduling and window annotations.
*******************************************************************************/
Settings settings()
{
return (settings);
}
/*******************************************************************************
** Set thread suppression, returning the previous state for nested callers.
*******************************************************************************/
static boolean suppress(boolean value)
{
boolean previous = SUPPRESSED.get();
if(value)
{
SUPPRESSED.set(true);
}
else
{
SUPPRESSED.remove();
}
return (previous);
}
/*******************************************************************************
** Immutable administrative settings.
*******************************************************************************/
record Settings(boolean enabled, double sampleRate, int flushSeconds, int maxKeys, int threshold)
{
}
/*******************************************************************************
** Attribution dimensions; sampling transitions remain separately interpretable.
** Null operation denotes a repeat pattern.
*******************************************************************************/
record Key(String kind, String name, String process, String table, String operation, String fingerprint, double sampleRate)
{
}
/*******************************************************************************
** Cached literal-free text and its digest.
*******************************************************************************/
private record Shape(String sql, String fingerprint)
{
}
/*******************************************************************************
** Detached window transferred to the flusher.
*******************************************************************************/
record Window(Instant start, Map<Key, Counters> attribution, Map<Key, Counters> repeats)
{
}
/*******************************************************************************
** Thread-confined unit, including its once-only sampling decision.
*******************************************************************************/
private static class Buffer
{
private final UnitOfWork unit;
private final Settings settings;
private final boolean sampled;
private final Map<Key, Counters> counts = new HashMap<>();
/*******************************************************************************
** Capture unit settings so refresh cannot partially sample a unit.
*******************************************************************************/
private Buffer(UnitOfWork unit, Settings settings, boolean sampled)
{
this.unit = unit;
this.settings = settings;
this.sampled = sampled;
}
}
/*******************************************************************************
** Additive counters plus an atomic maximum, shared by both window tables.
*******************************************************************************/
static class Counters
{
final String sql;
double sampleRate;
final LongAdder units = new LongAdder();
final LongAdder statements = new LongAdder();
final LongAdder nanos = new LongAdder();
final AtomicLong max = new AtomicLong();
final LongAdder rows = new LongAdder();
final LongAdder errors = new LongAdder();
/*******************************************************************************
** Retain only normalized SQL.
*******************************************************************************/
Counters(String sql, double sampleRate)
{
this.sql = sql;
this.sampleRate = sampleRate;
}
/*******************************************************************************
** Count one completed statement, including failures and unknown row counts.
*******************************************************************************/
void add(StatementExecution execution)
{
statements.increment();
nanos.add(execution.elapsedNanos());
max.accumulateAndGet(execution.elapsedNanos(), Math::max);
rows.add(execution.rowCount() == null ? 0 : execution.rowCount());
if(!execution.succeeded())
{
errors.increment();
}
}
/*******************************************************************************
** An overflow bucket spanning sampling rates has no single valid rate.
*******************************************************************************/
void sampleRate(double rate)
{
if(sampleRate != rate)
{
sampleRate = -1;
}
}
/*******************************************************************************
** Fold statement totals; distinct units are managed by the caller.
*******************************************************************************/
void merge(Counters other)
{
sampleRate(other.sampleRate);
statements.add(other.statements.sum());
nanos.add(other.nanos.sum());
max.accumulateAndGet(other.max.get(), Math::max);
rows.add(other.rows.sum());
errors.add(other.errors.sum());
}
}
}
@@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.attribution;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
/*******************************************************************************
** QApplicationLauncher lifecycle, after instance initialization and migrations.
*******************************************************************************/
public class AttributionRuntimeService implements QRuntimeServiceInterface
{
private DbPerfRuntime runtime;
/*******************************************************************************
** Name used by the host launcher.
*******************************************************************************/
@Override
public String getName()
{
return ("dbPerformanceAttribution");
}
/*******************************************************************************
** Start on each application pod, independently of the job scheduler.
*******************************************************************************/
@Override
public void start(QInstance instance)
{
runtime = DbPerfRuntime.of(instance);
runtime.startAttribution();
}
/*******************************************************************************
** Stop listener delivery and persist the final completed units.
*******************************************************************************/
@Override
public void stop()
{
if(runtime != null)
{
runtime.stopAttribution();
}
}
}
@@ -66,6 +66,48 @@ public final class DbPerfStore
}
/*******************************************************************************
** Keyset-paged read for tables that can hold more rows than fit in the heap
** (attribution writes hundreds of thousands per day). At most pageSize
** records are held at once; the consumer must not keep the page.
*******************************************************************************/
public static void forEachPage(String table, QQueryFilter filter, int pageSize, PageConsumer consumer) throws Exception
{
Serializable lastId = 0;
while(true)
{
QQueryFilter page = new QQueryFilter().withSubFilter(filter).withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, lastId))
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(pageSize);
List<QRecord> rows = query(table, page);
if(rows.isEmpty())
{
return;
}
consumer.accept(rows);
if(rows.size() < pageSize)
{
return;
}
lastId = rows.getLast().getValue("id");
}
}
/*******************************************************************************
** Receives one bounded page from forEachPage.
*******************************************************************************/
@FunctionalInterface
public interface PageConsumer
{
/***************************************************************************
** Handle one page.
***************************************************************************/
void accept(List<QRecord> page) throws Exception;
}
/*******************************************************************************
** First matching record.
*******************************************************************************/
@@ -168,7 +168,19 @@ public class RollupService
{
Map<String, QRecord> groups = new LinkedHashMap<>();
List<String> dimensions = List.of("unitKind", "unitName", "processName", "tableName", "operation", "fingerprint");
for(QRecord row : DbPerfStore.query("db_perf_attribution", range("windowStart", start, end)))
DbPerfStore.forEachPage("db_perf_attribution", range("windowStart", start, end), DbPerfStore.BATCH_SIZE, page -> mergeAttribution(day, page, dimensions, groups));
replace("db_perf_attribution_daily", day, new ArrayList<>(groups.values()));
}
/*******************************************************************************
** Fold one page of attribution detail into the day's groups. Memory is
** bounded by distinct dimension keys, never by the day's detail row count.
*******************************************************************************/
private static void mergeAttribution(LocalDate day, List<QRecord> page, List<String> dimensions, Map<String, QRecord> groups)
{
for(QRecord row : page)
{
StringBuilder key = new StringBuilder();
for(String field : dimensions)
@@ -187,7 +199,6 @@ public class RollupService
sumDecimal(aggregate, row, "totalMs");
aggregate.setValue("maxMs", SnapshotCollector.decimal(aggregate, "maxMs").max(SnapshotCollector.decimal(row, "maxMs")));
}
replace("db_perf_attribution_daily", day, new ArrayList<>(groups.values()));
}
@@ -47,7 +47,9 @@ import com.kingsrook.qbits.dbperformance.provider.IndexStats;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
import com.kingsrook.qbits.dbperformance.provider.StatementStats;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qbits.dbperformance.provider.TableStats;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -55,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@@ -62,6 +65,8 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat
*******************************************************************************/
public class SnapshotCollector
{
private static final QLogger LOG = QLogger.getLogger(SnapshotCollector.class);
private final DbPerfRuntime runtime;
private final Clock clock;
private final DbStatsProvider providerOverride;
@@ -104,6 +109,7 @@ public class SnapshotCollector
throw (new IllegalArgumentException("Backend is not configured for this runtime"));
}
QRecord settings = new DbPerfSettingsService(runtime).ensure(backendName);
failAbandonedRuns(backendName, settings);
QRecord inserted = runtime.asSystem(() -> DbPerfStore.insert("db_perf_snapshot", List.of(snapshot)).getFirst());
snapshot.setValue("id", inserted.getValueInteger("id"));
runtime.transaction(() ->
@@ -121,9 +127,19 @@ public class SnapshotCollector
return (null);
});
}
catch(Exception e)
catch(Throwable e)
{
// Never persist a JDBC exception message: it may contain SQL literals.
//////////////////////////////////////////////////////////////////////////
// Errors too (a StackOverflowError once left runs RUNNING with no log). //
// Never persist or log a JDBC exception message: it may contain SQL //
// literals. The class and the throwing frame identify the cause. //
//////////////////////////////////////////////////////////////////////////
if(e instanceof OutOfMemoryError outOfMemory)
{
throw (outOfMemory);
}
LOG.error("Database performance snapshot failed", logPair("backend", backendName), logPair("snapshotId", snapshot.getValueInteger("id")),
logPair("exception", e.getClass().getName()), logPair("at", e.getStackTrace().length == 0 ? null : e.getStackTrace()[0].toString()));
snapshot.withValue("status", "FAILED").withValue("errorMessage", "Collection failed (" + e.getClass().getSimpleName() + "). Check database connectivity, schema and statistics access.")
.withValue("durationMs", (int) Math.min(Integer.MAX_VALUE, (System.nanoTime() - started) / 1000000));
if(snapshot.getValueInteger("id") != null)
@@ -149,6 +165,33 @@ public class SnapshotCollector
}
/*******************************************************************************
** Runs left RUNNING by a crashed pod or an earlier failure are closed as
** FAILED once they are older than three collection intervals.
*******************************************************************************/
private void failAbandonedRuns(String backendName, QRecord settings) throws Exception
{
int interval = positive(settings, "snapshotIntervalMinutes", 5);
Instant cutoff = clock.instant().minus(Duration.ofMinutes(3L * interval));
List<QRecord> abandoned = runtime.asSystem(() -> DbPerfStore.query("db_perf_snapshot", DbPerfStore.eq("backendName", backendName)
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.EQUALS, "RUNNING"))
.withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.LESS_THAN, cutoff))));
if(abandoned.isEmpty())
{
return;
}
List<QRecord> updates = abandoned.stream().map(row -> new QRecord().withValue("id", row.getValueInteger("id")).withValue("status", "FAILED")
.withValue("errorMessage", "Collection did not finish (the run was interrupted or its pod stopped).")).toList();
runtime.asSystem(() ->
{
DbPerfStore.update("db_perf_snapshot", updates);
return (null);
});
LOG.warn("Closed abandoned database performance snapshots", logPair("backend", backendName), logPair("count", updates.size()));
}
/*******************************************************************************
** Statistics read and writes under the ambient system transaction.
*******************************************************************************/
@@ -175,7 +218,7 @@ public class SnapshotCollector
Instant now = snapshot.getValueInstant("capturedAt");
QQueryFilter filter = DbPerfStore.eq("backendName", backendName)
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("OK", "PARTIAL")))
.withCriteria(new QFilterCriteria("rawXactCommit", QCriteriaOperator.IS_NOT_BLANK))
.withCriteria(new QFilterCriteria("databaseSizeBytes", QCriteriaOperator.IS_NOT_BLANK))
.withOrderBy(new QFilterOrderBy("capturedAt", false));
QRecord previous = DbPerfStore.first("db_perf_snapshot", filter);
Instant previousCapture = previous == null ? null : previous.getValueInstant("capturedAt");
@@ -214,8 +257,9 @@ public class SnapshotCollector
}
}
snapshot.withValue("rawStatsReset", database.statsReset()).withValue("statsResetDetected", databaseReset)
.withValue("databaseSizeBytes", database.sizeBytes()).withValue("maxConnections", database.maxConnections())
.withValue("cacheHitRatio", ratio(number(snapshot, "blksHit"), number(snapshot, "blksHit") + number(snapshot, "blksRead")));
.withValue("databaseSizeBytes", database.sizeBytes())
.withValue("cacheHitRatio", capability.has(StatsFeature.CACHE_COUNTERS) ? ratio(number(snapshot, "blksHit"), number(snapshot, "blksHit") + number(snapshot, "blksRead")) : null)
.withValue("maxConnections", capability.has(StatsFeature.MAX_CONNECTIONS) ? database.maxConnections() : null);
if(capability.statements())
{
Instant reset = provider.readStatementResetInfo(connection).statsReset();
@@ -399,8 +443,8 @@ public class SnapshotCollector
QRecord row = new QRecord().withValue("backendName", backend).withValue("schemaName", entry.schemaName()).withValue("tableName", entry.tableName()).withValue("relid", entry.relid()).withValue("lastSeenAt", now);
entry.counters().forEach(row::setValue);
entry.maintenance().forEach(row::setValue);
row.withValue("deadRatio", ratio(number(row, "nDeadTup"), number(row, "nLiveTup") + number(row, "nDeadTup")))
.withValue("seqScanShare", ratio(number(row, "seqScan"), number(row, "seqScan") + number(row, "idxScan")));
row.withValue("deadRatio", entry.counters().containsKey("nDeadTup") ? ratio(number(row, "nDeadTup"), number(row, "nLiveTup") + number(row, "nDeadTup")) : null)
.withValue("seqScanShare", entry.counters().containsKey("seqScan") ? ratio(number(row, "seqScan"), number(row, "seqScan") + number(row, "idxScan")) : null);
if(old == null)
{
inserts.add(row);
@@ -0,0 +1,42 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.explain;
import java.util.List;
/*******************************************************************************
** Transient explain input. Parameters must never be persisted or logged.
*******************************************************************************/
public record ExplainRequest(String backendName, Integer statementId, String sql, Mode mode, List<String> parameters)
{
/*******************************************************************************
** Requested execution mode; GENERIC is selected automatically for PLAN.
*******************************************************************************/
public enum Mode
{
PLAN, ANALYZE
}
}
@@ -0,0 +1,34 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.explain;
import java.math.BigDecimal;
import java.util.List;
/*******************************************************************************
** Transient response; ANALYZE plans are returned only to the authorized caller.
*******************************************************************************/
public record ExplainResult(String mode, String planJson, String planText, BigDecimal totalCost, BigDecimal actualTotalMs,
List<String> warnings, Integer explainId, String sqlWorkbenchUrl, String assistantPrompt)
{
}
@@ -0,0 +1,369 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.explain;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import org.postgresql.core.QueryExecutor;
import org.postgresql.jdbc.PgStatement;
/*******************************************************************************
** Audited, read-only plan execution on a dedicated monitored-backend connection.
** Neither database error messages nor supplied values enter the persistent audit.
*******************************************************************************/
public class ExplainService
{
private final DbPerfRuntime runtime;
/*******************************************************************************
** Use the registered host runtime.
*******************************************************************************/
public ExplainService(DbPerfRuntime runtime)
{
this.runtime = runtime;
}
/*******************************************************************************
** Audit every request, including validation and authorization failures.
*******************************************************************************/
public ExplainResult explain(ExplainRequest request) throws Exception
{
long started = System.nanoTime();
QSession session = QContext.getQSession();
QRecord audit = new QRecord().withValue("requestedAt", Instant.now()).withValue("outcome", "ERROR")
.withValue("requestedBy", session == null || session.getUser() == null ? null : session.getUser().getIdReference())
.withValue("mode", request != null && request.mode() == ExplainRequest.Mode.ANALYZE ? "ANALYZE" : "PLAN")
.withValue("parameterCount", request == null || request.parameters() == null ? 0 : request.parameters().size());
String sql = request == null ? "" : request.sql();
String json = null;
PlanTextRenderer.Plan plan = null;
QException failure = null;
List<String> warnings = List.of();
try
{
String permission = runtime.getConfig().getPermissionBaseName() + ("ANALYZE".equals(audit.getValueString("mode")) ? ".explain_analyze" : ".explain");
if(session == null || !session.hasPermission(permission))
{
throw (new QPermissionDeniedException("Query plan permission required: " + permission));
}
if(request == null || request.mode() == null)
{
throw (new QBadRequestException("Choose PLAN or ANALYZE."));
}
if(!runtime.getConfig().getBackendName().equals(request.backendName()))
{
throw (new QBadRequestException("Choose the configured monitored backend."));
}
if((request.statementId() == null) == (sql == null || sql.isBlank()))
{
throw (new QBadRequestException("Supply either a statementId or SQL, not both."));
}
boolean storedStatement = request.statementId() != null;
if(storedStatement)
{
QRecord row = runtime.asSystem(() -> DbPerfStore.first("db_perf_statement", DbPerfStore.eq("id", request.statementId())));
if(row == null || !request.backendName().equals(row.getValueString("backendName")))
{
throw (new QBadRequestException("The statement was not found on the monitored backend."));
}
audit.setValue("statementId", request.statementId());
sql = row.getValueString("queryText");
}
ExplainSql.Parsed parsed = ExplainSql.parse(sql);
List<String> parameters = request.parameters() == null ? List.of() : request.parameters();
audit.setValue("parameterCount", parsed.parameterCount());
boolean analyze = request.mode() == ExplainRequest.Mode.ANALYZE;
if(analyze)
{
parsed.requireSelect();
if(parameters.size() != parsed.parameterCount())
{
throw (new QBadRequestException("Supply exactly " + parsed.parameterCount() + " parameter values for ANALYZE."));
}
}
else if(!parameters.isEmpty())
{
throw (new QBadRequestException("Parameter values are accepted only for ANALYZE."));
}
boolean generic = !analyze && parsed.parameterCount() > 0;
audit.setValue("mode", analyze ? "ANALYZE" : generic ? "GENERIC" : "PLAN");
QRecord settings = new DbPerfSettingsService(runtime).ensure(request.backendName());
Explained explained = execute(parsed, parameters, analyze, generic, SnapshotCollector.positive(settings, "explainTimeoutMs", 10000));
json = explained.json();
plan = explained.plan();
if(!analyze && (storedStatement || sql.equals(SqlNormalizer.normalize(sql))))
{
audit.setValue("planJson", json);
audit.setValue("planText", plan.text());
audit.setValue("totalCost", plan.totalCost());
}
else
{
warnings = List.of("This plan is transient; only a literal-free request audit is stored.");
}
audit.setValue("outcome", "OK");
}
catch(QPermissionDeniedException e)
{
audit.setValue("outcome", "DENIED");
failure = e;
}
catch(QBadRequestException e)
{
failure = e;
}
catch(Exception e)
{
boolean timeout = isTimeout(e);
audit.setValue("outcome", timeout ? "TIMEOUT" : "ERROR");
failure = new QException(timeout ? "Query plan timed out. Reduce the query scope or ask an administrator to review the explain timeout."
: "Query plan failed. Check SQL, parameter types and database access. Writes are not allowed.");
}
finally
{
String normalized = SqlNormalizer.normalize(sql);
audit.setValue("sqlText", normalized);
audit.setValue("sqlFingerprint", SqlNormalizer.fingerprint(normalized));
audit.setValue("durationMs", (System.nanoTime() - started) / 1000000);
audit.setValue("errorMessage", failure == null ? null : failure.getMessage());
QRecord inserted = runtime.asSystem(() -> DbPerfStore.insert("db_perf_explain", List.of(audit)).getFirst());
audit.setValue("id", inserted.getValueInteger("id"));
}
if(failure != null)
{
throw (failure);
}
return (new ExplainResult(audit.getValueString("mode"), json, plan.text(), plan.totalCost(), plan.actualTotalMs(), warnings,
audit.getValueInteger("id"), runtime.getConfig().getSqlWorkbenchRunnerPath() + "?sql=" + encode("EXPLAIN " + sql) + "&backend=" + encode(request.backendName()),
"Explain this " + (json == null ? "database" : "PostgreSQL") + " query plan and suggest improvements.\n\nQuery:\n" + SqlNormalizer.normalize(sql) + "\n\nPlan:\n" + plan.safeText()));
}
/*******************************************************************************
** Roll back even on parse, execution or timeout failure, deallocate after rollback,
** and restore pool state before releasing this dedicated connection.
*******************************************************************************/
private Explained execute(ExplainSql.Parsed parsed, List<String> parameters, boolean analyze, boolean generic, int timeout) throws Exception
{
try(Connection connection = openConnection())
{
if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName()))
{
//////////////////////////////////////////////////////////////////
// other vendors plan through their provider as text (no JSON) //
//////////////////////////////////////////////////////////////////
DbStatsProvider provider = DbStatsProviders.forConnection(connection);
ProviderCapabilities capability = provider.capabilities(connection);
if(!capability.has(StatsFeature.EXPLAIN_TEXT) || generic && !capability.has(StatsFeature.EXPLAIN_GENERIC))
{
throw (new QBadRequestException("Query plans are not available for this database (" + provider.getName() + ")."));
}
if(analyze && !capability.has(StatsFeature.EXPLAIN_ANALYZE))
{
throw (new QBadRequestException("ANALYZE is not available on this " + provider.getName() + " connection (it would execute the query without a read-only transaction). Use PLAN."));
}
String text;
try
{
text = provider.explainText(connection, parsed.indexedSql(), parameters, analyze, timeout);
}
catch(java.sql.SQLException e)
{
if("42000".equals(e.getSQLState()) && e.getMessage() != null && e.getMessage().contains("H2"))
{
throw (new QBadRequestException(e.getMessage()));
}
throw (e);
}
return (new Explained(null, new PlanTextRenderer.Plan(text, PlanTextRenderer.redactText(text), null, null)));
}
return (new Explained(postgres(connection, parsed.sql(), parameters, analyze, generic, timeout), null).rendered());
}
}
/*******************************************************************************
** PostgreSQL JSON plan; see {@link #execute}.
*******************************************************************************/
private String postgres(Connection connection, String sql, List<String> parameters, boolean analyze, boolean generic, int timeout) throws Exception
{
if(generic && connection.getMetaData().getDatabaseMajorVersion() < 16)
{
throw (new QBadRequestException("Generic plans require PostgreSQL 16 or later. Use ANALYZE with parameter values if authorized."));
}
boolean autoCommit = connection.getAutoCommit();
boolean readOnly = connection.isReadOnly();
String preparedName = "dbperf_" + UUID.randomUUID().toString().replace("-", "");
boolean prepared = false;
try
{
connection.setAutoCommit(false);
try(Statement statement = connection.createStatement())
{
statement.execute("SET TRANSACTION READ ONLY");
statement.execute("SET LOCAL statement_timeout = " + timeout);
statement.execute("SET LOCAL standard_conforming_strings = on");
if(!analyze)
{
// GENERIC_PLAN must not be parsed as an external JDBC bind request.
statement.unwrap(PgStatement.class).executeWithFlags("EXPLAIN (" + (generic ? "GENERIC_PLAN, " : "") + "FORMAT JSON) " + sql, QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
try(ResultSet rows = statement.getResultSet())
{
rows.next();
return (rows.getString(1));
}
}
statement.execute("PREPARE " + preparedName + " AS " + sql);
prepared = true;
}
String arguments = parameters.isEmpty() ? "" : "(" + String.join(", ", Collections.nCopies(parameters.size(), "?")) + ")";
try(PreparedStatement statement = connection.prepareStatement("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE " + preparedName + arguments))
{
for(int i = 0; i < parameters.size(); i++)
{
statement.setObject(i + 1, parameters.get(i), Types.OTHER);
}
// EXECUTE's arguments are utility expressions: unknown bind types cannot
// be inferred in extended Parse. Keep JDBC binding/escaping, but send this
// one command in simple mode so PostgreSQL coerces the quoted strings.
statement.unwrap(PgStatement.class).executeWithFlags(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
try(ResultSet rows = statement.getResultSet())
{
rows.next();
return (rows.getString(1));
}
}
}
finally
{
try
{
connection.rollback();
if(prepared)
{
try(Statement cleanup = connection.createStatement())
{
cleanup.execute("DEALLOCATE " + preparedName);
}
finally
{
connection.rollback();
}
}
}
finally
{
connection.setReadOnly(readOnly);
connection.setAutoCommit(autoCommit);
}
}
}
/*******************************************************************************
** Raw plan (JSON for PostgreSQL, null for text-plan vendors) and its rendering.
*******************************************************************************/
private record Explained(String json, PlanTextRenderer.Plan plan)
{
/*******************************************************************************
** Render a PostgreSQL JSON plan.
*******************************************************************************/
Explained rendered() throws Exception
{
return (new Explained(json, PlanTextRenderer.render(json)));
}
}
/*******************************************************************************
** Always obtain a dedicated connection, independent of any caller QQQ transaction.
** Protected for contract tests that verify pool state restoration on driver errors.
*******************************************************************************/
protected Connection openConnection() throws SQLException
{
RDBMSBackendMetaData backend = (RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName());
return (ConnectionManager.getConnection(backend));
}
/*******************************************************************************
** SQLSTATE survives JDBC wrapper exceptions without exposing their messages.
*******************************************************************************/
private static boolean isTimeout(Throwable exception)
{
for(Throwable current = exception; current != null; current = current.getCause())
{
if(current instanceof SQLException sql && "57014".equals(sql.getSQLState()))
{
return (true);
}
}
return (false);
}
/*******************************************************************************
** UTF-8 query parameter encoding.
*******************************************************************************/
private static String encode(String value)
{
return (URLEncoder.encode(value, StandardCharsets.UTF_8));
}
}
@@ -0,0 +1,280 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.explain;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
/*******************************************************************************
** Lex executable SQL without changing literals, identifiers or parameter positions.
** This is a conservative SELECT guard; PostgreSQL READ ONLY is the write boundary.
*******************************************************************************/
public final class ExplainSql
{
private static final Pattern DOLLAR_QUOTE = Pattern.compile("\\$(?:[A-Za-z_][A-Za-z_0-9]*)?\\$");
/*******************************************************************************
** Utility class.
*******************************************************************************/
private ExplainSql()
{
}
/*******************************************************************************
** Executable SQL with contiguous PostgreSQL bind slots and unquoted tokens,
** plus the same SQL with JDBC-indexed ?n slots for vendors such as H2.
*******************************************************************************/
public record Parsed(String sql, List<String> tokens, int parameterCount, String indexedSql)
{
/*******************************************************************************
** PostgreSQL-only form (no JDBC-indexed variant).
*******************************************************************************/
public Parsed(String sql, List<String> tokens, int parameterCount)
{
this(sql, tokens, parameterCount, sql);
}
/*******************************************************************************
** Reject writes, SELECT INTO and every locking SELECT, including nested CTEs.
*******************************************************************************/
public void requireSelect() throws QBadRequestException
{
if(tokens.isEmpty() || !Set.of("SELECT", "WITH").contains(tokens.getFirst())
|| tokens.stream().anyMatch(Set.of("INSERT", "UPDATE", "DELETE", "MERGE", "INTO", "SHARE")::contains))
{
throw (new QBadRequestException("ANALYZE requires a single SELECT without writes, INTO or row locks."));
}
}
}
/*******************************************************************************
** Skip SQL comments and quoted regions, reject multiple statements, and convert ?.
** Mixing JDBC and PostgreSQL parameter styles is rejected as ambiguous.
*******************************************************************************/
public static Parsed parse(String sql) throws QBadRequestException
{
if(sql == null || sql.isBlank() || sql.length() > 100000)
{
throw (new QBadRequestException("Supply one SQL statement of at most 100000 characters."));
}
StringBuilder result = new StringBuilder();
List<String> tokens = new ArrayList<>();
Set<Integer> positions = new HashSet<>();
List<int[]> slots = new ArrayList<>();
int jdbc = 0;
int i = 0;
boolean ended = false;
while(i < sql.length())
{
int start = i;
char c = sql.charAt(i);
if(Character.isWhitespace(c))
{
result.append(c);
i++;
continue;
}
if(sql.startsWith("--", i))
{
int end = sql.indexOf('\n', i);
i = end < 0 ? sql.length() : end;
result.append(' ');
continue;
}
if(sql.startsWith("/*", i))
{
int depth = 1;
i += 2;
while(i < sql.length() && depth > 0)
{
if(sql.startsWith("/*", i))
{
depth++;
i += 2;
}
else if(sql.startsWith("*/", i))
{
depth--;
i += 2;
}
else
{
i++;
}
}
if(depth != 0)
{
throw (new QBadRequestException("Unterminated SQL comment."));
}
result.append(' ');
continue;
}
if(ended)
{
throw (new QBadRequestException("Only one SQL statement is allowed."));
}
if(c == ';')
{
ended = true;
i++;
continue;
}
boolean escaped = (c == 'e' || c == 'E') && i + 1 < sql.length() && sql.charAt(i + 1) == '\'';
if(c == '\'' || c == '"' || escaped)
{
char quote = escaped ? '\'' : c;
i = quotedEnd(sql, i + (escaped ? 1 : 0), quote, escaped);
result.append(sql, start, i);
tokens.add("<QUOTED>");
continue;
}
if(c == '$')
{
Matcher delimiter = DOLLAR_QUOTE.matcher(sql).region(i, sql.length());
if(delimiter.lookingAt())
{
int end = sql.indexOf(delimiter.group(), delimiter.end());
if(end < 0)
{
throw (new QBadRequestException("Unterminated dollar-quoted SQL string."));
}
i = end + delimiter.group().length();
result.append(sql, start, i);
tokens.add("<QUOTED>");
continue;
}
i++;
while(i < sql.length() && Character.isDigit(sql.charAt(i)))
{
i++;
}
if(i > start + 1)
{
try
{
int position = Integer.parseInt(sql.substring(start + 1, i));
if(position < 1 || position > 65535)
{
throw (new NumberFormatException());
}
positions.add(position);
}
catch(NumberFormatException e)
{
throw (new QBadRequestException("SQL parameter positions must be between 1 and 65535."));
}
}
if(i > start + 1)
{
slots.add(new int[] {result.length(), result.length() + (i - start), Integer.parseInt(sql.substring(start + 1, i))});
}
result.append(sql, start, i);
continue;
}
if(c == '?')
{
// PostgreSQL's multi-character JSON operators are not JDBC bind slots.
if(i + 1 < sql.length() && "|&?".indexOf(sql.charAt(i + 1)) >= 0)
{
result.append(sql, i, i + 2);
i += 2;
}
else
{
String slot = "$" + (++jdbc);
slots.add(new int[] {result.length(), result.length() + slot.length(), jdbc});
result.append(slot);
i++;
}
continue;
}
if(Character.isLetter(c) || c == '_')
{
i++;
while(i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || "_$".indexOf(sql.charAt(i)) >= 0))
{
i++;
}
tokens.add(sql.substring(start, i).toUpperCase(Locale.ROOT));
result.append(sql, start, i);
continue;
}
result.append(c);
i++;
}
int maximum = positions.stream().mapToInt(Integer::intValue).max().orElse(0);
if(jdbc > 0 && maximum > 0 || maximum != positions.size())
{
throw (new QBadRequestException("Use either ? parameters or contiguous $1 through $n parameters."));
}
StringBuilder indexed = new StringBuilder(result);
for(int slot = slots.size() - 1; slot >= 0; slot--)
{
indexed.replace(slots.get(slot)[0], slots.get(slot)[1], "?" + slots.get(slot)[2]);
}
return (new Parsed(result.toString().trim(), List.copyOf(tokens), Math.max(jdbc, maximum), indexed.toString().trim()));
}
/*******************************************************************************
** A quoted region cannot introduce executable keywords or statement delimiters.
*******************************************************************************/
private static int quotedEnd(String sql, int start, char quote, boolean escaped) throws QBadRequestException
{
int i = start + 1;
while(i < sql.length())
{
if(escaped && sql.charAt(i) == '\\')
{
i += 2;
}
else if(sql.charAt(i++) == quote)
{
if(i < sql.length() && sql.charAt(i) == quote)
{
i++;
}
else
{
return (i);
}
}
}
throw (new QBadRequestException("Unterminated SQL quote."));
}
}
@@ -0,0 +1,137 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.explain;
import java.math.BigDecimal;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
/*******************************************************************************
** Readable PostgreSQL JSON plan tree, with a literal-free prompt variant.
*******************************************************************************/
public final class PlanTextRenderer
{
private static final ObjectMapper MAPPER = new ObjectMapper();
/*******************************************************************************
** Utility class.
*******************************************************************************/
private PlanTextRenderer()
{
}
/*******************************************************************************
** A parsed plan and its top-level measures.
*******************************************************************************/
public record Plan(String text, String safeText, BigDecimal totalCost, BigDecimal actualTotalMs)
{
}
/*******************************************************************************
** Literal-free form of a vendor text plan for prompts: quoted strings are
** redacted everywhere, numbers outside comments (H2 keeps index names and
** scan counts inside comments).
*******************************************************************************/
public static String redactText(String text)
{
if(text == null)
{
return (null);
}
StringBuilder result = new StringBuilder();
java.util.regex.Matcher comment = java.util.regex.Pattern.compile("/\\*.*?\\*/", java.util.regex.Pattern.DOTALL).matcher(text);
int last = 0;
while(comment.find())
{
result.append(redactLiterals(text.substring(last, comment.start()), true)).append(redactLiterals(comment.group(), false));
last = comment.end();
}
return (result.append(redactLiterals(text.substring(last), true)).toString());
}
/*******************************************************************************
** Quoted strings, and optionally standalone numbers, become ?.
*******************************************************************************/
private static String redactLiterals(String text, boolean numbers)
{
String result = text.replaceAll("'(?:[^']|'')*'", "?");
return (numbers ? result.replaceAll("(?<![\\w?$.\"])\\d+(?:\\.\\d+)?(?![\\w\"])", "?") : result);
}
/*******************************************************************************
** PostgreSQL emits an array containing the plan and optional execution timing.
*******************************************************************************/
public static Plan render(String json) throws Exception
{
JsonNode document = MAPPER.readTree(json);
JsonNode root = document.isArray() ? document.path(0) : document;
JsonNode plan = root.path("Plan");
if(!plan.isObject())
{
throw (new IllegalArgumentException("PostgreSQL did not return a plan tree."));
}
StringBuilder text = new StringBuilder();
StringBuilder safe = new StringBuilder();
append(plan, 0, text, false);
append(plan, 0, safe, true);
JsonNode actual = root.has("Execution Time") ? root.get("Execution Time") : plan.get("Actual Total Time");
return (new Plan(text.toString().stripTrailing(), safe.toString().stripTrailing(),
plan.has("Total Cost") ? plan.get("Total Cost").decimalValue() : null, actual == null ? null : actual.decimalValue()));
}
/*******************************************************************************
** Preserve tree structure and planner metrics; redact expression literals for AI.
*******************************************************************************/
private static void append(JsonNode node, int depth, StringBuilder out, boolean redact)
{
out.append(" ".repeat(depth)).append(node.path("Node Type").asText("Unknown"));
for(String field : List.of("Relation Name", "Index Name", "Startup Cost", "Total Cost", "Plan Rows", "Actual Total Time", "Actual Rows", "Filter", "Index Cond", "Hash Cond", "Join Filter", "Recheck Cond"))
{
if(node.has(field))
{
String value = node.get(field).asText();
boolean expression = field.endsWith("Cond") || field.endsWith("Filter");
out.append(" | ").append(field).append(": ").append(redact && expression ? SqlNormalizer.normalize(value) : value);
}
}
out.append('\n');
for(JsonNode child : node.path("Plans"))
{
append(child, depth + 1, out, redact);
}
}
}
@@ -0,0 +1,144 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.explain;
import java.util.Arrays;
import java.util.List;
import com.kingsrook.qbits.dbperformance.DbPerfPermissions;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
/*******************************************************************************
** The one-record Query plan action; user-supplied values remain transient.
*******************************************************************************/
public class QueryPlanProcess implements BackendStep
{
public static final String NAME = "dbPerfQueryPlan";
/*******************************************************************************
** Register form, execution and copyable result fields, without a navigation item.
*******************************************************************************/
public static void register(QInstance instance, DbPerfQBitConfig config)
{
instance.addPossibleValueSource(new QPossibleValueSource().withName("db_perf_ExplainRequestMode").withType(QPossibleValueSourceType.ENUM)
.withEnumValues(List.of(new QPossibleValue<>("PLAN", "PLAN"), new QPossibleValue<>("ANALYZE", "ANALYZE"))));
instance.addProcess(new QProcessMetaData().withName(NAME).withLabel("Query plan").withTableName("db_perf_statement")
.withMinInputRecords(1).withMaxInputRecords(1)
.withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".explain"))
.withStep(new QFrontendStepMetaData().withName("review").withLabel("Review")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormFields(List.of(new QFieldMetaData("mode", QFieldType.STRING).withLabel("Mode").withPossibleValueSourceName("db_perf_ExplainRequestMode").withDefaultValue("PLAN").withIsRequired(true),
new QFieldMetaData("parameterValues", QFieldType.TEXT).withLabel("Parameter values (ANALYZE only, one per line)").withIsSensitive(true))))
.withStep(new QBackendStepMetaData().withName("explain").withCode(new QCodeReference(QueryPlanProcess.class))
.withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName("db_perf_statement"))))
.withStep(new QFrontendStepMetaData().withName("result").withLabel("Result")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withViewField(code("planText", "Query plan"))
.withViewField(new QFieldMetaData("totalCost", QFieldType.DECIMAL).withLabel("Total cost"))
.withViewField(new QFieldMetaData("sqlWorkbenchUrl", QFieldType.STRING).withLabel("Open in SQL Workbench")
.withFieldAdornment(new FieldAdornment(AdornmentType.LINK)))
.withViewField(code("assistantPrompt", "Copy to assistant"))));
}
/*******************************************************************************
** Monospace text that remains selectable and copyable.
*******************************************************************************/
private static QFieldMetaData code(String name, String label)
{
return (new QFieldMetaData(name, QFieldType.TEXT).withLabel(label)
.withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("text"))));
}
/*******************************************************************************
** Re-check permissions inside the service even when invoked without UI middleware.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
try
{
if(input.getRecords() == null || input.getRecords().size() != 1)
{
throw (new QException("Select exactly one statement."));
}
DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance());
String values = input.getValueString("parameterValues");
List<String> parameters = values == null || values.isEmpty() ? List.of() : Arrays.asList(values.split("\\R", -1));
ExplainRequest.Mode mode;
try
{
mode = ExplainRequest.Mode.valueOf(input.getValueString("mode") == null ? "PLAN" : input.getValueString("mode"));
}
catch(IllegalArgumentException e)
{
mode = null;
}
ExplainResult result = new ExplainService(runtime).explain(new ExplainRequest(runtime.getConfig().getBackendName(),
input.getRecords().getFirst().getValueInteger("id"), null, mode, parameters));
output.addValue("planText", result.planText());
output.addValue("totalCost", result.totalCost());
output.addValue("sqlWorkbenchUrl", result.sqlWorkbenchUrl());
output.addValue("assistantPrompt", result.assistantPrompt());
}
catch(QException e)
{
throw (e);
}
catch(Exception e)
{
throw (new QException("Unable to complete the query plan request."));
}
finally
{
input.getValues().remove("parameterValues");
output.getValues().remove("parameterValues");
}
}
}
@@ -27,6 +27,7 @@ import java.util.List;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
@@ -111,15 +112,15 @@ public class DbPerfHealthChecks
return (HealthCheckOutcome.unknown("No statistics access observation yet"));
}
String user = ((RDBMSBackendMetaData) runtime.getInstance().getBackend(backend)).getUsername();
String remedy = DbStatsProviders.forProviderName(settings.getValueString("provider")).remedy(access, user);
HealthCheckOutcome outcome = switch(access)
{
case "FULL" -> HealthCheckOutcome.up("Full statistics access");
case "NO_READ_ALL_STATS" -> HealthCheckOutcome.degraded("grant pg_read_all_stats to " + user);
case "NO_PG_STAT_STATEMENTS" -> HealthCheckOutcome.down("Enable pg_stat_statements in shared_preload_libraries and create the extension");
case "UNSUPPORTED" -> HealthCheckOutcome.down("Use a supported PostgreSQL version (13 or newer) and statistics provider");
case "NO_READ_ALL_STATS", "NOT_ADMIN", "NO_QUERY_STATISTICS" -> HealthCheckOutcome.degraded(remedy);
case "NO_PG_STAT_STATEMENTS", "UNSUPPORTED" -> HealthCheckOutcome.down(remedy);
default -> HealthCheckOutcome.unknown("Statistics access has not been classified");
};
String safe = List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "UNSUPPORTED").contains(access) ? access : "UNKNOWN";
String safe = List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "NO_QUERY_STATISTICS", "NOT_ADMIN", "UNSUPPORTED").contains(access) ? access : "UNKNOWN";
return (outcome.withDetails(new HealthDetails().withValue("statsAccess", safe)));
}
var filter = DbPerfStore.eq("backendName", backend).withOrderBy(new QFilterOrderBy("capturedAt", false));
@@ -0,0 +1,94 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.metadata;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundle;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundleEntry;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
/*******************************************************************************
** Data-driven app/table help, also visible when the host has no help-content tables.
*******************************************************************************/
public final class DbPerfHelp
{
public static final String RESOURCE = "help/qbit-db-performance.yaml";
/*******************************************************************************
** Utility class.
*******************************************************************************/
private DbPerfHelp()
{
}
/*******************************************************************************
** Register the Maven-versioned bundle and fill only unclaimed help slots.
*******************************************************************************/
public static void register(QInstance instance, DbPerfQBitConfig config)
{
HelpBundles.register(instance, RESOURCE);
HelpBundle bundle = HelpBundles.of(instance).getBundles().getLast();
for(HelpBundleEntry entry : bundle.getEntries())
{
String key = entry.getKey();
if(key.startsWith("app:dbPerformance;"))
{
entry.setKey(key.replace("app:dbPerformance;", "app:" + config.getAppName() + ";"));
}
if(key.startsWith("table:") && key.contains(";slot:"))
{
String name = key.substring(6, key.indexOf(';'));
String slot = key.substring(key.indexOf(";slot:") + 6);
var table = instance.getTable(name);
if(table != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(slot)))
{
table.withHelpContent(slot, content(entry));
}
}
else if(key.equals("app:dbPerformance;slot:help"))
{
var app = instance.getApp(config.getAppName());
if(app.getHelpContent() == null || !app.getHelpContent().containsKey("help"))
{
app.withHelpContent("help", content(entry));
}
}
}
}
/*******************************************************************************
** Render a concise summary followed by the full data-driven markdown explanation.
*******************************************************************************/
private static QHelpContent content(HelpBundleEntry entry)
{
return (new QHelpContent().withContentAsMarkdown(entry.getSummary() + "\n\n" + (entry.getContent() == null ? "" : entry.getContent())));
}
}
@@ -23,6 +23,7 @@ package com.kingsrook.qbits.dbperformance.metadata;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dbperformance.DbPerfPermissions;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qbits.dbperformance.model.DbPerfActivitySample;
@@ -549,9 +550,10 @@ public final class DbPerfMetaData
repeatPattern.getField("unitsAffected").setBackendName("units_affected");
repeatPattern.getField("maxRepeats").setBackendName("max_repeats");
repeatPattern.getField("totalRepeats").setBackendName("total_repeats");
repeatPattern.getField("sampleRate").setBackendName("sample_rate");
repeatPattern.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
repeatPattern.withSection(new QFieldSection("identity", new QIcon("table_rows"), Tier.T1, List.of("id", "windowStart", "windowEnd", "podName", "unitKind", "unitName", "processName")));
repeatPattern.withSection(new QFieldSection("details", new QIcon("table_rows"), Tier.T2, List.of("tableName", "fingerprint", "sqlText", "unitsAffected", "maxRepeats", "totalRepeats")));
repeatPattern.withSection(new QFieldSection("details", new QIcon("table_rows"), Tier.T2, List.of("tableName", "fingerprint", "sqlText", "unitsAffected", "maxRepeats", "totalRepeats", "sampleRate")));
instance.addTable(repeatPattern);
QTableMetaData finding = base(config, DbPerfFinding.TABLE_NAME, "Findings", DbPerfFinding.class);
@@ -668,7 +670,7 @@ public final class DbPerfMetaData
child(instance, "db_perf_statement", "db_perf_finding", "statementId");
child(instance, "db_perf_table_stat", "db_perf_finding", "tableStatId");
child(instance, "db_perf_statement", "db_perf_explain", "statementId");
QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel("Database Performance").withIcon(new QIcon("speed"))
QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel("Database Performance").withIcon(new QIcon("storage"))
.withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"))
.withSection(new QAppSection().withName("data").withLabel("Data").withIcon(new QIcon("table_rows"))
.withTables(List.of("db_perf_finding", "db_perf_statement", "db_perf_table_stat", "db_perf_index_stat", "db_perf_snapshot", "db_perf_settings", "db_perf_rule")));
@@ -680,12 +682,36 @@ public final class DbPerfMetaData
}
/*******************************************************************************
** A distinct icon per table, so nav entries are told apart at a glance.
*******************************************************************************/
static final Map<String, String> TABLE_ICONS = Map.ofEntries(
Map.entry("db_perf_finding", "tips_and_updates"),
Map.entry("db_perf_statement", "query_stats"),
Map.entry("db_perf_table_stat", "grid_on"),
Map.entry("db_perf_index_stat", "bookmarks"),
Map.entry("db_perf_snapshot", "photo_camera"),
Map.entry("db_perf_settings", "settings_suggest"),
Map.entry("db_perf_rule", "playlist_add_check"),
Map.entry("db_perf_explain", "device_hub"),
Map.entry("db_perf_query_snapshot", "timeline"),
Map.entry("db_perf_query_daily", "calendar_month"),
Map.entry("db_perf_table_snapshot", "stacked_line_chart"),
Map.entry("db_perf_table_daily", "date_range"),
Map.entry("db_perf_activity_sample", "monitor_heart"),
Map.entry("db_perf_connection_sample", "lan"),
Map.entry("db_perf_attribution", "hub"),
Map.entry("db_perf_attribution_daily", "event_note"),
Map.entry("db_perf_repeat_pattern", "repeat"));
/*******************************************************************************
** Common storage, permissions and write guard.
*******************************************************************************/
private static QTableMetaData base(DbPerfQBitConfig config, String name, String label, Class<? extends QRecordEntity> entity) throws QException
{
QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withIcon(new QIcon("speed"))
QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withIcon(new QIcon(TABLE_ICONS.getOrDefault(name, "table_rows")))
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
.withPrimaryKeyField("id").withFieldsFromEntity(entity)
.withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"));
@@ -43,7 +43,7 @@ public final class DbPerfPossibleValues
*******************************************************************************/
public enum Provider implements PossibleValueEnum<String>
{
POSTGRES, MYSQL, UNSUPPORTED;
POSTGRES, H2, MYSQL, UNSUPPORTED;
/*******************************************************************************
@@ -73,7 +73,7 @@ public final class DbPerfPossibleValues
*******************************************************************************/
public enum StatsAccess implements PossibleValueEnum<String>
{
FULL, NO_READ_ALL_STATS, NO_PG_STAT_STATEMENTS, UNSUPPORTED;
FULL, NO_READ_ALL_STATS, NO_PG_STAT_STATEMENTS, NO_QUERY_STATISTICS, NOT_ADMIN, UNSUPPORTED;
/*******************************************************************************
@@ -21,6 +21,7 @@
package com.kingsrook.qbits.dbperformance.model;
import java.math.BigDecimal;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -73,6 +74,9 @@ public class DbPerfRepeatPattern extends QRecordEntity
@QField
private Long totalRepeats;
@QField
private BigDecimal sampleRate;
/*******************************************************************************
** Default constructor.
@@ -453,4 +457,31 @@ public class DbPerfRepeatPattern extends QRecordEntity
this.totalRepeats = totalRepeats;
return (this);
}
/*******************************************************************************
** Getter for sampleRate.
*******************************************************************************/
public BigDecimal getSampleRate()
{
return (sampleRate);
}
/*******************************************************************************
** Setter for sampleRate.
*******************************************************************************/
public void setSampleRate(BigDecimal sampleRate)
{
this.sampleRate = sampleRate;
}
/*******************************************************************************
** Fluent setter for sampleRate.
*******************************************************************************/
public DbPerfRepeatPattern withSampleRate(BigDecimal sampleRate)
{
this.sampleRate = sampleRate;
return (this);
}
}
@@ -24,7 +24,10 @@ package com.kingsrook.qbits.dbperformance.normalize;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HexFormat;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -200,12 +203,86 @@ public final class SqlNormalizer
*******************************************************************************/
private static String collapseLists(String normalized)
{
normalized = normalized.replaceAll("\\bin \\( \\?(?: , \\?)* \\)", "in ( ? )");
normalized = normalized.replaceAll("\\bvalues (\\( \\?(?: , \\?)* \\))(?: , \\( \\?(?: , \\?)* \\))+", "values $1");
return (normalized);
///////////////////////////////////////////////////////////////////////////
// a linear scan over the space-separated tokens: regular expressions //
// with nested repetition overflow the stack on very long IN and VALUES //
// lists (seen with 60 KB statements in production) //
///////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
// segments between quoted identifiers keep their edge spaces //
////////////////////////////////////////////////////////////////////////
String trimmed = normalized.strip();
if(trimmed.isEmpty())
{
return (normalized);
}
String leading = normalized.substring(0, normalized.indexOf(trimmed.charAt(0)));
String trailing = normalized.substring(leading.length() + trimmed.length());
String[] tokens = trimmed.split(" ");
List<String> out = new ArrayList<>(tokens.length);
int i = 0;
while(i < tokens.length)
{
String token = tokens[i];
if("in".equals(token))
{
int end = placeholderGroupEnd(tokens, i + 1);
if(end > 0)
{
out.addAll(List.of("in", "(", "?", ")"));
i = end;
continue;
}
}
if("values".equals(token))
{
int first = placeholderGroupEnd(tokens, i + 1);
if(first > 0)
{
int end = first;
while(end + 1 < tokens.length && ",".equals(tokens[end]))
{
int next = placeholderGroupEnd(tokens, end + 1);
if(next < 0)
{
break;
}
end = next;
}
out.add("values");
out.addAll(Arrays.asList(tokens).subList(i + 1, first));
i = end;
continue;
}
}
out.add(token);
i++;
}
return (leading + String.join(" ", out) + trailing);
}
/*******************************************************************************
** If tokens[start] opens "( ? , ? ... )" made only of placeholders, the index
** just after its ")"; otherwise -1.
*******************************************************************************/
private static int placeholderGroupEnd(String[] tokens, int start)
{
if(start + 2 >= tokens.length || !"(".equals(tokens[start]) || !"?".equals(tokens[start + 1]))
{
return (-1);
}
int i = start + 2;
while(i + 1 < tokens.length && ",".equals(tokens[i]) && "?".equals(tokens[i + 1]))
{
i += 2;
}
return (i < tokens.length && ")".equals(tokens[i]) ? i + 1 : -1);
}
/*******************************************************************************
** Consume a quote with SQL doubled-quote and optional escape handling.
*******************************************************************************/
@@ -0,0 +1,99 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.processes;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
/*******************************************************************************
** Settings action: ask the backend's provider to turn on statement statistics
** (H2: SET QUERY_STATISTICS TRUE as an admin), then record the new access.
** This is the qbit's only change to the monitored database and it is explicit.
*******************************************************************************/
public class DbPerfEnableStatisticsStep implements BackendStep
{
/*******************************************************************************
** Requires manage_settings; vendor errors become result lines.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance());
if(QContext.getQSession() == null || !QContext.getQSession().hasPermission(runtime.getConfig().getPermissionBaseName() + ".manage_settings"))
{
throw (new QException("Managing settings permission is required"));
}
List<ProcessSummaryLineInterface> lines = new ArrayList<>();
String backend = runtime.getConfig().getBackendName();
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(backend)))
{
DbStatsProvider provider = DbStatsProviders.forConnection(connection);
try
{
lines.add(new ProcessSummaryLine(Status.OK, 1, provider.enableStatementStatistics(connection)));
}
catch(SQLFeatureNotSupportedException e)
{
lines.add(new ProcessSummaryLine(Status.ERROR, 0, e.getMessage()));
}
catch(SQLException e)
{
lines.add(new ProcessSummaryLine(Status.ERROR, 0, "The database refused the change (SQLSTATE " + e.getSQLState() + "); an admin database user is required."));
}
ProviderCapabilities capability = provider.capabilities(connection);
QRecord settings = new DbPerfSettingsService(runtime).ensure(backend);
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", settings.getValueInteger("id")).withValue("provider", provider.getName())
.withValue("statsAccess", capability.statsAccess()).withValue("statsAccessMessage", capability.message())));
lines.add(new ProcessSummaryLine(Status.INFO, 0, "statistics access: " + capability.statsAccess()));
}
catch(QException e)
{
throw (e);
}
catch(Exception e)
{
throw (new QException("Unable to enable statement statistics", e));
}
output.addValue("processResults", new ArrayList<>(lines));
}
}
@@ -54,6 +54,7 @@ public final class DbPerfProcesses
public static final String COLLECT = "dbPerfCollectSnapshot";
public static final String ROLLUP = "dbPerfRollupAndPurge";
public static final String COLLECT_NOW = "dbPerfCollectNow";
public static final String ENABLE_STATISTICS = "dbPerfEnableStatementStatistics";
/*******************************************************************************
@@ -100,6 +101,10 @@ public final class DbPerfProcesses
.withTableName("db_perf_snapshot")
.withStep(new QFrontendStepMetaData().withName("result").withLabel("Result")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))));
instance.addProcess(base(config, ENABLE_STATISTICS, "Enable statement statistics", DbPerfEnableStatisticsStep.class, false)
.withTableName("db_perf_settings")
.withStep(new QFrontendStepMetaData().withName("result").withLabel("Result")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))));
}
@@ -23,12 +23,17 @@ package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.List;
import java.util.Map;
import java.util.Set;
/*******************************************************************************
** Read-only database statistics SPI. Implementations never change the database.
** Database statistics SPI. Every read is read-only and never changes the
** database; {@link #enableStatementStatistics} is the one explicit, opt-in
** change, run only from the settings action by a user who manages settings.
** A provider is chosen per JDBC product name by {@link DbStatsProviders}.
*******************************************************************************/
public interface DbStatsProvider
{
@@ -90,4 +95,57 @@ public interface DbStatsProvider
** readSettings using the supplied backend connection.
*******************************************************************************/
Map<String, String> readSettings(Connection connection) throws SQLException;
/*******************************************************************************
** Statistics areas this vendor can supply at best, independent of a
** connection. Analysis rules that need an absent area are skipped.
*******************************************************************************/
default Set<StatsFeature> features()
{
return (Set.of());
}
/*******************************************************************************
** Operator next step for a persisted statsAccess code (health and setup).
*******************************************************************************/
default String remedy(String statsAccess, String databaseUser)
{
return ("FULL".equals(statsAccess) ? "Full statistics access" : "Use a supported database vendor and statistics provider");
}
/*******************************************************************************
** Vendor prerequisite rows for the guided setup panel, from saved settings.
*******************************************************************************/
default List<SetupCheck> setupChecks(String statsAccess, String message)
{
return (List.of(new SetupCheck("Statistics access", statsAccess, message == null ? remedy(statsAccess, null) : message)));
}
/*******************************************************************************
** Turn on statement statistics where the vendor allows it at runtime, and
** return what was done. The default explains why it is not possible.
*******************************************************************************/
default String enableStatementStatistics(Connection connection) throws SQLException
{
throw (new SQLFeatureNotSupportedException("Statement statistics cannot be enabled at runtime for this database; see the setup panel."));
}
/*******************************************************************************
** Vendor text plan for providers without structured JSON plans. The SQL uses
** JDBC ?n slots; parameters are bound only for ANALYZE.
*******************************************************************************/
default String explainText(Connection connection, String sql, List<String> parameters, boolean analyze, int timeoutMs) throws SQLException
{
throw (new SQLFeatureNotSupportedException("Query plans are not available for this database."));
}
}
@@ -23,13 +23,31 @@ package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
/*******************************************************************************
** Select a provider from JDBC product identity.
** Provider registry keyed by JDBC {@code DatabaseMetaData.getDatabaseProductName()}.
** Adding a vendor is one implementation of {@link DbStatsProvider} plus one
** {@link #register} call (see docs/PROVIDERS.md); anything unregistered gets
** {@link UnsupportedStatsProvider}.
*******************************************************************************/
public final class DbStatsProviders
{
//////////////////////////////////////////////////////////////////////////
// JDBC product name (lower case) -> provider. MySQL reports "MySQL", //
// MariaDB "MariaDB" (Connector/J 3) and SQLite "SQLite"; none are //
// registered yet: docs/PROVIDERS.md says what each could supply. //
//////////////////////////////////////////////////////////////////////////
private static final Map<String, Supplier<DbStatsProvider>> BY_PRODUCT = new ConcurrentHashMap<>(Map.of(
"postgresql", PostgresStatsProvider::new,
"h2", H2StatsProvider::new));
/*******************************************************************************
** Utility class.
*******************************************************************************/
@@ -38,11 +56,53 @@ public final class DbStatsProviders
}
/*******************************************************************************
** PostgreSQL first; other vendors are explicitly unsupported.
** Provider for the connection's JDBC product.
*******************************************************************************/
public static DbStatsProvider forConnection(Connection connection) throws SQLException
{
return ("PostgreSQL".equalsIgnoreCase(connection.getMetaData().getDatabaseProductName()) ? new PostgresStatsProvider() : new UnsupportedStatsProvider());
return (forProductName(connection.getMetaData().getDatabaseProductName()));
}
/*******************************************************************************
** Provider for a JDBC product name; unsupported when not registered.
*******************************************************************************/
public static DbStatsProvider forProductName(String productName)
{
Supplier<DbStatsProvider> supplier = productName == null ? null : BY_PRODUCT.get(productName.toLowerCase(Locale.ROOT));
return (supplier == null ? new UnsupportedStatsProvider() : supplier.get());
}
/*******************************************************************************
** Provider by its saved name (db_perf_settings.provider), for code that has
** settings but no connection; unsupported for null or unknown names.
*******************************************************************************/
public static DbStatsProvider forProviderName(String providerName)
{
for(Supplier<DbStatsProvider> supplier : BY_PRODUCT.values())
{
DbStatsProvider provider = supplier.get();
if(provider.getName().equalsIgnoreCase(providerName))
{
return (provider);
}
}
return (new UnsupportedStatsProvider());
}
/*******************************************************************************
** Register (or replace) the provider for a JDBC product name. Hosts may add
** vendors before the qbit collects; the name must be a Provider possible value.
*******************************************************************************/
public static void register(String productName, Supplier<DbStatsProvider> provider)
{
BY_PRODUCT.put(productName.toLowerCase(Locale.ROOT), provider);
}
}
@@ -0,0 +1,691 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.instant;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.read;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.scalar;
/*******************************************************************************
** H2 2.x statistics from INFORMATION_SCHEMA. Statement statistics need the
** database setting QUERY_STATISTICS (URL {@code ;QUERY_STATISTICS=TRUE} or
** {@code SET QUERY_STATISTICS TRUE} as an admin); every other area works
** without it. H2 keeps no transaction, cache, scan or vacuum counters, so
** those areas are reported as unavailable rather than as zeros. See
** docs/PROVIDERS.md for what is exact, approximated and unsupported.
*******************************************************************************/
public class H2StatsProvider implements DbStatsProvider
{
public static final String NAME = "H2";
private static final String NO_QUERY_STATISTICS = "NO_QUERY_STATISTICS";
private static final String NOT_ADMIN = "NOT_ADMIN";
private static final int MAX_ENTRIES = 1000;
private static final Pattern WORD = Pattern.compile("[A-Za-z_][A-Za-z0-9_$]*");
//////////////////////////////////////////////////////////////////////////
// H2 has no READ ONLY transaction and a rollback does not undo file, //
// session, sequence or DDL side effects, so plans refuse SQL naming //
// these (and any user-defined alias) outright //
//////////////////////////////////////////////////////////////////////////
private static final Set<String> DENIED = Set.of("FILE_WRITE", "FILE_READ", "CSVWRITE", "CSVREAD", "LINK_SCHEMA", "SCRIPT", "RUNSCRIPT",
"DISK_SPACE_USED", "CANCEL_SESSION", "ABORT_SESSION", "BACKUP", "SHUTDOWN", "CALL", "SET", "DROP", "CREATE", "ALTER", "TRUNCATE", "GRANT",
"REVOKE", "COMMIT", "ROLLBACK", "SAVEPOINT", "CHECKPOINT", "NEXTVAL", "SETVAL", "CURRVAL", "LOCK", "EXECUTE", "INSERT", "UPDATE",
"DELETE", "MERGE", "UPSERT", "REPLACE", "INTO", "FOR", "JAVA_OBJECT", "SLEEP");
///////////////////////////////////////////////////////////////////////////
// statement that guarantees QUERY_STATISTICS has a row when it is on; //
// it and INFORMATION_SCHEMA reads (the collector's own) are not stored //
///////////////////////////////////////////////////////////////////////////
private static final String PROBE = "SELECT 1 AS DB_PERF_PROBE";
private static final String DISK_SPACE = "SELECT DISK_SPACE_USED(?)";
private static final Set<StatsFeature> FEATURES = StatsFeature.of(StatsFeature.DATABASE_SIZE, StatsFeature.STATEMENTS,
StatsFeature.TABLE_ROWS, StatsFeature.TABLE_SIZES, StatsFeature.INDEX_DEFINITIONS, StatsFeature.SESSIONS, StatsFeature.BLOCKING,
StatsFeature.CONNECTIONS, StatsFeature.EXPLAIN_TEXT, StatsFeature.EXPLAIN_GENERIC, StatsFeature.EXPLAIN_ANALYZE);
private static final List<String> SETTINGS = List.of("info.VERSION", "MODE", "MV_STORE", "info.FILE_SIZE", "info.PAGE_SIZE",
"info.CACHE_MAX_SIZE", "info.CACHE_SIZE", "info.CACHE_HIT_RATIO", "info.FILE_READ", "info.FILE_WRITE", "MAX_QUERY_TIMEOUT", "QUERY_TIMEOUT");
/*******************************************************************************
** Provider name.
*******************************************************************************/
@Override
public String getName()
{
return (NAME);
}
/*******************************************************************************
** Version, admin rights and whether QUERY_STATISTICS is collecting.
*******************************************************************************/
@Override
public ProviderCapabilities capabilities(Connection connection) throws SQLException
{
int major = connection.getMetaData().getDatabaseMajorVersion();
int version = version(connection);
boolean supported = major >= 2;
if(!supported)
{
return (new ProviderCapabilities(false, version, false, false, "UNSUPPORTED", "H2 2.0 or newer is required.", Set.of()));
}
boolean statistics = queryStatistics(connection);
boolean admin = admin(connection);
String access = !statistics ? NO_QUERY_STATISTICS : !admin ? NOT_ADMIN : "FULL";
String message = switch(access)
{
case NO_QUERY_STATISTICS -> "H2 query statistics are off. Run Enable statement statistics on Settings as an H2 admin user, or add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 to the JDBC URL.";
case NOT_ADMIN -> "The H2 user is not an admin, so only its own session is visible. Use an admin user (ALTER USER ... ADMIN TRUE) to see every session and blocking.";
default -> "Full statistics access (H2 does not keep transaction, cache, scan or vacuum counters).";
};
EnumSet<StatsFeature> available = EnumSet.copyOf(FEATURES);
if(!statistics)
{
available.remove(StatsFeature.STATEMENTS);
}
if(!admin)
{
available.remove(StatsFeature.BLOCKING);
}
else
{
////////////////////////////////////////////////////////////////////
// an admin may call file and session functions; ANALYZE executes //
// the query, so it is offered only to non-admin users //
////////////////////////////////////////////////////////////////////
available.remove(StatsFeature.EXPLAIN_ANALYZE);
}
return (new ProviderCapabilities(true, version, statistics, admin, access, message, available));
}
/*******************************************************************************
** Areas H2 can supply at best.
*******************************************************************************/
@Override
public Set<StatsFeature> features()
{
return (FEATURES);
}
/*******************************************************************************
** Size from the store; H2 has no cumulative counters, reset time or limit.
*******************************************************************************/
@Override
public DatabaseStats readDatabase(Connection connection) throws SQLException
{
List<Long> size = read(connection, "SELECT SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS WHERE SETTING_NAME = 'info.FILE_SIZE'", row -> parseLong(row.getString(1)));
return (new DatabaseStats(Map.of(), null, 0, size.isEmpty() || size.getFirst() == null ? 0 : size.getFirst()));
}
/*******************************************************************************
** QUERY_STATISTICS rows (times in milliseconds), without connection setup
** (see {@link #excluded}). H2 has no query id or user, so the id is a stable
** hash of the statement text and the user is null.
*******************************************************************************/
@Override
public List<StatementStats> readStatements(Connection connection) throws SQLException
{
List<StatementStats> result = new ArrayList<>();
read(connection, "SELECT SQL_STATEMENT, EXECUTION_COUNT, CUMULATIVE_EXECUTION_TIME, AVERAGE_EXECUTION_TIME, MAX_EXECUTION_TIME, CUMULATIVE_ROW_COUNT FROM INFORMATION_SCHEMA.QUERY_STATISTICS", row ->
{
String sql = row.getString("SQL_STATEMENT");
if(sql != null && !excluded(sql))
{
result.add(new StatementStats(hash(sql), null, true, sql, row.getLong("EXECUTION_COUNT"), millis(row.getDouble("CUMULATIVE_EXECUTION_TIME")),
millis(row.getDouble("AVERAGE_EXECUTION_TIME")), millis(row.getDouble("MAX_EXECUTION_TIME")), row.getLong("CUMULATIVE_ROW_COUNT"), 0, 0, 0, null, null));
}
return (null);
});
return (result);
}
/*******************************************************************************
** Not application statements: the collector's own probe, table sizing and
** INFORMATION_SCHEMA reads, and connection setup. H2 replays every JDBC URL setting as a SET on
** each new connection and drivers call DATABASE() for the catalog, so with
** unpooled connections these dominate the call counts without being work.
*******************************************************************************/
static boolean excluded(String sql)
{
String upper = sql.strip().toUpperCase(Locale.ROOT);
return (PROBE.equals(sql) || DISK_SPACE.equals(sql) || upper.contains("INFORMATION_SCHEMA.") || upper.startsWith("SET ") || upper.equals("CALL DATABASE()"));
}
/*******************************************************************************
** H2 keeps no reset time; a reset shows as falling call counts instead.
*******************************************************************************/
@Override
public StatementResetInfo readStatementResetInfo(Connection connection)
{
return (new StatementResetInfo(null, 0));
}
/*******************************************************************************
** Base tables with H2's row-count estimate and DISK_SPACE_USED (table data
** only: H2 cannot size indexes, so totalBytes equals tableBytes).
*******************************************************************************/
@Override
public List<TableStats> readTables(Connection connection) throws SQLException
{
List<TableStats> result = new ArrayList<>();
List<String[]> tables = read(connection, "SELECT TABLE_SCHEMA, TABLE_NAME, ROW_COUNT_ESTIMATE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA <> 'INFORMATION_SCHEMA' ORDER BY TABLE_SCHEMA, TABLE_NAME",
row -> new String[] {row.getString(1), row.getString(2), row.getString(3)});
try(PreparedStatement size = connection.prepareStatement(DISK_SPACE))
{
size.setQueryTimeout(30);
for(String[] table : tables)
{
Map<String, Long> counters = new LinkedHashMap<>();
counters.put("nLiveTup", table[2] == null ? 0L : Long.parseLong(table[2]));
Long bytes = diskSpace(size, quote(table[0]) + "." + quote(table[1]));
if(bytes != null)
{
counters.put("tableBytes", bytes);
counters.put("totalBytes", bytes);
}
result.add(new TableStats(table[0], table[1], hash("TABLE:" + table[0] + "." + table[1]) & Long.MAX_VALUE, counters, Map.of()));
}
}
return (result);
}
/*******************************************************************************
** Index definitions and ordered key columns. H2 has no index usage or size
** statistics; constraint-generated indexes are marked USING CONSTRAINT so
** they are never proposed for removal.
*******************************************************************************/
@Override
public List<IndexStats> readIndexes(Connection connection) throws SQLException
{
Map<String, IndexBuilder> indexes = new LinkedHashMap<>();
String sql = "SELECT i.INDEX_SCHEMA, i.TABLE_NAME, i.INDEX_NAME, i.INDEX_TYPE_NAME, i.IS_GENERATED, c.COLUMN_NAME, c.ORDERING_SPECIFICATION FROM INFORMATION_SCHEMA.INDEXES i "
+ "JOIN INFORMATION_SCHEMA.INDEX_COLUMNS c ON c.INDEX_SCHEMA = i.INDEX_SCHEMA AND c.TABLE_NAME = i.TABLE_NAME AND c.INDEX_NAME = i.INDEX_NAME "
+ "WHERE i.INDEX_SCHEMA <> 'INFORMATION_SCHEMA' ORDER BY i.INDEX_SCHEMA, i.TABLE_NAME, i.INDEX_NAME, c.ORDINAL_POSITION";
read(connection, sql, row ->
{
IndexBuilder index = indexes.computeIfAbsent(row.getString(1) + "." + row.getString(3), key -> new IndexBuilder());
index.schema = row.getString(1);
index.table = row.getString(2);
index.name = row.getString(3);
index.type = row.getString(4) == null ? "" : row.getString(4).toUpperCase(Locale.ROOT);
index.generated = row.getBoolean(5);
String order = "DESC".equalsIgnoreCase(row.getString(7)) ? " DESC" : "";
index.columns.add(row.getString(6) + order);
index.quoted.add(quote(row.getString(6)) + order);
return (null);
});
List<IndexStats> result = new ArrayList<>();
for(IndexBuilder index : indexes.values())
{
boolean primary = index.type.contains("PRIMARY");
boolean unique = primary || index.type.contains("UNIQUE");
String columns = String.join(", ", index.columns);
String keys = String.join(", ", index.quoted);
String definition = primary ? "ALTER TABLE " + quote(index.schema) + "." + quote(index.table) + " ADD PRIMARY KEY (" + keys + ")"
: "CREATE " + (unique ? "UNIQUE " : "") + "INDEX " + quote(index.name) + " ON " + quote(index.schema) + "." + quote(index.table) + (index.generated ? " USING CONSTRAINT" : "") + " (" + keys + ")";
result.add(new IndexStats(index.schema, index.table, index.name, hash("INDEX:" + index.schema + "." + index.name) & Long.MAX_VALUE, definition, columns, unique, primary, true, 0, 0, 0, 0, 0));
}
return (result);
}
/*******************************************************************************
** Other sessions in PostgreSQL's state vocabulary: RUNNING is active, BLOCKED
** is active waiting on a Lock, SLEEP with uncommitted changes is idle in
** transaction. H2 has no transaction start, so an idle transaction's age is
** the time since it went idle (a lower bound).
*******************************************************************************/
@Override
public List<SessionInfo> readSessions(Connection connection) throws SQLException
{
Instant now = Instant.now();
String sql = "SELECT SESSION_ID, USER_NAME, CLIENT_INFO, CLIENT_ADDR, SERVER, SESSION_STATE, CONTAINS_UNCOMMITTED, BLOCKER_ID, EXECUTING_STATEMENT, EXECUTING_STATEMENT_START, SLEEP_SINCE "
+ "FROM INFORMATION_SCHEMA.SESSIONS WHERE SESSION_ID <> SESSION_ID() ORDER BY SESSION_ID";
return (read(connection, sql, row ->
{
String state = state(row.getString("SESSION_STATE"), row.getBoolean("CONTAINS_UNCOMMITTED"));
boolean blocked = "BLOCKED".equalsIgnoreCase(row.getString("SESSION_STATE"));
Instant queryStart = instant(row, "EXECUTING_STATEMENT_START");
Instant sleepSince = instant(row, "SLEEP_SINCE");
boolean active = "active".equals(state);
String query = row.getString("EXECUTING_STATEMENT");
Integer blocker = row.getObject("BLOCKER_ID") == null ? null : row.getInt("BLOCKER_ID");
return (new SessionInfo(row.getInt("SESSION_ID"), row.getString("USER_NAME"), row.getString("CLIENT_INFO"), row.getString("CLIENT_ADDR"), state,
blocked ? "Lock" : null, blocked ? "row lock" : null, row.getString("SERVER") == null ? "embedded" : row.getString("SERVER"),
null, queryStart, active ? queryStart : sleepSince, active ? seconds(queryStart, now) : null,
state.startsWith("idle in transaction") ? seconds(sleepSince, now) : null, blocker == null ? List.of() : List.of(blocker), query == null ? null : hash(query), query));
}));
}
/*******************************************************************************
** Every visible session, including this one, by user, client and state.
*******************************************************************************/
@Override
public List<ConnectionCount> readConnections(Connection connection) throws SQLException
{
Map<List<String>, Integer> counts = new LinkedHashMap<>();
read(connection, "SELECT USER_NAME, CLIENT_INFO, SESSION_STATE, CONTAINS_UNCOMMITTED FROM INFORMATION_SCHEMA.SESSIONS ORDER BY SESSION_ID", row ->
counts.merge(Arrays.asList(row.getString(1), row.getString(2), state(row.getString(3), row.getBoolean(4))), 1, Integer::sum));
List<ConnectionCount> result = new ArrayList<>();
counts.forEach((key, count) -> result.add(new ConnectionCount(key.get(0), key.get(1), key.get(2), count)));
return (result);
}
/*******************************************************************************
** Operational, non-secret settings plus the QUERY_STATISTICS state.
*******************************************************************************/
@Override
public Map<String, String> readSettings(Connection connection) throws SQLException
{
Map<String, String> result = new LinkedHashMap<>();
read(connection, "SELECT SETTING_NAME, SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS", row -> SETTINGS.contains(row.getString(1)) ? result.put(row.getString(1), row.getString(2)) : null);
result.put("QUERY_STATISTICS", queryStatistics(connection) ? "TRUE" : "FALSE");
return (result);
}
/*******************************************************************************
** Health and setup next steps for H2's access codes.
*******************************************************************************/
@Override
public String remedy(String statsAccess, String databaseUser)
{
return (switch(statsAccess == null ? "" : statsAccess)
{
case "FULL" -> "Full statistics access";
case NO_QUERY_STATISTICS -> "Turn on H2 query statistics: run Enable statement statistics on Settings, or add ;QUERY_STATISTICS=TRUE to the JDBC URL";
case NOT_ADMIN -> "Make " + databaseUser + " an H2 admin (ALTER USER " + databaseUser + " ADMIN TRUE) to see every session";
default -> "Use H2 2.0 or newer";
});
}
/*******************************************************************************
** Guided setup rows: query statistics and the admin role.
*******************************************************************************/
@Override
public List<SetupCheck> setupChecks(String statsAccess, String message)
{
return (List.of(new SetupCheck("H2 QUERY_STATISTICS", statsAccess, "Run Enable statement statistics on Settings (admin user), or add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 to the JDBC URL; then run Collect now."),
new SetupCheck("H2 admin user", statsAccess, message == null ? "Use an H2 admin user to see every session and blocking." : message)));
}
/*******************************************************************************
** SET QUERY_STATISTICS TRUE (admin only), raising H2's default of 100 kept
** statements: H2 evicts a third of them, oldest first, whenever the cap is
** passed, and an application easily runs more distinct statements. H2 does
** not persist either setting: they last until the database closes, so also
** put them in the JDBC URL. An existing setup is left untouched.
*******************************************************************************/
@Override
public String enableStatementStatistics(Connection connection) throws SQLException
{
if(queryStatistics(connection))
{
return ("H2 query statistics are already on.");
}
try(Statement statement = connection.createStatement())
{
statement.execute("SET QUERY_STATISTICS_MAX_ENTRIES " + MAX_ENTRIES);
statement.execute("SET QUERY_STATISTICS TRUE");
}
return ("H2 query statistics are on (up to " + MAX_ENTRIES + " statements) until the database closes. Add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=" + MAX_ENTRIES + " to the JDBC URL to keep them on after a restart.");
}
/*******************************************************************************
** EXPLAIN (not executed) or EXPLAIN ANALYZE (executed with bound values).
** H2 has no READ ONLY transaction, so both refuse more than one statement
** and any SQL naming a side-effecting function, DDL/DML keyword or
** user-defined alias; ANALYZE additionally requires a single SELECT/WITH
** and a non-admin user. The connection is marked read-only (a hint in H2)
** and the transaction always rolls back.
*******************************************************************************/
@Override
public String explainText(Connection connection, String sql, List<String> parameters, boolean analyze, int timeoutMs) throws SQLException
{
List<String> words = words(sql);
if(statements(sql) > 1)
{
throw (new SQLException("Query plans on H2 accept exactly one statement.", "42000"));
}
List<String> aliases = read(connection, "SELECT UPPER(ROUTINE_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA <> 'INFORMATION_SCHEMA'", row -> row.getString(1));
for(String word : words)
{
if(DENIED.contains(word) || aliases.contains(word))
{
throw (new SQLException("Query plans on H2 refuse " + word + ": H2 cannot run it read-only.", "42000"));
}
}
if(analyze)
{
if(words.isEmpty() || !Set.of("SELECT", "WITH").contains(words.getFirst()))
{
throw (new SQLException("EXPLAIN ANALYZE on H2 accepts a single SELECT only.", "42000"));
}
if(admin(connection))
{
throw (new SQLException("EXPLAIN ANALYZE on H2 is disabled for admin users (it executes the query and H2 has no read-only transactions). Use PLAN, or connect as a non-admin user.", "42000"));
}
}
boolean autoCommit = connection.getAutoCommit();
boolean readOnly = connection.isReadOnly();
try
{
connection.setReadOnly(true);
connection.setAutoCommit(false);
if(!analyze)
{
////////////////////////////////////////////////////////////////
// a plain statement plans ?n slots without values (generic) //
////////////////////////////////////////////////////////////////
try(Statement statement = connection.createStatement())
{
statement.setQueryTimeout(Math.max(1, (timeoutMs + 999) / 1000));
return (plan(statement.executeQuery("EXPLAIN " + sql)));
}
}
try(PreparedStatement statement = connection.prepareStatement("EXPLAIN ANALYZE " + sql))
{
statement.setQueryTimeout(Math.max(1, (timeoutMs + 999) / 1000));
for(int i = 0; i < parameters.size(); i++)
{
statement.setString(i + 1, parameters.get(i));
}
return (plan(statement.executeQuery()));
}
}
finally
{
try
{
connection.rollback();
}
finally
{
connection.setAutoCommit(autoCommit);
connection.setReadOnly(readOnly);
}
}
}
/*******************************************************************************
** Upper-case words outside string literals and comments; double-quoted
** identifiers count as words, since H2 resolves quoted function names.
*******************************************************************************/
static List<String> words(String sql)
{
String code = code(sql);
List<String> words = new ArrayList<>();
Matcher matcher = WORD.matcher(code);
while(matcher.find())
{
words.add(matcher.group().toUpperCase(Locale.ROOT));
}
return (words);
}
/*******************************************************************************
** Statements separated by semicolons outside literals and comments.
*******************************************************************************/
static int statements(String sql)
{
int count = 0;
for(String part : code(sql).split(";"))
{
count += part.isBlank() ? 0 : 1;
}
return (count);
}
/*******************************************************************************
** SQL with string literals and comments blanked, quoted identifiers unquoted.
*******************************************************************************/
private static String code(String sql)
{
return (sql.replaceAll("(?s)/\\*.*?\\*/", " ").replaceAll("--[^\n]*", " ").replaceAll("'(?:[^']|'')*'", " ").replace('"', ' '));
}
/*******************************************************************************
** Plan rows joined as text; closes the result.
*******************************************************************************/
private static String plan(ResultSet rows) throws SQLException
{
try(rows)
{
StringBuilder plan = new StringBuilder();
while(rows.next())
{
plan.append(plan.isEmpty() ? "" : "\n").append(rows.getString(1));
}
return (plan.toString());
}
}
/*******************************************************************************
** QUERY_STATISTICS is not exposed in SETTINGS; when it is on, the probe just
** run is always recorded, and when it is off the view is empty.
*******************************************************************************/
boolean queryStatistics(Connection connection) throws SQLException
{
read(connection, PROBE, row -> null);
return (scalar(connection, "SELECT COUNT(*) FROM INFORMATION_SCHEMA.QUERY_STATISTICS") > 0);
}
/*******************************************************************************
** Non-admin users see only their own USERS row and session.
*******************************************************************************/
private boolean admin(Connection connection) throws SQLException
{
return (!read(connection, "SELECT IS_ADMIN FROM INFORMATION_SCHEMA.USERS WHERE USER_NAME = CURRENT_USER", row -> row.getBoolean(1)).stream().filter(Boolean::booleanValue).toList().isEmpty());
}
/*******************************************************************************
** major*1000000 + minor*1000 + build, e.g. 2004240 for 2.4.240.
*******************************************************************************/
private int version(Connection connection) throws SQLException
{
String[] parts = read(connection, "SELECT H2VERSION()", row -> row.getString(1)).getFirst().split("[.\\s]");
int version = 0;
for(int i = 0; i < 3; i++)
{
version = version * 1000 + (i < parts.length && parts[i].matches("\\d+") ? Integer.parseInt(parts[i]) : 0);
}
return (version);
}
/*******************************************************************************
** PostgreSQL-compatible state names used by collection and presentation.
*******************************************************************************/
static String state(String h2State, boolean uncommitted)
{
String value = h2State == null ? "" : h2State.toUpperCase(Locale.ROOT);
return (switch(value)
{
case "RUNNING", "BLOCKED" -> "active";
case "SLEEP" -> uncommitted ? "idle in transaction" : "idle";
default -> value.toLowerCase(Locale.ROOT);
});
}
/*******************************************************************************
** Stable 64-bit identity for text (statement, table or index key).
*******************************************************************************/
static long hash(String text)
{
try
{
return (ByteBuffer.wrap(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))).getLong());
}
catch(NoSuchAlgorithmException e)
{
throw (new IllegalStateException("SHA-256 is required", e));
}
}
/*******************************************************************************
** DISK_SPACE_USED for one quoted table; null when H2 cannot size it.
*******************************************************************************/
private static Long diskSpace(PreparedStatement size, String table)
{
try
{
size.setString(1, table);
try(ResultSet rows = size.executeQuery())
{
return (rows.next() ? rows.getLong(1) : null);
}
}
catch(SQLException e)
{
return (null);
}
}
/*******************************************************************************
** Quoted identifier.
*******************************************************************************/
private static String quote(String identifier)
{
return ("\"" + identifier.replace("\"", "\"\"") + "\"");
}
/*******************************************************************************
** Milliseconds as a 3-decimal BigDecimal.
*******************************************************************************/
private static BigDecimal millis(double value)
{
return (BigDecimal.valueOf(value).setScale(3, RoundingMode.HALF_UP));
}
/*******************************************************************************
** Elapsed seconds, or null without a start.
*******************************************************************************/
private static BigDecimal seconds(Instant start, Instant now)
{
return (start == null ? null : BigDecimal.valueOf(Math.max(0, Duration.between(start, now).toMillis())).divide(BigDecimal.valueOf(1000), 3, RoundingMode.HALF_UP));
}
/*******************************************************************************
** Numeric setting value, or null.
*******************************************************************************/
private static Long parseLong(String value)
{
return (value != null && value.matches("\\d+") ? Long.parseLong(value) : null);
}
/*******************************************************************************
** Index rows folded per index.
*******************************************************************************/
private static class IndexBuilder
{
private String schema;
private String table;
private String name;
private String type;
private boolean generated;
private final List<String> columns = new ArrayList<>();
private final List<String> quoted = new ArrayList<>();
}
}
@@ -0,0 +1,101 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** Bounded-time JDBC reads shared by statistics providers.
*******************************************************************************/
final class JdbcReads
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private JdbcReads()
{
}
/*******************************************************************************
** Execute one bounded-time read and close JDBC resources.
*******************************************************************************/
static <T> List<T> read(Connection connection, String sql, RowReader<T> mapper) throws SQLException
{
List<T> result = new ArrayList<>();
try(Statement statement = connection.createStatement())
{
statement.setQueryTimeout(30);
try(ResultSet rows = statement.executeQuery(sql))
{
while(rows.next())
{
result.add(mapper.read(rows));
}
}
}
return (result);
}
/*******************************************************************************
** A single numeric read.
*******************************************************************************/
static long scalar(Connection connection, String sql) throws SQLException
{
return (read(connection, sql, row -> row.getLong(1)).getFirst());
}
/*******************************************************************************
** Timestamp conversion preserves null values.
*******************************************************************************/
static Instant instant(ResultSet row, String column) throws SQLException
{
Timestamp timestamp = row.getTimestamp(column);
return (timestamp == null ? null : timestamp.toInstant());
}
/*******************************************************************************
** Checked JDBC row mapping.
*******************************************************************************/
interface RowReader<T>
{
/*******************************************************************************
** Map one row.
*******************************************************************************/
T read(ResultSet row) throws SQLException;
}
}
@@ -24,13 +24,16 @@ package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.instant;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.read;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.scalar;
/*******************************************************************************
@@ -85,7 +88,60 @@ public class PostgresStatsProvider implements DbStatsProvider
}
}
}
return (new ProviderCapabilities(version >= 130000, version, installed, privileged, access, message));
EnumSet<StatsFeature> available = EnumSet.copyOf(features());
if(!installed)
{
available.removeAll(EnumSet.of(StatsFeature.STATEMENTS, StatsFeature.STATEMENT_IO, StatsFeature.STATEMENT_RESET));
}
if(version < 140000)
{
available.remove(StatsFeature.STATEMENT_RESET);
}
if(version < 160000)
{
available.remove(StatsFeature.EXPLAIN_GENERIC);
}
return (new ProviderCapabilities(version >= 130000, version, installed, privileged, access, message, version >= 130000 ? available : EnumSet.noneOf(StatsFeature.class)));
}
/*******************************************************************************
** Every area: the cumulative statistics views cover them all.
*******************************************************************************/
@Override
public Set<StatsFeature> features()
{
return (StatsFeature.of(StatsFeature.values()));
}
/*******************************************************************************
** Role and extension prerequisites for each access code.
*******************************************************************************/
@Override
public String remedy(String statsAccess, String databaseUser)
{
return (switch(statsAccess == null ? "" : statsAccess)
{
case "FULL" -> "Full statistics access";
case "NO_READ_ALL_STATS" -> "grant pg_read_all_stats to " + databaseUser;
case "NO_PG_STAT_STATEMENTS" -> "Enable pg_stat_statements in shared_preload_libraries and create the extension";
default -> "Use a supported PostgreSQL version (13 or newer) and statistics provider";
});
}
/*******************************************************************************
** Guided setup rows: the extension and the statistics role.
*******************************************************************************/
@Override
public List<SetupCheck> setupChecks(String statsAccess, String message)
{
return (List.of(new SetupCheck("pg_stat_statements", statsAccess, "Preload and create pg_stat_statements; then run Collect now."),
new SetupCheck("pg_read_all_stats", statsAccess, message == null ? "Grant pg_read_all_stats to the application's database role." : message)));
}
@@ -237,25 +293,6 @@ public class PostgresStatsProvider implements DbStatsProvider
}
/*******************************************************************************
** A single numeric read.
*******************************************************************************/
private long scalar(Connection connection, String sql) throws SQLException
{
return (read(connection, sql, row -> row.getLong(1)).getFirst());
}
/*******************************************************************************
** Timestamp conversion preserves null reset/maintenance values.
*******************************************************************************/
private static Instant instant(ResultSet row, String column) throws SQLException
{
Timestamp timestamp = row.getTimestamp(column);
return (timestamp == null ? null : timestamp.toInstant());
}
/*******************************************************************************
** Copy cumulative counters to the entity's camelCase names.
*******************************************************************************/
@@ -282,37 +319,4 @@ public class PostgresStatsProvider implements DbStatsProvider
}
return (value.toString());
}
/*******************************************************************************
** Execute one bounded-time read and close JDBC resources.
*******************************************************************************/
private static <T> List<T> read(Connection connection, String sql, RowReader<T> mapper) throws SQLException
{
List<T> result = new ArrayList<>();
try(Statement statement = connection.createStatement())
{
statement.setQueryTimeout(30);
try(ResultSet rows = statement.executeQuery(sql))
{
while(rows.next())
{
result.add(mapper.read(rows));
}
}
}
return (result);
}
/*******************************************************************************
** Checked JDBC row mapping.
*******************************************************************************/
private interface RowReader<T>
{
/*******************************************************************************
** Map one row.
*******************************************************************************/
T read(ResultSet row) throws SQLException;
}
}
@@ -21,11 +21,44 @@
package com.kingsrook.qbits.dbperformance.provider;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
/*******************************************************************************
** ProviderCapabilities read from database statistics; no persistence or side effects.
** What one provider can read on one connection; no persistence or side effects.
** {@code statsAccess} is a stable code persisted on settings (FULL, or the most
** important missing prerequisite); {@code message} is the provider's own
** next step; {@code features} lists the statistics areas available right now.
*******************************************************************************/
public record ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message)
public record ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message, Set<StatsFeature> features)
{
/*******************************************************************************
** Defensive copy.
*******************************************************************************/
public ProviderCapabilities
{
features = features == null || features.isEmpty() ? Set.of() : Collections.unmodifiableSet(EnumSet.copyOf(features));
}
/*******************************************************************************
** Pre-feature form: a supported provider is assumed to offer every area.
*******************************************************************************/
public ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message)
{
this(supported, version, statements, readAllStats, statsAccess, message, supported ? EnumSet.allOf(StatsFeature.class) : Set.of());
}
/*******************************************************************************
** Whether an area is available on this connection.
*******************************************************************************/
public boolean has(StatsFeature feature)
{
return (features.contains(feature));
}
}
@@ -0,0 +1,31 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
/*******************************************************************************
** One guided-setup row: the prerequisite, its observed state and the next step.
*******************************************************************************/
public record SetupCheck(String check, String state, String nextStep)
{
}
@@ -0,0 +1,117 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
/*******************************************************************************
** One area of database statistics. A provider declares which areas its vendor
** can supply ({@link DbStatsProvider#features()}) and which are available on a
** given connection ({@link ProviderCapabilities#features()}); presentation and
** analysis degrade per area instead of per vendor.
*******************************************************************************/
public enum StatsFeature
{
/** Transaction, block, temp-file and deadlock counters for the database. */
DATABASE_COUNTERS("Database counters"),
/** Buffer-cache hit and read counters (cache hit ratio). */
CACHE_COUNTERS("Cache hit ratio"),
/** Database size on disk. */
DATABASE_SIZE("Database size"),
/** Configured connection limit. */
MAX_CONNECTIONS("Connection limit"),
/** Cumulative per-statement calls, time and rows. */
STATEMENTS("Statement statistics"),
/** Per-statement shared-block and temp-block I/O. */
STATEMENT_IO("Statement I/O"),
/** A server-side reset time for statement statistics. */
STATEMENT_RESET("Statement reset time"),
/** Live row estimates per table. */
TABLE_ROWS("Table row estimates"),
/** Table, index and toast sizes. */
TABLE_SIZES("Table sizes"),
/** Sequential/index scan and tuple write counters per table. */
TABLE_ACCESS("Table access counters"),
/** Dead tuples, vacuum/analyze times and freeze age. */
TABLE_MAINTENANCE("Table maintenance"),
/** Index definitions and key columns. */
INDEX_DEFINITIONS("Index definitions"),
/** Per-index scan counters. */
INDEX_USAGE("Index usage"),
/** Per-index size. */
INDEX_SIZES("Index sizes"),
/** Live sessions with state and current statement. */
SESSIONS("Live sessions"),
/** Which session blocks which. */
BLOCKING("Blocking"),
/** Wait event type and name per session. */
WAIT_EVENTS("Wait events"),
/** Transaction start time per session. */
TRANSACTION_AGE("Transaction age"),
/** Connection counts by user, application and state. */
CONNECTIONS("Connection counts"),
/** Plans as structured JSON with costs (renderable tree). */
EXPLAIN_JSON("Structured query plans"),
/** Plans as vendor text (no cost tree). */
EXPLAIN_TEXT("Text query plans"),
/** Plans for parameterized SQL without values. */
EXPLAIN_GENERIC("Generic plans"),
/** Executing the query to measure the plan (ANALYZE). */
EXPLAIN_ANALYZE("Explain analyze");
private final String label;
/*******************************************************************************
** Constructor.
*******************************************************************************/
StatsFeature(String label)
{
this.label = label;
}
/*******************************************************************************
** Human-readable area name.
*******************************************************************************/
public String getLabel()
{
return (label);
}
/*******************************************************************************
** Immutable copy of a feature list.
*******************************************************************************/
public static Set<StatsFeature> of(StatsFeature... features)
{
EnumSet<StatsFeature> set = EnumSet.noneOf(StatsFeature.class);
Collections.addAll(set, features);
return (Collections.unmodifiableSet(set));
}
}
@@ -47,7 +47,7 @@ public class UnsupportedStatsProvider implements DbStatsProvider
@Override
public ProviderCapabilities capabilities(Connection connection)
{
return (new ProviderCapabilities(false, 0, false, false, "UNSUPPORTED", "This database vendor is not supported; PostgreSQL 13 or newer is required."));
return (new ProviderCapabilities(false, 0, false, false, "UNSUPPORTED", "This database vendor is not supported; PostgreSQL 13+ and H2 2.x are (see docs/PROVIDERS.md)."));
}
@@ -0,0 +1,44 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.widgets;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
/*******************************************************************************
** Snapshot time series rendered by the standard typed line-chart component.
*******************************************************************************/
public class DbPerfTimeChart extends ChartData
{
/*******************************************************************************
** Match the registered metadata type.
*******************************************************************************/
@Override
public String getType()
{
return (WidgetType.LINE_CHART.getType());
}
}
@@ -0,0 +1,547 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.widgets;
import java.sql.Connection;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.api.DbPerformanceData;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.provider.ConnectionCount;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.SetupCheck;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ParentWidgetData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockLink;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextStyles;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.map;
import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.number;
/*******************************************************************************
** Typed data only: paginated tables, full-width panels and explicit empty guidance.
*******************************************************************************/
public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
{
/*******************************************************************************
** No privileged read context: both widget and service enforce the caller's view grant.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance());
DbPerformanceData data = new DbPerformanceData(runtime);
data.requireView(null);
try
{
QWidgetData result = switch(input.getWidgetMetaData().getName())
{
case "dbPerfSetup" -> setup(runtime, data);
case "dbPerfDatabaseTime" -> databaseTime(data);
case "dbPerfTopQueries" -> topQueries(data);
case "dbPerfFindings" -> findings(data);
case "dbPerfLiveActivity" -> liveActivity(data);
case "dbPerfConnections" -> connections();
case "dbPerfConnectionStatistics" -> connectionStatistics(data);
case "dbPerfConnectionsByApplication" -> connectionsByApplication(runtime);
case "dbPerfAttribution" -> attribution();
case "dbPerfTables" -> tables(data);
default -> throw new QException("Unknown Database Performance widget.");
};
return (new RenderWidgetOutput(result));
}
catch(QException e)
{
throw (e);
}
catch(Exception e)
{
throw (new QException("Database Performance panel unavailable. Check the backend and statistics access."));
}
}
/*******************************************************************************
** Guided checks do not write settings or require a first successful snapshot.
*******************************************************************************/
public TableData setup(DbPerfRuntime runtime, DbPerformanceData data) throws Exception
{
QRecord settings = data.settings();
String access = settings == null ? "NOT_COLLECTED" : settings.getValueString("statsAccess");
Instant last = settings == null ? null : settings.getValueInstant("lastSnapshotAt");
boolean attribution = settings == null ? Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()) : Boolean.TRUE.equals(settings.getValueBoolean("attributionEnabled"));
List<Map<String, Object>> rows = new ArrayList<>();
rows.add(check("Monitored backend", runtime.getConfig().getBackendName(), "Review Settings for this backend."));
rows.add(check("Provider", settings == null ? "Not detected yet" : settings.getValueString("provider"), "PostgreSQL 13+ and H2 2.x are supported."));
DbStatsProvider provider = settings == null ? providerForBackend(runtime) : DbStatsProviders.forProviderName(settings.getValueString("provider"));
for(SetupCheck check : provider.setupChecks(access, settings == null ? null : settings.getValueString("statsAccessMessage")))
{
rows.add(check(check.check(), check.state(), check.nextStep()));
}
rows.add(check("Last snapshot", last == null ? "Nothing collected yet" : Duration.between(last, Instant.now()).toSeconds() + " seconds ago",
last == null ? "Run Collect now from Snapshots, or wait for the next scheduled collection." : "Collection normally runs every " + SnapshotCollector.positive(settings, "snapshotIntervalMinutes", 5) + " minutes."));
rows.add(check("Attribution", attribution ? "On" : "Off", attribution ? "Attribution starts after the next flush." : "Enable attribution in Settings and the host configuration."));
rows.add(check("Schedules", runtime.getConfig().getSchedulerName() == null ? "Not configured" : "Configured", "Set schedulerName on the qbit and register that scheduler in the host."));
return (table("Run Collect now to start collecting database performance.", rows, "Check", "State", "Next step"));
}
/*******************************************************************************
** Before the first collection the vendor comes from the backend itself;
** an unreachable backend falls back to the unsupported checks.
*******************************************************************************/
private static DbStatsProvider providerForBackend(DbPerfRuntime runtime)
{
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName())))
{
return (DbStatsProviders.forConnection(connection));
}
catch(Exception e)
{
return (DbStatsProviders.forProviderName(null));
}
}
/*******************************************************************************
** Plain text check row, never markup.
*******************************************************************************/
private static Map<String, Object> check(String name, String state, String next)
{
return (map("Check", name, "State", state, "Next step", next));
}
/*******************************************************************************
** Separate labeled time and call series from each snapshot's interval deltas.
*******************************************************************************/
public ChartData databaseTime(DbPerformanceData data) throws QException
{
List<QRecord> snapshots = data.snapshots();
List<String> labels = snapshots.stream().map(row -> row.getValueString("capturedAt")).toList();
List<Number> milliseconds = snapshots.stream().map(row -> (Number) row.getValue("statementTotalMs")).toList();
List<Number> calls = snapshots.stream().map(row -> (Number) row.getValue("statementCalls")).toList();
return (new DbPerfTimeChart().withDescription(snapshots.isEmpty() ? "Database time appears after the next snapshot." : "Last 24 hours; each point is one collection interval.")
.withChartData(new ChartData.Data().withLabels(labels).withDatasets(List.of(
new ChartData.Data.Dataset().withLabel("Database time (ms)").withData(milliseconds).withBackgroundColor("#2563eb"),
new ChartData.Data.Dataset().withLabel("Calls").withData(calls).withBackgroundColor("#16a34a")))));
}
/*******************************************************************************
** Truncated query shapes link to their record and paginate rather than scroll.
*******************************************************************************/
public TableData topQueries(DbPerformanceData data) throws QException
{
List<Map<String, Object>> rows = new ArrayList<>();
for(Map<String, Object> statement : data.statements(100, null))
{
rows.add(map("Query", stack((String) statement.get("queryText"), (String) statement.get("historyPath"),
format(statement.get("calls24h")) + " calls, mean " + format(statement.get("meanMs24h")) + " ms"),
"Total ms (24h)", statement.get("totalMs24h"), "Share of DB time %", statement.get("shareOfDbTime24h"), "Cache hit ratio", statement.get("hitRatio")));
}
TableData table = table("Top queries appear after two snapshots establish interval deltas.", rows, "Query", "Total ms (24h)", "Share of DB time %", "Cache hit ratio");
table.getColumns().getFirst().setType("composite");
return (table);
}
/*******************************************************************************
** Show actionable findings from the parallel findings engine, without changing them.
*******************************************************************************/
public TableData findings(DbPerformanceData data) throws QException
{
List<Map<String, Object>> rows = new ArrayList<>();
for(QRecord row : data.findings(100))
{
rows.add(map("Severity", row.getValueString("severity"), "Title", link(row.getValueString("title"), data.recordPath("db_perf_finding", row.getValueInteger("id"))),
"Rule", row.getValueString("ruleCode"), "Last seen", row.getValue("lastSeenAt")));
}
TableData table = table("Findings appear after the next analysis of collected snapshots.", rows, "Severity", "Title", "Rule", "Last seen");
table.getColumns().get(1).setType("composite");
return (table);
}
/*******************************************************************************
** Even callers with live-text permission see only fingerprinted text on home.
*******************************************************************************/
@SuppressWarnings("unchecked")
public TableData liveActivity(DbPerformanceData data) throws Exception
{
QRecord settings = data.settings();
int longRunning = settings == null ? 30 : SnapshotCollector.positive(settings, "activityLongRunningSeconds", 30);
int idle = settings == null ? 60 : SnapshotCollector.positive(settings, "idleInTransactionSeconds", 60);
List<Map<String, Object>> sessions = (List<Map<String, Object>>) data.activity((String) null, false).get("sessions");
List<Map<String, Object>> rows = sessions.stream().filter(row -> !((List<?>) row.get("blockedBy")).isEmpty() || ((Number) row.get("blockingCount")).intValue() > 0
|| "active".equals(row.get("state")) && numeric(row.get("querySeconds")) >= longRunning
|| Objects.toString(row.get("state"), "").startsWith("idle in transaction") && numeric(row.get("xactSeconds")) >= idle).toList();
List<Map<String, Object>> cells = new ArrayList<>();
for(Map<String, Object> row : rows)
{
boolean blocked = !((List<?>) row.get("blockedBy")).isEmpty();
int blocking = ((Number) row.get("blockingCount")).intValue();
cells.add(map("Session", stack("pid " + row.get("pid") + ", " + Objects.toString(row.get("user"), "?") + ", " + Objects.toString(row.get("application"), ""), null,
Objects.toString(row.get("normalizedText"), "")),
"State", Objects.toString(row.get("state"), "") + (row.get("waitEvent") == null ? "" : " (" + row.get("waitEvent") + ")"),
"Query / transaction s", format(row.get("querySeconds")) + " / " + format(row.get("xactSeconds")),
"Blocking", blocked ? "blocked by " + row.get("blockedBy") : (blocking > 0 ? "blocks " + blocking : "")));
}
TableData table = table("No long-running or blocked sessions right now", cells, "Session", "State", "Query / transaction s", "Blocking");
table.getColumns().getFirst().setType("composite");
return (table);
}
/*******************************************************************************
** Capacity and breakdown are stacked, each occupying the whole row.
*******************************************************************************/
public ParentWidgetData connections()
{
return (new ParentWidgetData().withChildWidgetNameList(List.of("dbPerfConnectionStatistics", "dbPerfConnectionsByApplication")));
}
/*******************************************************************************
** Last observed capacity plus summed 24-hour delta metrics, never cumulative totals.
*******************************************************************************/
public MultiStatisticsData connectionStatistics(DbPerformanceData data) throws QException
{
QRecord latest = data.latest();
List<QRecord> snapshots = data.snapshots();
List<MultiStatisticsData.StatisticsGroupData.Statistic> values = new ArrayList<>();
values.add(stat("Total connections", latest == null ? null : number(latest, "connectionsTotal")));
////////////////////////////////////////////////////////////////////////
// figures the provider does not keep (H2: limit, cache, temp, //
// deadlocks) are stored as null and left out rather than shown as 0 //
////////////////////////////////////////////////////////////////////////
boolean limit = latest == null || latest.getValue("maxConnections") != null;
boolean cache = latest == null || latest.getValue("cacheHitRatio") != null;
boolean counters = snapshots.isEmpty() || snapshots.stream().anyMatch(row -> row.getValue("deadlocks") != null || row.getValue("tempBytes") != null);
if(limit)
{
values.add(stat("Maximum connections", latest == null ? null : number(latest, "maxConnections")));
}
values.add(stat("Active", latest == null ? null : number(latest, "connectionsActive")));
values.add(stat("Idle", latest == null ? null : number(latest, "connectionsIdle")));
values.add(stat("Idle in transaction", latest == null ? null : number(latest, "connectionsIdleInTx")));
if(cache)
{
values.add(stat("Cache hit %", latest == null ? null : 100 * number(latest, "cacheHitRatio")));
}
if(counters)
{
values.add(stat("Temp MB (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "tempBytes")).sum() / 1048576));
values.add(stat("Deadlocks (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "deadlocks")).sum()));
}
return (new MultiStatisticsData(null, List.of(new MultiStatisticsData.StatisticsGroupData().withHeader("Connection capacity")
.withSubheader(latest == null ? "Capacity appears after the next snapshot." : "Latest snapshot; percentages and MB rounded to whole numbers.").withStatisticList(values))));
}
/*******************************************************************************
** Current connection counts by application, user and state, directly from provider.
*******************************************************************************/
public TableData connectionsByApplication(DbPerfRuntime runtime) throws Exception
{
List<Map<String, Object>> rows = new ArrayList<>();
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName())))
{
for(ConnectionCount count : DbStatsProviders.forConnection(connection).readConnections(connection))
{
rows.add(map("Application", count.applicationName(), "User", count.dbUser(), "State", count.state(), "Connections", count.connectionCount()));
}
}
return (table("No connection counts available for this provider.", rows, "Application", "User", "State", "Connections"));
}
/*******************************************************************************
** Most (unit, table) groups summed for the attribution panel. Production
** shows about a thousand; the cap only guards pathological unit names.
*******************************************************************************/
public static final int MAX_ATTRIBUTION_GROUPS = 5000;
/*******************************************************************************
** Aggregate units over 24 hours and show repeated-statement patterns below them.
** The sums run in the database: a day of attribution detail is hundreds of
** thousands of rows (about 540 MB), and loading it into the heap on a widget
** request ran the web pod out of memory (2026-10-16).
*******************************************************************************/
public MultiTableData attribution() throws QException
{
GroupBy kindGroup = new GroupBy(QFieldType.STRING, "unitKind");
GroupBy nameGroup = new GroupBy(QFieldType.STRING, "unitName");
GroupBy tableGroup = new GroupBy(QFieldType.STRING, "tableName");
Aggregate statementsSum = new Aggregate("statements", AggregateOperator.SUM);
Aggregate totalMsSum = new Aggregate("totalMs", AggregateOperator.SUM);
List<AggregateResult> results = new AggregateAction().execute(new AggregateInput("db_perf_attribution")
.withFilter(since("windowEnd").withOrderBy(new QFilterOrderByAggregate(totalMsSum, false)))
.withGroupBy(kindGroup).withGroupBy(nameGroup).withGroupBy(tableGroup)
.withAggregate(statementsSum).withAggregate(totalMsSum)
.withLimit(MAX_ATTRIBUTION_GROUPS)
.withTransaction(QContext.getQBackendTransaction())).getResults();
Map<String, Map<String, Object>> grouped = new LinkedHashMap<>();
Map<String, Map<String, Double>> tableTimes = new LinkedHashMap<>();
double total = results.stream().mapToDouble(row -> numeric(row.getAggregateValue(totalMsSum))).sum();
for(AggregateResult row : results)
{
String kind = (String) row.getGroupByValue(kindGroup);
String name = (String) row.getGroupByValue(nameGroup);
String key = kind + "\n" + name;
Map<String, Object> group = grouped.computeIfAbsent(key, ignored -> map("Kind", kind, "Name", name, "Statements", 0.0, "Total ms", 0.0));
group.put("Statements", numeric(group.get("Statements")) + numeric(row.getAggregateValue(statementsSum)));
group.put("Total ms", numeric(group.get("Total ms")) + numeric(row.getAggregateValue(totalMsSum)));
tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getGroupByValue(tableGroup), "Unknown"), numeric(row.getAggregateValue(totalMsSum)), Double::sum);
}
grouped.forEach((key, group) ->
{
group.put("Share %", total == 0 ? 0 : 100 * numeric(group.get("Total ms")) / total);
group.put("Top table", tableTimes.get(key).entrySet().stream().max(Map.Entry.comparingByValue()).orElseThrow().getKey());
});
List<Map<String, Object>> units = grouped.values().stream().sorted(Comparator.comparingDouble((Map<String, Object> row) -> numeric(row.get("Total ms"))).reversed())
.map(row -> map("Unit of work", stack(Objects.toString(row.get("Name"), ""), null, kindLabel(row.get("Kind")) + ", top table " + row.get("Top table")),
"Statements", row.get("Statements"), "Total ms (24h)", row.get("Total ms"), "Share %", row.get("Share %"))).toList();
List<Map<String, Object>> repeats = DbPerfStore.query("db_perf_repeat_pattern", since("windowEnd").withOrderBy(new QFilterOrderBy("maxRepeats", false)).withLimit(100)).stream()
.map(row -> map("Unit of work", stack(Objects.toString(row.getValueString("unitName"), ""), null,
kindLabel(row.getValueString("unitKind")) + ": " + truncate(row.getValueString("sqlText"), 160)),
"Most repeats in one unit", row.getValue("maxRepeats"), "Units affected", row.getValue("unitsAffected"), "Total repeats", row.getValue("totalRepeats"))).toList();
TableData unitTable = table("Attribution starts after the next flush", units, "Unit of work", "Statements", "Total ms (24h)", "Share %");
unitTable.getColumns().getFirst().setType("composite");
TableData repeatTable = table("Repeated statements appear after the next attribution flush when the repetition threshold is reached.", repeats,
"Unit of work", "Most repeats in one unit", "Units affected", "Total repeats");
repeatTable.getColumns().getFirst().setType("composite");
return (new MultiTableData(List.of(unitTable, repeatTable)));
}
/*******************************************************************************
** Sequential-scan pressure and seven-day size growth are separate ranked tables.
*******************************************************************************/
public MultiTableData tables(DbPerformanceData data) throws QException
{
QRecord settings = data.settings();
DbStatsProvider provider = settings == null || settings.getValueString("provider") == null ? null : DbStatsProviders.forProviderName(settings.getValueString("provider"));
List<QRecord> growing = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("growthBytes7d", false)).withLimit(100));
TableData growth = table("Growth appears after table history has accumulated.", growing.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")),
"Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(),
"Table", "Size MB", "Growth MB (7 days)");
if(provider != null && !provider.features().contains(StatsFeature.TABLE_ACCESS))
{
//////////////////////////////////////////////////////////////////
// vendors without scan counters (H2) list rows and size instead //
//////////////////////////////////////////////////////////////////
List<QRecord> largest = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("nLiveTup", false)).withLimit(100));
return (new MultiTableData(List.of(table("Tables appear after the next table snapshot (normally hourly).", largest.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Rows (estimate)", row.getValue("nLiveTup"),
"Size MB", mb(row.getValue("totalBytes")))).toList(),
"Table", "Rows (estimate)", "Size MB"), growth)));
}
List<QRecord> scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100));
return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Sequential scans", row.getValue("seqScan"),
"Index scans", row.getValue("idxScan"), "Sequential share %", row.getValue("seqScanShare"))).toList(),
"Table", "Sequential scans", "Index scans", "Sequential share %"), growth)));
}
/*******************************************************************************
** The core TABLE renderer paginates when fixedHeight is absent.
*******************************************************************************/
private static TableData table(String empty, List<Map<String, Object>> rows, String... columns)
{
return (new TableData(null, Arrays.stream(columns).map(name -> new TableData.Column("default", name, name, null, "left")).toList(), rows)
.withRowsPerPage(10).withHidePaginationDropdown(false).withNoRowsFoundHTML(empty));
}
/*******************************************************************************
** Typed record link, never an HTML fragment.
*******************************************************************************/
private static CompositeWidgetData link(String text, String path)
{
return (new CompositeWidgetData().withLayout(CompositeWidgetData.Layout.FLEX_COLUMN)
.withBlock(new TextBlockData().withValues(new TextValues(text)).withLink(new BlockLink(path))));
}
/*******************************************************************************
** A stacked cell: the main text (a record link when a path is given) over a
** smaller secondary line, so tables keep at most four columns.
*******************************************************************************/
private static CompositeWidgetData stack(String main, String path, String secondary)
{
TextBlockData first = new TextBlockData().withValues(new TextValues(truncate(main, 300)));
if(path != null)
{
first.withLink(new BlockLink(path));
}
CompositeWidgetData cell = new CompositeWidgetData().withLayout(CompositeWidgetData.Layout.FLEX_COLUMN).withBlock(first);
if(secondary != null && !secondary.isBlank())
{
cell.withBlock(new TextBlockData().withValues(new TextValues(truncate(secondary, 300))).withStyles(new TextStyles().withColor("MUTED")));
}
return (cell);
}
/*******************************************************************************
** Plain label for a unit-of-work kind.
*******************************************************************************/
private static String kindLabel(Object kind)
{
return (switch(Objects.toString(kind, ""))
{
case "HTTP_REQUEST" -> "Request";
case "SCHEDULED_JOB" -> "Scheduled job";
case "PROCESS" -> "Process";
default -> "Other";
});
}
/*******************************************************************************
** Text capped with an ellipsis.
*******************************************************************************/
private static String truncate(String value, int max)
{
String text = value == null ? "" : value;
return (text.length() <= max ? text : text.substring(0, max - 1) + "…");
}
/*******************************************************************************
** A number for a secondary line: whole numbers without decimals.
*******************************************************************************/
private static String format(Object value)
{
if(value == null)
{
return ("0");
}
double number = ((Number) value).doubleValue();
return (number == Math.rint(number) ? String.valueOf((long) number) : String.format(Locale.ROOT, "%.1f", number));
}
/*******************************************************************************
** Bytes as whole megabytes.
*******************************************************************************/
private static Long mb(Object bytes)
{
return (bytes == null ? null : Math.round(((Number) bytes).doubleValue() / 1048576));
}
/*******************************************************************************
** Trailing-day filter for time-window records.
*******************************************************************************/
private static QQueryFilter since(String field)
{
return (new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, Instant.now().minusSeconds(86400))));
}
/*******************************************************************************
** Core multi-statistics values are nullable integers.
*******************************************************************************/
private static MultiStatisticsData.StatisticsGroupData.Statistic stat(String label, Double value)
{
return (new MultiStatisticsData.StatisticsGroupData.Statistic(label, value == null ? null : (int) Math.min(Integer.MAX_VALUE, Math.round(value)), null));
}
/*******************************************************************************
** Numbers in the shared read model retain their numeric type.
*******************************************************************************/
private static double numeric(Object value)
{
return (value == null ? 0 : ((Number) value).doubleValue());
}
}
@@ -0,0 +1,81 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.widgets;
import java.util.List;
import com.kingsrook.qbits.dbperformance.DbPerfPermissions;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
/*******************************************************************************
** Full-width typed home panels, with one panel per row and bounded refresh rates.
*******************************************************************************/
public final class DbPerfWidgets
{
public static final List<String> HOME = List.of("dbPerfSetup", "dbPerfDatabaseTime", "dbPerfTopQueries", "dbPerfFindings",
"dbPerfLiveActivity", "dbPerfConnections", "dbPerfAttribution", "dbPerfTables");
/*******************************************************************************
** Utility class.
*******************************************************************************/
private DbPerfWidgets()
{
}
/*******************************************************************************
** Additive registration, preserving phase A record-child widgets and app sections.
*******************************************************************************/
public static void register(QInstance instance, DbPerfQBitConfig config)
{
add(instance, config, HOME.get(0), "Setup", WidgetType.TABLE, 60);
add(instance, config, HOME.get(1), "Database time", WidgetType.LINE_CHART, 60);
add(instance, config, HOME.get(2), "Top queries", WidgetType.TABLE, 60);
add(instance, config, HOME.get(3), "Findings", WidgetType.TABLE, 60);
add(instance, config, HOME.get(4), "Live activity", WidgetType.TABLE, 30);
add(instance, config, HOME.get(5), "Connections", WidgetType.PARENT_WIDGET, 30);
add(instance, config, "dbPerfConnectionStatistics", "Connection capacity", WidgetType.MULTI_STATISTICS, 30);
add(instance, config, "dbPerfConnectionsByApplication", "Connections by application and user", WidgetType.TABLE, 30);
add(instance, config, HOME.get(6), "QQQ attribution", WidgetType.MULTI_TABLE, 60);
add(instance, config, HOME.get(7), "Tables", WidgetType.MULTI_TABLE, 60);
instance.getApp(config.getAppName()).withWidgets(HOME);
}
/*******************************************************************************
** Permission and responsive width also apply when a widget is requested directly.
*******************************************************************************/
private static void add(QInstance instance, DbPerfQBitConfig config, String name, String label, WidgetType type, int refresh)
{
instance.addWidget(new QWidgetMetaData().withName(name).withLabel(label).withType(type.getType()).withGridColumns(12)
.withCodeReference(new QCodeReference(DbPerfWidgetRenderer.class)).withDefaultValue("refreshSeconds", refresh)
.withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view")));
}
}
@@ -578,30 +578,34 @@ ALTER TABLE db_perf_explain ADD CONSTRAINT fk_explain_statement_id FOREIGN KEY (
-- changeset qbit-db-performance:33
-- comment: Suggestion Rules insert-only defaults; administrator edits survive.
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'SLOW_QUERY', 'Slow query', 'Slow query.', TRUE, 'WARNING', 500, 'Mean execution time (ms)', 5, 'Database time (%)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'SLOW_QUERY');
SELECT 'SLOW_QUERY', 'Slow query', 'Statements whose average run time is high, or that use a large share of all database time over the last 24 hours. Open the query and run Query plan to see why.', TRUE, 'WARNING', 500, 'Average run time (ms)', 5, 'Share of all database time (%)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'SLOW_QUERY');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'MISSING_INDEX', 'Possible missing index', 'Possible missing index.', TRUE, 'WARNING', 50, 'Sequential scan share (%)', 1000000, 'Tuples read per "day"' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'MISSING_INDEX');
SELECT 'MISSING_INDEX', 'Possible missing index', 'Tables that are mostly read by full sequential scans of many rows, where the busiest queries filter on columns no index starts with. The finding includes a CREATE INDEX CONCURRENTLY changeset to review and commit.', TRUE, 'WARNING', 50, 'Share of scans that are sequential (%)', 1000000, 'Rows read by sequential scans per day' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'MISSING_INDEX');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'UNUSED_INDEX', 'Unused index', 'Unused index.', TRUE, 'WARNING', 14, 'Days since statistics reset', 1, 'Index size (MB)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'UNUSED_INDEX');
SELECT 'UNUSED_INDEX', 'Unused index', 'Non-unique indexes that no query has used since statistics were last reset. Every write still pays to maintain them. The finding includes a DROP INDEX CONCURRENTLY changeset with a rollback.', TRUE, 'WARNING', 14, 'Days of statistics before judging', 1, 'Smallest index size to report (MB)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'UNUSED_INDEX');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'DUPLICATE_INDEX', 'Duplicate index', 'Duplicate index.', TRUE, 'WARNING', NULL, NULL, NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'DUPLICATE_INDEX');
SELECT 'DUPLICATE_INDEX', 'Duplicate index', 'Indexes with the same columns as another index on the table, or whose columns are a leading prefix of another non-unique index. Unique and primary key indexes are never proposed for removal.', TRUE, 'WARNING', NULL, NULL, NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'DUPLICATE_INDEX');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'N_PLUS_ONE', 'Repeated statements', 'Repeated statements.', TRUE, 'WARNING', 10, 'Repeats per unit', 5, 'Affected units' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'N_PLUS_ONE');
SELECT 'N_PLUS_ONE', 'Repeated statements (N+1)', 'The same statement shape run many times within one request, process or scheduled job, usually a lookup inside a loop. Batch it into one query with IN (...) or a join.', TRUE, 'WARNING', 10, 'Repeats in one unit of work', 5, 'Units of work affected per day' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'N_PLUS_ONE');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'HOT_JOB', 'Expensive job', 'Expensive job.', TRUE, 'WARNING', 10, 'Database time (%)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'HOT_JOB');
SELECT 'HOT_JOB', 'Expensive job or process', 'Scheduled jobs and processes that account for a large share of the database time QQQ issued in the last 24 hours.', TRUE, 'WARNING', 10, 'Share of attributed database time (%)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'HOT_JOB');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'OVER_AUDITED', 'High audit volume', 'High audit volume.', TRUE, 'WARNING', 50000, 'Audit inserts per "day"', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'OVER_AUDITED');
SELECT 'OVER_AUDITED', 'High audit volume', 'QQQ audit tables that grow by many rows a day. Lowering the audit level of the noisiest tables reduces writes and storage.', TRUE, 'WARNING', 50000, 'Audit rows inserted per day', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'OVER_AUDITED');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'BLOAT', 'Dead tuples', 'Dead tuples.', TRUE, 'WARNING', 20, 'Dead tuple ratio (%)', 10000, 'Dead tuples' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'BLOAT');
SELECT 'BLOAT', 'Dead rows (bloat)', 'Tables where a large share of rows are dead versions waiting for vacuum. Bloat slows scans and wastes space. Tune autovacuum for the table.', TRUE, 'WARNING', 20, 'Dead rows as a share of live rows (%)', 10000, 'Fewest dead rows to report' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'BLOAT');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'VACUUM_STALE', 'Stale vacuum', 'Stale vacuum.', TRUE, 'WARNING', 7, 'Days since vacuum', 150000000, 'Transaction freeze age' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'VACUUM_STALE');
SELECT 'VACUUM_STALE', 'Vacuum overdue', 'Tables with recent writes that have not been vacuumed for a long time, or whose oldest transaction ID is getting close to the freeze limit.', TRUE, 'WARNING', 7, 'Days since the last vacuum', 150000000, 'Transaction ID age (freeze age)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'VACUUM_STALE');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'POOL_SIZING', 'Connection pool sizing', 'Connection pool sizing.', TRUE, 'WARNING', 80, 'Connections used (%)', 50, 'Idle connections (%)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'POOL_SIZING');
SELECT 'POOL_SIZING', 'Connection pool sizing', 'Peak connections close to max_connections, or many idle connections held by an application. The finding suggests pool sizes per application.', TRUE, 'WARNING', 80, 'Peak connections as a share of max_connections (%)', 50, 'Idle connections as a share of all connections (%)' WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'POOL_SIZING');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'IDLE_IN_TRANSACTION', 'Idle transaction', 'Idle transaction.', TRUE, 'WARNING', 60, 'Idle seconds', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'IDLE_IN_TRANSACTION');
SELECT 'IDLE_IN_TRANSACTION', 'Idle in transaction', 'Sessions that opened a transaction and then sat idle, holding locks and blocking vacuum. Fix the transaction scope in the application.', TRUE, 'WARNING', 60, 'Seconds idle in a transaction', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'IDLE_IN_TRANSACTION');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'CACHE_HIT_LOW', 'Low cache hit ratio', 'Low cache hit ratio.', TRUE, 'WARNING', 95, 'Cache hit ratio (%)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'CACHE_HIT_LOW');
SELECT 'CACHE_HIT_LOW', 'Low cache hit ratio', 'The share of block reads served from shared buffers fell below the threshold over the last 24 hours, so more reads go to disk.', TRUE, 'WARNING', 95, 'Cache hit ratio (%)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'CACHE_HIT_LOW');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'TEMP_SPILL', 'Temporary disk spill', 'Temporary disk spill.', TRUE, 'WARNING', 100, 'Temporary data per "day" (MB)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'TEMP_SPILL');
SELECT 'TEMP_SPILL', 'Temporary disk spill', 'Statements that write temporary files because sorts or hashes do not fit in work_mem.', TRUE, 'WARNING', 100, 'Temporary data written per day (MB)', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'TEMP_SPILL');
INSERT INTO db_perf_rule (code,label,description,enabled,severity,threshold_1,threshold_1_label,threshold_2,threshold_2_label)
SELECT 'DEADLOCKS', 'Deadlocks', 'Deadlocks.', TRUE, 'WARNING', 1, 'Deadlocks per "day"', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'DEADLOCKS');
SELECT 'DEADLOCKS', 'Deadlocks', 'Transactions that deadlocked and were cancelled by PostgreSQL. Usually two code paths that lock rows in a different order.', TRUE, 'WARNING', 1, 'Deadlocks per day', NULL, NULL WHERE NOT EXISTS (SELECT 1 FROM db_perf_rule WHERE code = 'DEADLOCKS');
-- changeset qbit-db-performance:34
-- comment: Repeated Statements: the sampling rate of each window, so repeat counts can be read at scale.
ALTER TABLE db_perf_repeat_pattern ADD COLUMN IF NOT EXISTS sample_rate DECIMAL(5,4);
@@ -0,0 +1 @@
${project.version}
@@ -0,0 +1,976 @@
bundle: qbit-db-performance
version: ${project.version}
layer: QBIT
entries:
- key: app:dbPerformance;slot:help
summary: "Find expensive queries, inspect query plans and review database capacity."
content: |
Start with Setup. Run {{label:dbPerfCollectNow}} from Snapshots to establish the first baseline. Two collections produce useful statement deltas. Table and index data normally refresh hourly; the dashboard refreshes every 30 to 60 seconds.
Use {{label:dbPerfQueryPlan}} on a query to inspect its plan. PLAN does not execute the query; generic plans require PostgreSQL 16+. ANALYZE executes a read-only SELECT with a timeout and always rolls back. Parameters and ANALYZE plans are never stored. Raw live SQL is permission-gated and never persisted.
All data lives in standard QQQ tables. The host owns scheduler registration and permissions. Attribution and findings require their corresponding runtime collectors; empty panels explain when data will arrive.
- key: table:db_perf_activity_sample;slot:help
summary: "Notable session samples with normalized text, never raw query text."
content: |
The collector samples long-running, blocked and idle-in-transaction sessions every collection. Quiet databases can remain empty. Retained for detailRetentionDays (30). Use Live activity for current sessions.
- key: table:db_perf_activity_sample;slot:emptyState
summary: "Notable session samples with normalized text, never raw query text."
content: |
The collector samples long-running, blocked and idle-in-transaction sessions every collection. Quiet databases can remain empty. Retained for detailRetentionDays (30). Use Live activity for current sessions.
- key: table:db_perf_activity_sample;field:snapshotId
summary: "Reference to the related snapshot record."
- key: table:db_perf_activity_sample;field:capturedAt
summary: "Time this database sample was captured."
- key: table:db_perf_activity_sample;field:pid
summary: "Pid reported for this activity sample record."
- key: table:db_perf_activity_sample;field:dbUser
summary: "Database role associated with this statement, session or connection group."
- key: table:db_perf_activity_sample;field:applicationName
summary: "Client application name reported by PostgreSQL."
- key: table:db_perf_activity_sample;field:clientAddr
summary: "Client address reported by the statistics provider."
- key: table:db_perf_activity_sample;field:state
summary: "State reported for this activity sample record."
- key: table:db_perf_activity_sample;field:waitEventType
summary: "PostgreSQL category of the current wait, such as Lock or IO."
- key: table:db_perf_activity_sample;field:waitEvent
summary: "PostgreSQL event on which this backend is currently waiting."
- key: table:db_perf_activity_sample;field:backendType
summary: "Backend type reported for this activity sample record."
- key: table:db_perf_activity_sample;field:xactStart
summary: "Xact start reported for this activity sample record."
- key: table:db_perf_activity_sample;field:queryStart
summary: "Query start reported for this activity sample record."
- key: table:db_perf_activity_sample;field:stateChange
summary: "State change reported for this activity sample record."
- key: table:db_perf_activity_sample;field:querySeconds
summary: "Age of the current query in seconds."
- key: table:db_perf_activity_sample;field:xactSeconds
summary: "Age of the current transaction in seconds."
- key: table:db_perf_activity_sample;field:blockedBy
summary: "Blocking process identifiers reported by PostgreSQL."
- key: table:db_perf_activity_sample;field:blockingCount
summary: "Number of sessions directly blocked by this session."
- key: table:db_perf_activity_sample;field:reason
summary: "Reason reported for this activity sample record."
- key: table:db_perf_activity_sample;field:queryId
summary: "PostgreSQL query identifier; fingerprint provides cross-source normalized matching."
- key: table:db_perf_activity_sample;field:statementId
summary: "Reference to the related statement record."
- key: table:db_perf_activity_sample;field:queryFingerprint
summary: "Query fingerprint reported for this activity sample record."
- key: table:db_perf_activity_sample;field:queryTextNormalized
summary: "Query text normalized reported for this activity sample record."
- key: table:db_perf_attribution;slot:help
summary: "QQQ request, process and job database time by statement shape."
content: |
The host attribution listener flushes one batch per attributionFlushSeconds (60 by default). Retained for detailRetentionDays (30). Attribution starts after the next flush; enable the listener in the host if empty.
- key: table:db_perf_attribution;slot:emptyState
summary: "QQQ request, process and job database time by statement shape."
content: |
The host attribution listener flushes one batch per attributionFlushSeconds (60 by default). Retained for detailRetentionDays (30). Attribution starts after the next flush; enable the listener in the host if empty.
- key: table:db_perf_attribution;field:windowStart
summary: "Beginning of the attribution aggregation window."
- key: table:db_perf_attribution;field:windowEnd
summary: "End of the attribution aggregation window."
- key: table:db_perf_attribution;field:podName
summary: "Application pod that observed and flushed this attribution window."
- key: table:db_perf_attribution;field:unitKind
summary: "Kind of QQQ unit of work, such as route, process or scheduled job."
- key: table:db_perf_attribution;field:unitName
summary: "Name of the QQQ unit responsible for database work."
- key: table:db_perf_attribution;field:processName
summary: "QQQ process responsible for the attributed work, when known."
- key: table:db_perf_attribution;field:tableName
summary: "Table name reported for this attribution record."
- key: table:db_perf_attribution;field:operation
summary: "Normalized SQL operation, such as SELECT, INSERT, UPDATE or DELETE."
- key: table:db_perf_attribution;field:fingerprint
summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded."
- key: table:db_perf_attribution;field:sqlText
summary: "Normalized SQL shape without literal values. Parameter values are never stored."
- key: table:db_perf_attribution;field:units
summary: "Number of sampled QQQ units contributing to this aggregate."
- key: table:db_perf_attribution;field:statements
summary: "Number of attributed SQL executions during the window."
- key: table:db_perf_attribution;field:totalMs
summary: "Total ms in milliseconds for this measurement."
- key: table:db_perf_attribution;field:maxMs
summary: "Max ms in milliseconds for this measurement."
- key: table:db_perf_attribution;field:rowsCount
summary: "Rows processed during the measured execution window."
- key: table:db_perf_attribution;field:errors
summary: "Number of failed SQL executions observed in the attribution window."
- key: table:db_perf_attribution;field:sampleRate
summary: "Fraction of units sampled in this attribution window."
- key: table:db_perf_attribution_daily;slot:help
summary: "Daily database-time attribution for QQQ units of work."
content: |
The daily rollup aggregates flushed attribution windows at 03:15 by default. Retained for rollupRetentionDays (365). Wait for a flush and the next daily rollup when empty.
- key: table:db_perf_attribution_daily;slot:emptyState
summary: "Daily database-time attribution for QQQ units of work."
content: |
The daily rollup aggregates flushed attribution windows at 03:15 by default. Retained for rollupRetentionDays (365). Wait for a flush and the next daily rollup when empty.
- key: table:db_perf_attribution_daily;field:day
summary: "Calendar day in the configured qbit time zone for this rollup."
- key: table:db_perf_attribution_daily;field:rollupKey
summary: "Stable identity used to replace a daily aggregate idempotently."
- key: table:db_perf_attribution_daily;field:unitKind
summary: "Kind of QQQ unit of work, such as route, process or scheduled job."
- key: table:db_perf_attribution_daily;field:unitName
summary: "Name of the QQQ unit responsible for database work."
- key: table:db_perf_attribution_daily;field:processName
summary: "QQQ process responsible for the attributed work, when known."
- key: table:db_perf_attribution_daily;field:tableName
summary: "Table name reported for this attribution daily record."
- key: table:db_perf_attribution_daily;field:operation
summary: "Normalized SQL operation, such as SELECT, INSERT, UPDATE or DELETE."
- key: table:db_perf_attribution_daily;field:fingerprint
summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded."
- key: table:db_perf_attribution_daily;field:sqlText
summary: "Normalized SQL shape without literal values. Parameter values are never stored."
- key: table:db_perf_attribution_daily;field:units
summary: "Number of sampled QQQ units contributing to this aggregate."
- key: table:db_perf_attribution_daily;field:statements
summary: "Number of attributed SQL executions during the window."
- key: table:db_perf_attribution_daily;field:totalMs
summary: "Total ms in milliseconds for this measurement."
- key: table:db_perf_attribution_daily;field:maxMs
summary: "Max ms in milliseconds for this measurement."
- key: table:db_perf_attribution_daily;field:rowsCount
summary: "Rows processed during the measured execution window."
- key: table:db_perf_attribution_daily;field:errors
summary: "Number of failed SQL executions observed in the attribution window."
- key: table:db_perf_connection_sample;slot:help
summary: "Connection counts grouped by application, user and state for each snapshot."
content: |
The collector writes these rows every snapshot. Retained for detailRetentionDays (30). Run Collect now or wait for the next scheduled pass when empty.
- key: table:db_perf_connection_sample;slot:emptyState
summary: "Connection counts grouped by application, user and state for each snapshot."
content: |
The collector writes these rows every snapshot. Retained for detailRetentionDays (30). Run Collect now or wait for the next scheduled pass when empty.
- key: table:db_perf_connection_sample;field:snapshotId
summary: "Reference to the related snapshot record."
- key: table:db_perf_connection_sample;field:capturedAt
summary: "Time this database sample was captured."
- key: table:db_perf_connection_sample;field:dbUser
summary: "Database role associated with this statement, session or connection group."
- key: table:db_perf_connection_sample;field:applicationName
summary: "Client application name reported by PostgreSQL."
- key: table:db_perf_connection_sample;field:state
summary: "State reported for this connection sample record."
- key: table:db_perf_connection_sample;field:connectionCount
summary: "Connection count reported for this connection sample record."
- key: table:db_perf_explain;slot:help
summary: "Audited plan requests, including denied, failed and timed-out attempts."
content: |
Query plan and the explain UI API write one audit row per service request. Only literal-free PLAN or GENERIC results may be saved; ANALYZE plans and all parameter values remain transient. Retained for detailRetentionDays (30). Run Query plan on a statement when empty.
- key: table:db_perf_explain;slot:emptyState
summary: "Audited plan requests, including denied, failed and timed-out attempts."
content: |
Query plan and the explain UI API write one audit row per service request. Only literal-free PLAN or GENERIC results may be saved; ANALYZE plans and all parameter values remain transient. Retained for detailRetentionDays (30). Run Query plan on a statement when empty.
- key: table:db_perf_explain;field:statementId
summary: "Reference to the related statement record."
- key: table:db_perf_explain;field:requestedBy
summary: "Session user identifier that requested the plan, even for a denied request."
- key: table:db_perf_explain;field:requestedAt
summary: "When the plan request was received."
- key: table:db_perf_explain;field:mode
summary: "PLAN, automatically selected GENERIC, or explicitly authorized ANALYZE."
- key: table:db_perf_explain;field:sqlFingerprint
summary: "SHA-256 of the normalized query in this explain audit."
- key: table:db_perf_explain;field:sqlText
summary: "Normalized SQL shape without literal values. Parameter values are never stored."
- key: table:db_perf_explain;field:parameterCount
summary: "Number of positional parameter slots; values are never stored."
- key: table:db_perf_explain;field:durationMs
summary: "Duration ms in milliseconds for this measurement."
- key: table:db_perf_explain;field:outcome
summary: "OK, ERROR, DENIED or TIMEOUT; every explain service request is audited."
- key: table:db_perf_explain;field:errorMessage
summary: "Sanitized failure description. Database SQL errors and parameter values are excluded."
- key: table:db_perf_explain;field:planJson
summary: "Saved PostgreSQL JSON plan, only for eligible literal-free PLAN or GENERIC requests."
- key: table:db_perf_explain;field:planText
summary: "Readable indented plan tree; never stored for ANALYZE."
- key: table:db_perf_explain;field:totalCost
summary: "Total cost reported for this explain record."
- key: table:db_perf_finding;slot:help
summary: "Ranked, deduplicated suggestions with supporting evidence and proposed fixes."
content: |
The findings engine analyzes collected data hourly. OPEN and ACKNOWLEDGED findings remain actionable; resolved or ignored stale findings are purged after rollupRetentionDays (365). Wait for collection and the next analysis when empty. Proposed migrations require developer review; this qbit never applies them.
- key: table:db_perf_finding;slot:emptyState
summary: "Ranked, deduplicated suggestions with supporting evidence and proposed fixes."
content: |
The findings engine analyzes collected data hourly. OPEN and ACKNOWLEDGED findings remain actionable; resolved or ignored stale findings are purged after rollupRetentionDays (365). Wait for collection and the next analysis when empty. Proposed migrations require developer review; this qbit never applies them.
- key: table:db_perf_finding;field:backendName
summary: "The configured QQQ backend whose database is monitored."
- key: table:db_perf_finding;field:ruleId
summary: "Reference to the related rule record."
- key: table:db_perf_finding;field:ruleCode
summary: "Rule code reported for this finding record."
- key: table:db_perf_finding;field:dedupeKey
summary: "Stable rule-plus-subject identity that updates an existing finding instead of creating duplicates."
- key: table:db_perf_finding;field:status
summary: "Current lifecycle state. Review the status-specific guidance and related messages."
- key: table:db_perf_finding;field:severity
summary: "Configured importance of a finding or rule; contributes to the ranking score."
- key: table:db_perf_finding;field:score
summary: "Severity weight multiplied by normalized impact; larger scores rank first."
- key: table:db_perf_finding;field:title
summary: "Title reported for this finding record."
- key: table:db_perf_finding;field:summary
summary: "Summary reported for this finding record."
- key: table:db_perf_finding;field:evidence
summary: "Measurements supporting the finding, without raw SQL values."
- key: table:db_perf_finding;field:recommendation
summary: "Suggested developer or operator action; the qbit never applies it automatically."
- key: table:db_perf_finding;field:migrationSnippet
summary: "Proposed Liquibase migration text for review and source control; never executed by this qbit."
- key: table:db_perf_finding;field:subjectTable
summary: "Subject table associated with the finding."
- key: table:db_perf_finding;field:statementId
summary: "Reference to the related statement record."
- key: table:db_perf_finding;field:tableStatId
summary: "Reference to the related table stat record."
- key: table:db_perf_finding;field:indexStatId
summary: "Reference to the related index stat record."
- key: table:db_perf_finding;field:unitName
summary: "Name of the QQQ unit responsible for database work."
- key: table:db_perf_finding;field:firstSeenAt
summary: "First recorded observation of this statement or finding."
- key: table:db_perf_finding;field:lastSeenAt
summary: "Most recent observation of this statement, table, index or finding."
- key: table:db_perf_finding;field:occurrences
summary: "Number of analyses in which this finding has been observed."
- key: table:db_perf_finding;field:resolvedAt
summary: "When this finding was marked resolved."
- key: table:db_perf_finding;field:statusNote
summary: "Human explanation for the latest finding status change."
- key: table:db_perf_finding;field:statusChangedBy
summary: "User identifier responsible for the latest manual status change."
- key: table:db_perf_index_stat;slot:help
summary: "Current index definitions, usage, size and uniqueness."
content: |
The collector upserts indexes with the hourly table pass. Stale indexes are purged after rollupRetentionDays (365). Run Collect now to force collection when empty.
- key: table:db_perf_index_stat;slot:emptyState
summary: "Current index definitions, usage, size and uniqueness."
content: |
The collector upserts indexes with the hourly table pass. Stale indexes are purged after rollupRetentionDays (365). Run Collect now to force collection when empty.
- key: table:db_perf_index_stat;field:backendName
summary: "The configured QQQ backend whose database is monitored."
- key: table:db_perf_index_stat;field:tableStatId
summary: "Reference to the related table stat record."
- key: table:db_perf_index_stat;field:schemaName
summary: "Schema name reported for this index stat record."
- key: table:db_perf_index_stat;field:tableName
summary: "Table name reported for this index stat record."
- key: table:db_perf_index_stat;field:indexName
summary: "Index name reported for this index stat record."
- key: table:db_perf_index_stat;field:relid
summary: "Relid reported for this index stat record."
- key: table:db_perf_index_stat;field:definition
summary: "Definition reported for this index stat record."
- key: table:db_perf_index_stat;field:columns
summary: "Columns reported for this index stat record."
- key: table:db_perf_index_stat;field:isUnique
summary: "Whether this index enforces uniqueness."
- key: table:db_perf_index_stat;field:isPrimary
summary: "Whether this index enforces a primary key."
- key: table:db_perf_index_stat;field:isValid
summary: "Whether PostgreSQL considers the index valid for queries."
- key: table:db_perf_index_stat;field:idxScan
summary: "Idx scan reported for this index stat record."
- key: table:db_perf_index_stat;field:idxTupRead
summary: "Idx tup read reported for this index stat record."
- key: table:db_perf_index_stat;field:idxTupFetch
summary: "Idx tup fetch reported for this index stat record."
- key: table:db_perf_index_stat;field:sizeBytes
summary: "Size bytes; size measured in bytes."
- key: table:db_perf_index_stat;field:writeCost
summary: "Write cost reported for this index stat record."
- key: table:db_perf_index_stat;field:scansSinceReset
summary: "Scans since reset reported for this index stat record."
- key: table:db_perf_index_stat;field:lastSeenAt
summary: "Most recent observation of this statement, table, index or finding."
- key: table:db_perf_query_daily;slot:help
summary: "Daily totals from retained statement snapshot details."
content: |
Roll up and purge writes completed days, normally at 03:15 in the configured time zone. Retained for rollupRetentionDays (365). Wait for the next daily run after query snapshots exist.
- key: table:db_perf_query_daily;slot:emptyState
summary: "Daily totals from retained statement snapshot details."
content: |
Roll up and purge writes completed days, normally at 03:15 in the configured time zone. Retained for rollupRetentionDays (365). Wait for the next daily run after query snapshots exist.
- key: table:db_perf_query_daily;field:statementId
summary: "Reference to the related statement record."
- key: table:db_perf_query_daily;field:day
summary: "Calendar day in the configured qbit time zone for this rollup."
- key: table:db_perf_query_daily;field:calls
summary: "Calls reported for this query daily record."
- key: table:db_perf_query_daily;field:totalMs
summary: "Total ms in milliseconds for this measurement."
- key: table:db_perf_query_daily;field:meanMs
summary: "Mean ms in milliseconds for this measurement."
- key: table:db_perf_query_daily;field:maxMs
summary: "Max ms in milliseconds for this measurement."
- key: table:db_perf_query_daily;field:rowsCount
summary: "Rows processed during the measured execution window."
- key: table:db_perf_query_daily;field:hitRatio
summary: "Shared block hits divided by hits plus reads, from 0 to 1; null when no blocks were accessed."
- key: table:db_perf_query_daily;field:shareOfDbTime
summary: "Percentage of all database statement time in this interval, from 0 to 100."
- key: table:db_perf_query_daily;field:snapshots
summary: "Snapshots reported for this query daily record."
- key: table:db_perf_query_snapshot;slot:help
summary: "Interval deltas for the highest-cost statements in each collection."
content: |
The collector writes the top configured number of statement deltas every collection. New or reset counters establish baselines instead. Retained for detailRetentionDays (30). Collect twice when empty.
- key: table:db_perf_query_snapshot;slot:emptyState
summary: "Interval deltas for the highest-cost statements in each collection."
content: |
The collector writes the top configured number of statement deltas every collection. New or reset counters establish baselines instead. Retained for detailRetentionDays (30). Collect twice when empty.
- key: table:db_perf_query_snapshot;field:snapshotId
summary: "Reference to the related snapshot record."
- key: table:db_perf_query_snapshot;field:statementId
summary: "Reference to the related statement record."
- key: table:db_perf_query_snapshot;field:capturedAt
summary: "Time this database sample was captured."
- key: table:db_perf_query_snapshot;field:calls
summary: "Calls reported for this query snapshot record."
- key: table:db_perf_query_snapshot;field:totalMs
summary: "Total ms in milliseconds for this measurement."
- key: table:db_perf_query_snapshot;field:meanMs
summary: "Mean ms in milliseconds for this measurement."
- key: table:db_perf_query_snapshot;field:maxMs
summary: "Max ms in milliseconds for this measurement."
- key: table:db_perf_query_snapshot;field:rowsCount
summary: "Rows processed during the measured execution window."
- key: table:db_perf_query_snapshot;field:sharedBlksHit
summary: "Shared blks hit reported for this query snapshot record."
- key: table:db_perf_query_snapshot;field:sharedBlksRead
summary: "Shared blks read reported for this query snapshot record."
- key: table:db_perf_query_snapshot;field:hitRatio
summary: "Shared block hits divided by hits plus reads, from 0 to 1; null when no blocks were accessed."
- key: table:db_perf_query_snapshot;field:tempBlksWritten
summary: "Temp blks written reported for this query snapshot record."
- key: table:db_perf_query_snapshot;field:shareOfDbTime
summary: "Percentage of all database statement time in this interval, from 0 to 100."
- key: table:db_perf_repeat_pattern;slot:help
summary: "Repeated statement shapes within a QQQ unit of work, indicating possible N+1 queries."
content: |
The attribution listener flushes patterns that reach the configured repetition threshold. Retained for detailRetentionDays (30). Empty is normal when no repeated-statement pattern meets the threshold.
- key: table:db_perf_repeat_pattern;slot:emptyState
summary: "Repeated statement shapes within a QQQ unit of work, indicating possible N+1 queries."
content: |
The attribution listener flushes patterns that reach the configured repetition threshold. Retained for detailRetentionDays (30). Empty is normal when no repeated-statement pattern meets the threshold.
- key: table:db_perf_repeat_pattern;field:windowStart
summary: "Beginning of the attribution aggregation window."
- key: table:db_perf_repeat_pattern;field:windowEnd
summary: "End of the attribution aggregation window."
- key: table:db_perf_repeat_pattern;field:podName
summary: "Application pod that observed and flushed this attribution window."
- key: table:db_perf_repeat_pattern;field:unitKind
summary: "Kind of QQQ unit of work, such as route, process or scheduled job."
- key: table:db_perf_repeat_pattern;field:unitName
summary: "Name of the QQQ unit responsible for database work."
- key: table:db_perf_repeat_pattern;field:processName
summary: "QQQ process responsible for the attributed work, when known."
- key: table:db_perf_repeat_pattern;field:tableName
summary: "Table name reported for this repeat pattern record."
- key: table:db_perf_repeat_pattern;field:fingerprint
summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded."
- key: table:db_perf_repeat_pattern;field:sqlText
summary: "Normalized SQL shape without literal values. Parameter values are never stored."
- key: table:db_perf_repeat_pattern;field:unitsAffected
summary: "Number of sampled units that exceeded the repeated-statement threshold."
- key: table:db_perf_repeat_pattern;field:maxRepeats
summary: "Largest repetition count for one fingerprint in a single sampled unit of work."
- key: table:db_perf_repeat_pattern;field:totalRepeats
summary: "Total repeats reported for this repeat pattern record."
- key: table:db_perf_repeat_pattern;field:sampleRate
summary: "Share of units of work that were measured in this window (1 means all)."
- key: table:db_perf_rule;slot:help
summary: "Editable thresholds and severity for each suggestion rule."
content: |
Seeded insert-only by the Liquibase changelog; existing edits are preserved. Changes affect the next analysis. Rules are retained indefinitely. Apply the changelog when the list is empty.
Threshold guide (stored labels are authoritative):
- DUPLICATE_INDEX: no numeric thresholds; compares index definitions for identical or prefix-redundant coverage.
- SLOW_QUERY: threshold 1 = Mean execution time (ms) (default 500); threshold 2 = Database time (%) (default 5).
- MISSING_INDEX: threshold 1 = Sequential scan share (%) (default 50); threshold 2 = Tuples read per "day" (default 1000000).
- UNUSED_INDEX: threshold 1 = Days since statistics reset (default 14); threshold 2 = Index size (MB) (default 1).
- N_PLUS_ONE: threshold 1 = Repeats per unit (default 10); threshold 2 = Affected units (default 5).
- HOT_JOB: threshold 1 = Database time (%) (default 10); threshold 2 is unused.
- OVER_AUDITED: threshold 1 = Audit inserts per "day" (default 50000); threshold 2 is unused.
- BLOAT: threshold 1 = Dead tuple ratio (%) (default 20); threshold 2 = Dead tuples (default 10000).
- VACUUM_STALE: threshold 1 = Days since vacuum (default 7); threshold 2 = Transaction freeze age (default 150000000).
- POOL_SIZING: threshold 1 = Connections used (%) (default 80); threshold 2 = Idle connections (%) (default 50).
- IDLE_IN_TRANSACTION: threshold 1 = Idle seconds (default 60); threshold 2 is unused.
- CACHE_HIT_LOW: threshold 1 = Cache hit ratio (%) (default 95); threshold 2 is unused.
- TEMP_SPILL: threshold 1 = Temporary data per "day" (MB) (default 100); threshold 2 is unused.
- DEADLOCKS: threshold 1 = Deadlocks per "day" (default 1); threshold 2 is unused.
- key: table:db_perf_rule;slot:emptyState
summary: "Editable thresholds and severity for each suggestion rule."
content: |
Seeded insert-only by the Liquibase changelog; existing edits are preserved. Changes affect the next analysis. Rules are retained indefinitely. Apply the changelog when the list is empty.
- key: table:db_perf_rule;field:code
summary: "Code reported for this rule record."
- key: table:db_perf_rule;field:label
summary: "Label reported for this rule record."
- key: table:db_perf_rule;field:description
summary: "Description reported for this rule record."
- key: table:db_perf_rule;field:enabled
summary: "Whether collection or this suggestion rule participates in the next scheduled run."
- key: table:db_perf_rule;field:severity
summary: "Configured importance of a finding or rule; contributes to the ranking score."
- key: table:db_perf_rule;field:threshold1
summary: "Numeric rule threshold; its label defines the unit and meaning. See the rule threshold guide."
content: |
Rule thresholds use the units named on each rule row.
- SLOW_QUERY: threshold 1 = Mean execution time (ms) (default 500); threshold 2 = Database time (%) (default 5).
- MISSING_INDEX: threshold 1 = Sequential scan share (%) (default 50); threshold 2 = Tuples read per "day" (default 1000000).
- UNUSED_INDEX: threshold 1 = Days since statistics reset (default 14); threshold 2 = Index size (MB) (default 1).
- N_PLUS_ONE: threshold 1 = Repeats per unit (default 10); threshold 2 = Affected units (default 5).
- HOT_JOB: threshold 1 = Database time (%) (default 10); threshold 2 is unused.
- OVER_AUDITED: threshold 1 = Audit inserts per "day" (default 50000); threshold 2 is unused.
- BLOAT: threshold 1 = Dead tuple ratio (%) (default 20); threshold 2 = Dead tuples (default 10000).
- VACUUM_STALE: threshold 1 = Days since vacuum (default 7); threshold 2 = Transaction freeze age (default 150000000).
- POOL_SIZING: threshold 1 = Connections used (%) (default 80); threshold 2 = Idle connections (%) (default 50).
- IDLE_IN_TRANSACTION: threshold 1 = Idle seconds (default 60); threshold 2 is unused.
- CACHE_HIT_LOW: threshold 1 = Cache hit ratio (%) (default 95); threshold 2 is unused.
- TEMP_SPILL: threshold 1 = Temporary data per "day" (MB) (default 100); threshold 2 is unused.
- DEADLOCKS: threshold 1 = Deadlocks per "day" (default 1); threshold 2 is unused.
- key: table:db_perf_rule;field:threshold2
summary: "Numeric rule threshold; its label defines the unit and meaning. See the rule threshold guide."
content: |
Rule thresholds use the units named on each rule row.
- SLOW_QUERY: threshold 1 = Mean execution time (ms) (default 500); threshold 2 = Database time (%) (default 5).
- MISSING_INDEX: threshold 1 = Sequential scan share (%) (default 50); threshold 2 = Tuples read per "day" (default 1000000).
- UNUSED_INDEX: threshold 1 = Days since statistics reset (default 14); threshold 2 = Index size (MB) (default 1).
- N_PLUS_ONE: threshold 1 = Repeats per unit (default 10); threshold 2 = Affected units (default 5).
- HOT_JOB: threshold 1 = Database time (%) (default 10); threshold 2 is unused.
- OVER_AUDITED: threshold 1 = Audit inserts per "day" (default 50000); threshold 2 is unused.
- BLOAT: threshold 1 = Dead tuple ratio (%) (default 20); threshold 2 = Dead tuples (default 10000).
- VACUUM_STALE: threshold 1 = Days since vacuum (default 7); threshold 2 = Transaction freeze age (default 150000000).
- POOL_SIZING: threshold 1 = Connections used (%) (default 80); threshold 2 = Idle connections (%) (default 50).
- IDLE_IN_TRANSACTION: threshold 1 = Idle seconds (default 60); threshold 2 is unused.
- CACHE_HIT_LOW: threshold 1 = Cache hit ratio (%) (default 95); threshold 2 is unused.
- TEMP_SPILL: threshold 1 = Temporary data per "day" (MB) (default 100); threshold 2 is unused.
- DEADLOCKS: threshold 1 = Deadlocks per "day" (default 1); threshold 2 is unused.
- key: table:db_perf_rule;field:threshold1Label
summary: "Threshold1 label reported for this rule record."
- key: table:db_perf_rule;field:threshold2Label
summary: "Threshold2 label reported for this rule record."
- key: table:db_perf_settings;slot:help
summary: "Per-backend collection, retention, attribution and query-plan settings."
content: |
Created on the first collection or explain request. Administrators edit these values; dashboard reads do not create settings. Settings are retained until removed by the host. Run Collect now when empty.
- key: table:db_perf_settings;slot:emptyState
summary: "Per-backend collection, retention, attribution and query-plan settings."
content: |
Created on the first collection or explain request. Administrators edit these values; dashboard reads do not create settings. Settings are retained until removed by the host. Run Collect now when empty.
- key: table:db_perf_settings;field:backendName
summary: "The configured QQQ backend whose database is monitored."
- key: table:db_perf_settings;field:enabled
summary: "Whether collection or this suggestion rule participates in the next scheduled run."
- key: table:db_perf_settings;field:provider
summary: "Detected statistics provider; unsupported vendors report a setup message."
- key: table:db_perf_settings;field:snapshotIntervalMinutes
summary: "Minimum minutes between scheduled snapshots; default 5. Collect now forces collection."
- key: table:db_perf_settings;field:tableSnapshotMinutes
summary: "Minutes between table and index collection passes; default 60."
- key: table:db_perf_settings;field:detailRetentionDays
summary: "Age in days before detailed samples and plan audits are purged; default 30."
- key: table:db_perf_settings;field:rollupRetentionDays
summary: "Retention in days for daily rows and stale inventory or closed findings; default 365."
- key: table:db_perf_settings;field:topStatementsPerSnapshot
summary: "Maximum statement details saved per pass, ranked by database time; default 200."
- key: table:db_perf_settings;field:activityLongRunningSeconds
summary: "Active query age in seconds that makes a session notable; default 30."
- key: table:db_perf_settings;field:idleInTransactionSeconds
summary: "Idle transaction age in seconds that makes a session notable; default 60."
- key: table:db_perf_settings;field:attributionEnabled
summary: "Enable QQQ statement attribution; the host listener must also be enabled."
- key: table:db_perf_settings;field:attributionSampleRate
summary: "Fraction of QQQ units sampled, from 0 to 1; default 1 captures every unit."
- key: table:db_perf_settings;field:attributionFlushSeconds
summary: "Seconds between attribution batch flushes; default 60. A crash can lose one window."
- key: table:db_perf_settings;field:attributionMaxKeys
summary: "Maximum in-memory attribution groups per pod; overflow goes to an other group. Default 10000."
- key: table:db_perf_settings;field:nPlusOneThreshold
summary: "Repeated occurrences of a statement shape in one unit before recording a pattern; default 10."
- key: table:db_perf_settings;field:explainTimeoutMs
summary: "Maximum query-plan execution time in milliseconds; default 10000 for PLAN and ANALYZE."
- key: table:db_perf_settings;field:alertsEnabled
summary: "Enable database performance alert emission when the alert integration is installed."
- key: table:db_perf_settings;field:spikeFactor
summary: "Multiplier above the seven-day interval median needed for a query-spike alert; default 3."
- key: table:db_perf_settings;field:spikeMinTotalMs
summary: "Minimum interval query time in milliseconds before a spike can alert; default 5000."
- key: table:db_perf_settings;field:connectionAlertPercent
summary: "Percentage of max connections at which saturation can alert; default 80."
- key: table:db_perf_settings;field:blockedAlertSeconds
summary: "Blocked session duration in seconds required for a blocking-chain alert; default 30."
- key: table:db_perf_settings;field:lastSnapshotAt
summary: "Time of the latest collection tracked by the backend settings."
- key: table:db_perf_settings;field:lastTableSnapshotAt
summary: "Time of the latest table and index collection pass."
- key: table:db_perf_settings;field:statsAccess
summary: "Statistics capability: FULL, reduced visibility, statement statistics off, or unsupported provider."
- key: table:db_perf_settings;field:statsAccessMessage
summary: "The collector explains missing capabilities and the next grant or setup step."
- key: table:db_perf_snapshot;slot:help
summary: "One collection pass and its database-wide interval totals."
content: |
The collector writes this table every five minutes by default, including partial and failed passes. The first pass establishes a baseline. Retained for detailRetentionDays (30 by default). Run Collect now when empty.
- key: table:db_perf_snapshot;slot:emptyState
summary: "One collection pass and its database-wide interval totals."
content: |
The collector writes this table every five minutes by default, including partial and failed passes. The first pass establishes a baseline. Retained for detailRetentionDays (30 by default). Run Collect now when empty.
- key: table:db_perf_snapshot;field:backendName
summary: "The configured QQQ backend whose database is monitored."
- key: table:db_perf_snapshot;field:capturedAt
summary: "Time this database sample was captured."
- key: table:db_perf_snapshot;field:intervalSeconds
summary: "Interval seconds reported for this snapshot record."
- key: table:db_perf_snapshot;field:status
summary: "Current lifecycle state. Review the status-specific guidance and related messages."
- key: table:db_perf_snapshot;field:durationMs
summary: "Duration ms in milliseconds for this measurement."
- key: table:db_perf_snapshot;field:errorMessage
summary: "Sanitized failure description. Database SQL errors and parameter values are excluded."
- key: table:db_perf_snapshot;field:statsResetDetected
summary: "Counters restarted or decreased; that pass establishes a new baseline."
- key: table:db_perf_snapshot;field:includesTables
summary: "Includes tables reported for this snapshot record."
- key: table:db_perf_snapshot;field:xactCommit
summary: "Xact commit reported for this snapshot record."
- key: table:db_perf_snapshot;field:xactRollback
summary: "Xact rollback reported for this snapshot record."
- key: table:db_perf_snapshot;field:blksHit
summary: "Blks hit reported for this snapshot record."
- key: table:db_perf_snapshot;field:blksRead
summary: "Blks read reported for this snapshot record."
- key: table:db_perf_snapshot;field:cacheHitRatio
summary: "Database cache hits divided by hits plus reads, from 0 to 1."
- key: table:db_perf_snapshot;field:tempFiles
summary: "Temp files reported for this snapshot record."
- key: table:db_perf_snapshot;field:tempBytes
summary: "Temp bytes; size measured in bytes."
- key: table:db_perf_snapshot;field:deadlocks
summary: "Deadlocks reported for this snapshot record."
- key: table:db_perf_snapshot;field:conflicts
summary: "Conflicts reported for this snapshot record."
- key: table:db_perf_snapshot;field:connectionsTotal
summary: "Connections total reported for this snapshot record."
- key: table:db_perf_snapshot;field:connectionsActive
summary: "Connections active reported for this snapshot record."
- key: table:db_perf_snapshot;field:connectionsIdle
summary: "Connections idle reported for this snapshot record."
- key: table:db_perf_snapshot;field:connectionsIdleInTx
summary: "Connections idle in tx reported for this snapshot record."
- key: table:db_perf_snapshot;field:maxConnections
summary: "Max connections reported for this snapshot record."
- key: table:db_perf_snapshot;field:databaseSizeBytes
summary: "Database size bytes; size measured in bytes."
- key: table:db_perf_snapshot;field:statementCount
summary: "Statement count reported for this snapshot record."
- key: table:db_perf_snapshot;field:statementCalls
summary: "Statement calls reported for this snapshot record."
- key: table:db_perf_snapshot;field:statementTotalMs
summary: "Statement total ms in milliseconds for this measurement."
- key: table:db_perf_snapshot;field:storedStatementCount
summary: "Stored statement count reported for this snapshot record."
- key: table:db_perf_snapshot;field:rawXactCommit
summary: "Previous cumulative xact commit counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawXactRollback
summary: "Previous cumulative xact rollback counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawBlksHit
summary: "Previous cumulative blks hit counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawBlksRead
summary: "Previous cumulative blks read counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawTempFiles
summary: "Previous cumulative temp files counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawTempBytes
summary: "Previous cumulative temp bytes counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawDeadlocks
summary: "Previous cumulative deadlocks counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawConflicts
summary: "Previous cumulative conflicts counter used for reset-safe deltas."
- key: table:db_perf_snapshot;field:rawStatsReset
summary: "Previous database counter reset timestamp used for delta calculations."
- key: table:db_perf_snapshot;field:rawStatementStatsReset
summary: "Previous provider reset marker used to detect counter resets, not raw SQL."
- key: table:db_perf_statement;slot:help
summary: "Normalized query shapes with the latest counters and retained 24-hour totals."
content: |
The collector upserts statements observed in pg_stat_statements each collection. Two snapshots establish deltas. Retained while observed; stale records are purged after rollupRetentionDays (365 by default). Enable pg_stat_statements and collect twice when empty.
- key: table:db_perf_statement;slot:emptyState
summary: "Normalized query shapes with the latest counters and retained 24-hour totals."
content: |
The collector upserts statements observed in pg_stat_statements each collection. Two snapshots establish deltas. Retained while observed; stale records are purged after rollupRetentionDays (365 by default). Enable pg_stat_statements and collect twice when empty.
- key: table:db_perf_statement;field:backendName
summary: "The configured QQQ backend whose database is monitored."
- key: table:db_perf_statement;field:queryId
summary: "PostgreSQL query identifier; fingerprint provides cross-source normalized matching."
- key: table:db_perf_statement;field:dbUser
summary: "Database role associated with this statement, session or connection group."
- key: table:db_perf_statement;field:toplevel
summary: "Toplevel reported for this statement record."
- key: table:db_perf_statement;field:fingerprint
summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded."
- key: table:db_perf_statement;field:queryText
summary: "Normalized query text with literals replaced by placeholders; never raw live text."
- key: table:db_perf_statement;field:operation
summary: "Normalized SQL operation, such as SELECT, INSERT, UPDATE or DELETE."
- key: table:db_perf_statement;field:mainTable
summary: "Best-effort main table extracted from normalized SQL."
- key: table:db_perf_statement;field:firstSeenAt
summary: "First recorded observation of this statement or finding."
- key: table:db_perf_statement;field:lastSeenAt
summary: "Most recent observation of this statement, table, index or finding."
- key: table:db_perf_statement;field:lastCalls
summary: "Last calls reported for this statement record."
- key: table:db_perf_statement;field:lastTotalExecMs
summary: "Last total exec ms in milliseconds for this measurement."
- key: table:db_perf_statement;field:lastRows
summary: "Last rows reported for this statement record."
- key: table:db_perf_statement;field:lastSharedBlksHit
summary: "Last shared blks hit reported for this statement record."
- key: table:db_perf_statement;field:lastSharedBlksRead
summary: "Last shared blks read reported for this statement record."
- key: table:db_perf_statement;field:lastTempBlksWritten
summary: "Last temp blks written reported for this statement record."
- key: table:db_perf_statement;field:lastMaxExecMs
summary: "Latest cumulative maximum execution time in milliseconds; not an interval maximum."
- key: table:db_perf_statement;field:lastStatsSince
summary: "Per-statement counter start time when the PostgreSQL version supplies it."
- key: table:db_perf_statement;field:calls24h
summary: "Calls in retained statement details from the last 24 hours."
- key: table:db_perf_statement;field:totalMs24h
summary: "Execution time in milliseconds in retained statement details from the last 24 hours."
- key: table:db_perf_statement;field:meanMs24h
summary: "Total retained execution milliseconds divided by retained calls over 24 hours."
- key: table:db_perf_statement;field:shareOfDbTime24h
summary: "Percentage of database time over 24 hours; numerator uses retained statement details."
- key: table:db_perf_table_daily;slot:help
summary: "Daily table scan deltas and ending size used to track growth."
content: |
The daily rollup writes these rows from retained table samples, normally at 03:15. Retained for rollupRetentionDays (365). Wait for the daily rollup after table snapshots exist.
- key: table:db_perf_table_daily;slot:emptyState
summary: "Daily table scan deltas and ending size used to track growth."
content: |
The daily rollup writes these rows from retained table samples, normally at 03:15. Retained for rollupRetentionDays (365). Wait for the daily rollup after table snapshots exist.
- key: table:db_perf_table_daily;field:tableStatId
summary: "Reference to the related table stat record."
- key: table:db_perf_table_daily;field:day
summary: "Calendar day in the configured qbit time zone for this rollup."
- key: table:db_perf_table_daily;field:seqScan
summary: "Seq scan reported for this table daily record."
- key: table:db_perf_table_daily;field:idxScan
summary: "Idx scan reported for this table daily record."
- key: table:db_perf_table_daily;field:seqTupRead
summary: "Seq tup read reported for this table daily record."
- key: table:db_perf_table_daily;field:nTupIns
summary: "PostgreSQL n tup ins tuple statistic for the sampled table."
- key: table:db_perf_table_daily;field:nTupUpd
summary: "PostgreSQL n tup upd tuple statistic for the sampled table."
- key: table:db_perf_table_daily;field:nTupDel
summary: "PostgreSQL n tup del tuple statistic for the sampled table."
- key: table:db_perf_table_daily;field:nLiveTup
summary: "PostgreSQL n live tup tuple statistic for the sampled table."
- key: table:db_perf_table_daily;field:nDeadTup
summary: "PostgreSQL n dead tup tuple statistic for the sampled table."
- key: table:db_perf_table_daily;field:totalBytes
summary: "Total bytes; size measured in bytes."
- key: table:db_perf_table_snapshot;slot:help
summary: "Historical samples of table scans, tuples, size and maintenance."
content: |
The collector writes table snapshots every tableSnapshotMinutes (60). Retained for detailRetentionDays (30). Run Collect now or wait for the next hourly table pass.
- key: table:db_perf_table_snapshot;slot:emptyState
summary: "Historical samples of table scans, tuples, size and maintenance."
content: |
The collector writes table snapshots every tableSnapshotMinutes (60). Retained for detailRetentionDays (30). Run Collect now or wait for the next hourly table pass.
- key: table:db_perf_table_snapshot;field:snapshotId
summary: "Reference to the related snapshot record."
- key: table:db_perf_table_snapshot;field:tableStatId
summary: "Reference to the related table stat record."
- key: table:db_perf_table_snapshot;field:capturedAt
summary: "Time this database sample was captured."
- key: table:db_perf_table_snapshot;field:seqScan
summary: "Seq scan reported for this table snapshot record."
- key: table:db_perf_table_snapshot;field:seqTupRead
summary: "Seq tup read reported for this table snapshot record."
- key: table:db_perf_table_snapshot;field:idxScan
summary: "Idx scan reported for this table snapshot record."
- key: table:db_perf_table_snapshot;field:idxTupFetch
summary: "Idx tup fetch reported for this table snapshot record."
- key: table:db_perf_table_snapshot;field:nTupIns
summary: "PostgreSQL n tup ins tuple statistic for the sampled table."
- key: table:db_perf_table_snapshot;field:nTupUpd
summary: "PostgreSQL n tup upd tuple statistic for the sampled table."
- key: table:db_perf_table_snapshot;field:nTupDel
summary: "PostgreSQL n tup del tuple statistic for the sampled table."
- key: table:db_perf_table_snapshot;field:nTupHotUpd
summary: "PostgreSQL n tup hot upd tuple statistic for the sampled table."
- key: table:db_perf_table_snapshot;field:nLiveTup
summary: "PostgreSQL n live tup tuple statistic for the sampled table."
- key: table:db_perf_table_snapshot;field:nDeadTup
summary: "PostgreSQL n dead tup tuple statistic for the sampled table."
- key: table:db_perf_table_snapshot;field:totalBytes
summary: "Total bytes; size measured in bytes."
- key: table:db_perf_table_snapshot;field:tableBytes
summary: "Table bytes; size measured in bytes."
- key: table:db_perf_table_snapshot;field:indexBytes
summary: "Index bytes; size measured in bytes."
- key: table:db_perf_table_snapshot;field:autovacuumCount
summary: "Autovacuum count reported for this table snapshot record."
- key: table:db_perf_table_snapshot;field:autoanalyzeCount
summary: "Autoanalyze count reported for this table snapshot record."
- key: table:db_perf_table_snapshot;field:freezeAge
summary: "Transaction-ID age used to assess vacuum and wraparound risk."
- key: table:db_perf_table_stat;slot:help
summary: "Current table size, scans, tuple counts and maintenance statistics."
content: |
The collector upserts table data every tableSnapshotMinutes (60). Stale records are purged after rollupRetentionDays (365). Run Collect now to force table collection when empty.
- key: table:db_perf_table_stat;slot:emptyState
summary: "Current table size, scans, tuple counts and maintenance statistics."
content: |
The collector upserts table data every tableSnapshotMinutes (60). Stale records are purged after rollupRetentionDays (365). Run Collect now to force table collection when empty.
- key: table:db_perf_table_stat;field:backendName
summary: "The configured QQQ backend whose database is monitored."
- key: table:db_perf_table_stat;field:schemaName
summary: "Schema name reported for this table stat record."
- key: table:db_perf_table_stat;field:tableName
summary: "Table name reported for this table stat record."
- key: table:db_perf_table_stat;field:relid
summary: "Relid reported for this table stat record."
- key: table:db_perf_table_stat;field:seqScan
summary: "Seq scan reported for this table stat record."
- key: table:db_perf_table_stat;field:seqTupRead
summary: "Seq tup read reported for this table stat record."
- key: table:db_perf_table_stat;field:idxScan
summary: "Idx scan reported for this table stat record."
- key: table:db_perf_table_stat;field:idxTupFetch
summary: "Idx tup fetch reported for this table stat record."
- key: table:db_perf_table_stat;field:nTupIns
summary: "PostgreSQL n tup ins tuple statistic for the sampled table."
- key: table:db_perf_table_stat;field:nTupUpd
summary: "PostgreSQL n tup upd tuple statistic for the sampled table."
- key: table:db_perf_table_stat;field:nTupDel
summary: "PostgreSQL n tup del tuple statistic for the sampled table."
- key: table:db_perf_table_stat;field:nTupHotUpd
summary: "PostgreSQL n tup hot upd tuple statistic for the sampled table."
- key: table:db_perf_table_stat;field:nLiveTup
summary: "PostgreSQL n live tup tuple statistic for the sampled table."
- key: table:db_perf_table_stat;field:nDeadTup
summary: "PostgreSQL n dead tup tuple statistic for the sampled table."
- key: table:db_perf_table_stat;field:deadRatio
summary: "Dead tuples divided by live plus dead tuples, from 0 to 1; an estimate used for maintenance review."
- key: table:db_perf_table_stat;field:totalBytes
summary: "Total bytes; size measured in bytes."
- key: table:db_perf_table_stat;field:tableBytes
summary: "Table bytes; size measured in bytes."
- key: table:db_perf_table_stat;field:indexBytes
summary: "Index bytes; size measured in bytes."
- key: table:db_perf_table_stat;field:toastBytes
summary: "Toast bytes; size measured in bytes."
- key: table:db_perf_table_stat;field:growthBytes7d
summary: "Change in table size in bytes over seven days, once enough daily history exists."
- key: table:db_perf_table_stat;field:lastVacuum
summary: "Last manual vacuum reported by PostgreSQL."
- key: table:db_perf_table_stat;field:lastAutovacuum
summary: "Last automatic vacuum reported by PostgreSQL."
- key: table:db_perf_table_stat;field:lastAnalyze
summary: "Last manual statistics analyze reported by PostgreSQL."
- key: table:db_perf_table_stat;field:lastAutoanalyze
summary: "Last automatic statistics analyze reported by PostgreSQL."
- key: table:db_perf_table_stat;field:vacuumCount
summary: "Vacuum count reported for this table stat record."
- key: table:db_perf_table_stat;field:autovacuumCount
summary: "Autovacuum count reported for this table stat record."
- key: table:db_perf_table_stat;field:analyzeCount
summary: "Analyze count reported for this table stat record."
- key: table:db_perf_table_stat;field:autoanalyzeCount
summary: "Autoanalyze count reported for this table stat record."
- key: table:db_perf_table_stat;field:freezeAge
summary: "Transaction-ID age used to assess vacuum and wraparound risk."
- key: table:db_perf_table_stat;field:seqScanShare
summary: "Fraction of table scans that are sequential, from 0 to 1; high values merit query and index review."
- key: table:db_perf_table_stat;field:lastSeenAt
summary: "Most recent observation of this statement, table, index or finding."
- key: process:dbPerfCollectSnapshot;slot:help
summary: "Collect database snapshots on the host schedule."
content: |
Runs at the configured snapshot cadence, normally every five minutes. Reads statistics and writes interval deltas and notable activity; table/index passes normally run hourly. Requires manage settings.
- key: process:dbPerfCollectNow;slot:help
summary: "Collect an immediate snapshot and force table and index statistics."
content: |
Run from the Snapshots table. Requires manage settings. The Result screen reports success, partial access or collection failure. The first pass establishes counter baselines.
- key: process:dbPerfEnableStatementStatistics;slot:help
summary: "Turn on statement statistics where the database allows it at runtime."
content: |
Run from Settings. Requires manage settings. On H2 it runs SET QUERY_STATISTICS TRUE, which needs an H2 admin user and lasts until the database closes; add ;QUERY_STATISTICS=TRUE to the JDBC URL to keep it on. PostgreSQL cannot do this at runtime (pg_stat_statements needs shared_preload_libraries and CREATE EXTENSION), so the result explains the manual step. The Result screen shows the new statistics access.
- key: process:dbPerfRollupAndPurge;slot:help
summary: "Roll up completed days and remove expired details."
content: |
Runs daily at 03:15 by default. Uses the configured time zone, detail retention and rollup retention. Reruns replace daily aggregates instead of duplicating them.
- key: process:dbPerfQueryPlan;slot:help
summary: "Inspect the plan of one statement, with optional read-only execution."
content: |
Choose PLAN (default) or ANALYZE. PLAN uses a generic plan for parameterized SQL on PostgreSQL 16+ and H2. H2 plans are text (index choice and scan counts) without a cost tree; H2 refuses SQL naming side-effecting functions, DDL or aliases, and allows ANALYZE only for non-admin users. ANALYZE requires explain analyze permission and exactly one SELECT with no modifying CTEs, INTO or row locks. Enter parameter values only for ANALYZE, one per line in placeholder order. Values are sent transiently; they and the ANALYZE plan are never stored. Every request is audited. Results include copyable plan text, cost, a SQL Workbench link and a literal-free assistant prompt. No model is called by this qbit.
- key: widget:dbPerfSetup;slot:label
summary: "Check backend, statistics access, collection, attribution and scheduler setup."
- key: widget:dbPerfDatabaseTime;slot:label
summary: "Database execution milliseconds and call counts per snapshot over the last 24 hours."
- key: widget:dbPerfTopQueries;slot:label
summary: "Most expensive retained query shapes over 24 hours; open a query to inspect its history."
- key: widget:dbPerfFindings;slot:label
summary: "Open and acknowledged findings ranked by impact."
- key: widget:dbPerfLiveActivity;slot:label
summary: "Current notable sessions with normalized text, blocking and transaction ages."
- key: widget:dbPerfConnections;slot:label
summary: "Connection capacity followed by a live application and user breakdown."
- key: widget:dbPerfConnectionStatistics;slot:label
summary: "Latest snapshot capacity and cache hit percentage, with 24-hour temp and deadlock totals."
- key: widget:dbPerfConnectionsByApplication;slot:label
summary: "Current connections grouped by application, database user and state."
- key: widget:dbPerfAttribution;slot:label
summary: "QQQ units ranked by database time, followed by repeated-statement patterns."
- key: widget:dbPerfTables;slot:label
summary: "Sequential-scan-heavy tables and tables with the fastest seven-day size growth."
- key: widget:db_perf_snapshot_db_perf_query_snapshot;slot:label
summary: "Related interval deltas for the highest-cost statements in each collection."
- key: widget:db_perf_statement_db_perf_query_snapshot;slot:label
summary: "Related interval deltas for the highest-cost statements in each collection."
- key: widget:db_perf_statement_db_perf_query_daily;slot:label
summary: "Related daily totals from retained statement snapshot details."
- key: widget:db_perf_snapshot_db_perf_table_snapshot;slot:label
summary: "Related historical samples of table scans, tuples, size and maintenance."
- key: widget:db_perf_table_stat_db_perf_table_snapshot;slot:label
summary: "Related historical samples of table scans, tuples, size and maintenance."
- key: widget:db_perf_table_stat_db_perf_table_daily;slot:label
summary: "Related daily table scan deltas and ending size used to track growth."
- key: widget:db_perf_table_stat_db_perf_index_stat;slot:label
summary: "Related current index definitions, usage, size and uniqueness."
- key: widget:db_perf_snapshot_db_perf_activity_sample;slot:label
summary: "Related notable session samples with normalized text, never raw query text."
- key: widget:db_perf_snapshot_db_perf_connection_sample;slot:label
summary: "Related connection counts grouped by application, user and state for each snapshot."
- key: widget:db_perf_rule_db_perf_finding;slot:label
summary: "Related ranked, deduplicated suggestions with supporting evidence and proposed fixes."
- key: widget:db_perf_statement_db_perf_finding;slot:label
summary: "Related ranked, deduplicated suggestions with supporting evidence and proposed fixes."
- key: widget:db_perf_table_stat_db_perf_finding;slot:label
summary: "Related ranked, deduplicated suggestions with supporting evidence and proposed fixes."
- key: widget:db_perf_statement_db_perf_explain;slot:label
summary: "Related audited plan requests, including denied, failed and timed-out attempts."
- key: possibleValueSource:db_perf_ExplainMode;slot:help
summary: "Available values for explainmode."
- key: possibleValueSource:db_perf_FindingStatus;slot:help
summary: "Available values for findingstatus."
- key: possibleValueSource:db_perf_Operation;slot:help
summary: "Available values for operation."
- key: possibleValueSource:db_perf_Outcome;slot:help
summary: "Available values for outcome."
- key: possibleValueSource:db_perf_Provider;slot:help
summary: "Available values for provider."
- key: possibleValueSource:db_perf_Reason;slot:help
summary: "Available values for reason."
- key: possibleValueSource:db_perf_Severity;slot:help
summary: "Available values for severity."
- key: possibleValueSource:db_perf_SnapshotStatus;slot:help
summary: "Available values for snapshotstatus."
- key: possibleValueSource:db_perf_State;slot:help
summary: "Available values for state."
- key: possibleValueSource:db_perf_StatsAccess;slot:help
summary: "Available values for statsaccess."
- key: possibleValueSource:db_perf_UnitKind;slot:help
summary: "Available values for unitkind."
- key: possibleValueSource:db_perf_activity_sample;slot:help
summary: "Available values for activity sample."
- key: possibleValueSource:db_perf_attribution;slot:help
summary: "Available values for attribution."
- key: possibleValueSource:db_perf_attribution_daily;slot:help
summary: "Available values for attribution daily."
- key: possibleValueSource:db_perf_connection_sample;slot:help
summary: "Available values for connection sample."
- key: possibleValueSource:db_perf_explain;slot:help
summary: "Available values for explain."
- key: possibleValueSource:db_perf_finding;slot:help
summary: "Available values for finding."
- key: possibleValueSource:db_perf_index_stat;slot:help
summary: "Available values for index stat."
- key: possibleValueSource:db_perf_query_daily;slot:help
summary: "Available values for query daily."
- key: possibleValueSource:db_perf_query_snapshot;slot:help
summary: "Available values for query snapshot."
- key: possibleValueSource:db_perf_repeat_pattern;slot:help
summary: "Available values for repeat pattern."
- key: possibleValueSource:db_perf_rule;slot:help
summary: "Available values for rule."
- key: possibleValueSource:db_perf_settings;slot:help
summary: "Available values for settings."
- key: possibleValueSource:db_perf_snapshot;slot:help
summary: "Available values for snapshot."
- key: possibleValueSource:db_perf_statement;slot:help
summary: "Available values for statement."
- key: possibleValueSource:db_perf_table_daily;slot:help
summary: "Available values for table daily."
- key: possibleValueSource:db_perf_table_snapshot;slot:help
summary: "Available values for table snapshot."
- key: possibleValueSource:db_perf_table_stat;slot:help
summary: "Available values for table stat."
- key: process:db_perf_settings.bulkEdit;slot:help
summary: Review and apply changes to database performance settings records.
content: |
Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema.
- key: process:db_perf_settings.bulkEditWithFile;slot:help
summary: Review and apply changes to database performance settings records.
content: |
Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema.
- key: process:db_perf_rule.bulkInsert;slot:help
summary: Review and apply changes to database performance rule records.
content: |
Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema.
- key: process:db_perf_rule.bulkEdit;slot:help
summary: Review and apply changes to database performance rule records.
content: |
Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema.
- key: process:db_perf_rule.bulkEditWithFile;slot:help
summary: Review and apply changes to database performance rule records.
content: |
Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema.
- key: process:db_perf_rule.bulkDelete;slot:help
summary: Review and apply changes to database performance rule records.
content: |
Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema.
- key: possibleValueSource:db_perf_ExplainRequestMode;slot:help
summary: PLAN inspects the query; ANALYZE executes a guarded read-only SELECT with separate permission.
- key: process:dbPerfAnalyze;slot:help
summary: "Scheduled run of the suggestion rules that creates, updates and resolves findings."
content: |
Runs hourly by default. Each enabled rule in Suggestion Rules reads the stored statistics and produces findings with evidence. A rerun updates existing findings instead of adding new ones, resolves findings that are no longer detected, and never changes Ignored findings. It never changes the live database.
- key: process:dbPerfAnalyzeNow;slot:help
summary: "Run the suggestion rules now instead of waiting for the hourly run."
content: |
Needs the manage settings permission. The result shows new, updated and resolved findings per rule. Run {{label:dbPerfCollectNow}} first if the statistics are old.
- key: process:dbPerfFindingACKNOWLEDGED;slot:help
summary: "Mark the selected findings as acknowledged: seen and being worked on."
content: |
Acknowledged findings stay on the list and are still updated with new evidence. They resolve automatically when no longer detected.
- key: process:dbPerfFindingRESOLVED;slot:help
summary: "Mark the selected findings as resolved."
content: |
Use after shipping the fix. If the rule detects the problem again, the finding reopens.
- key: process:dbPerfFindingIGNORED;slot:help
summary: "Ignore the selected findings; a note saying why is required."
content: |
Ignored findings are never reopened or resolved automatically. Use Reopen to bring one back.
- key: process:dbPerfFindingOPEN;slot:help
summary: "Reopen the selected findings."
content: |
Returns acknowledged, resolved or ignored findings to Open so the rules manage them again.
@@ -0,0 +1,620 @@
components:
schemas:
BasicErrorResponseV1:
description: "The error body of every UI API operation (core and extensions)."
properties:
error:
description: "The error message (the same as message; kept for clients of\
\ UI API 1.0 to 1.5)."
type: "string"
code:
description: "A stable, machine-readable code. Well-known values: BAD_REQUEST\
\ (400), UNAUTHENTICATED (401),\nPERMISSION_DENIED (403), NOT_FOUND (404),\
\ CONFLICT (409), RATE_LIMITED (429), INTERNAL_ERROR (500),\nERROR (other).\
\ Extensions may add codes; treat an unknown code by the HTTP status.\
\ Since 1.6.0."
type: "string"
message:
description: "A message to show the user. Since 1.6.0."
type: "string"
fieldErrors:
description: "Errors about particular request fields, when known. Since\
\ 1.6.0."
items:
properties:
field:
description: "The field (or parameter) name."
type: "string"
message:
description: "The error about that field."
type: "string"
code:
description: "An optional machine-readable code, e.g. REQUIRED, TOO_LONG,\
\ PATTERN."
type: "string"
type: "object"
type: "array"
traceId:
description: "Identifies this request in the server logs (also the X-QQQ-Trace-Id\
\ response header); quote it when reporting a problem. Since 1.6.0."
type: "string"
type: "object"
DbPerformanceActivityV1:
properties:
blockingChains:
items:
$ref: "#/components/schemas/DbPerformanceBlockingChainV1"
type: "array"
liveTextRedacted:
type: "boolean"
sessions:
items:
$ref: "#/components/schemas/DbPerformanceSessionV1"
type: "array"
type: "object"
DbPerformanceBlockingChainV1:
properties:
blockedPids:
items:
nullable: true
type: "integer"
type: "array"
rootPid:
nullable: true
type: "integer"
type: "object"
DbPerformanceErrorV1:
properties:
error:
nullable: true
type: "string"
type: "object"
DbPerformanceExplainRequestV1:
description: "Supply exactly one of statementId or sql. Parameters are strings,\
\ accepted only for ANALYZE and never stored."
properties:
backend:
nullable: true
type: "string"
mode:
enum:
- "PLAN"
- "ANALYZE"
type: "string"
parameters:
items:
nullable: true
type: "string"
type: "array"
sql:
nullable: true
type: "string"
statementId:
nullable: true
type: "integer"
type: "object"
DbPerformanceExplainResultV1:
properties:
actualTotalMs:
nullable: true
type: "number"
assistantPrompt:
nullable: true
type: "string"
explainId:
nullable: true
type: "integer"
mode:
nullable: true
type: "string"
planJson:
nullable: true
type: "string"
planText:
nullable: true
type: "string"
sqlWorkbenchUrl:
nullable: true
type: "string"
totalCost:
nullable: true
type: "number"
warnings:
items:
nullable: true
type: "string"
type: "array"
type: "object"
DbPerformanceFindingV1:
properties:
id:
nullable: true
type: "integer"
score:
nullable: true
type: "number"
severity:
nullable: true
type: "string"
title:
nullable: true
type: "string"
type: "object"
DbPerformanceLookupV1:
properties:
fingerprint:
nullable: true
type: "string"
matches:
items:
$ref: "#/components/schemas/DbPerformanceStatementV1"
type: "array"
normalizedSql:
nullable: true
type: "string"
type: "object"
DbPerformanceOverviewV1:
properties:
findingsBySeverity:
additionalProperties:
nullable: true
type: "integer"
type: "object"
settings:
$ref: "#/components/schemas/DbPerformanceSettingsV1"
snapshot:
$ref: "#/components/schemas/DbPerformanceSnapshotV1"
topFindings:
items:
$ref: "#/components/schemas/DbPerformanceFindingV1"
type: "array"
topStatements:
items:
$ref: "#/components/schemas/DbPerformanceStatementV1"
type: "array"
type: "object"
DbPerformanceSessionV1:
properties:
application:
nullable: true
type: "string"
blockedBy:
items:
nullable: true
type: "integer"
type: "array"
blockingCount:
nullable: true
type: "integer"
fingerprint:
nullable: true
type: "string"
liveTextRedacted:
type: "boolean"
normalizedText:
nullable: true
type: "string"
pid:
nullable: true
type: "integer"
query:
description: "Raw live SQL only with view_live_text; never persisted."
nullable: true
type: "string"
querySeconds:
nullable: true
type: "number"
state:
nullable: true
type: "string"
user:
nullable: true
type: "string"
waitEvent:
nullable: true
type: "string"
xactSeconds:
nullable: true
type: "number"
type: "object"
DbPerformanceSettingsV1:
properties:
lastSnapshotAt:
nullable: true
type: "string"
provider:
nullable: true
type: "string"
statsAccess:
nullable: true
type: "string"
statsAccessMessage:
nullable: true
type: "string"
type: "object"
DbPerformanceSnapshotV1:
properties:
cacheHitRatio:
nullable: true
type: "number"
capturedAt:
nullable: true
type: "string"
connectionsTotal:
nullable: true
type: "integer"
deadlocks:
nullable: true
type: "integer"
id:
nullable: true
type: "integer"
maxConnections:
nullable: true
type: "integer"
statementCalls:
nullable: true
type: "integer"
statementTotalMs:
nullable: true
type: "number"
status:
nullable: true
type: "string"
tempBytes:
nullable: true
type: "integer"
type: "object"
DbPerformanceStatementV1:
properties:
calls24h:
nullable: true
type: "integer"
historyPath:
nullable: true
type: "string"
hitRatio:
nullable: true
type: "number"
id:
nullable: true
type: "integer"
meanMs24h:
nullable: true
type: "number"
queryText:
description: "Normalized SQL, truncated to 300 characters."
nullable: true
type: "string"
shareOfDbTime24h:
nullable: true
type: "number"
totalMs24h:
nullable: true
type: "number"
type: "object"
securitySchemes:
sessionUuidCookie:
description: "The session cookie set by POST /qqq/v1/manageSession (web clients)."
in: "cookie"
name: "sessionUUID"
type: "apiKey"
bearerAuth:
bearerFormat: "JWT or session UUID"
description: "`Authorization: Bearer <token>`, where the token is either\n-\
\ an OAuth2 access token (a signed JWT) issued by the app's identity provider\
\ to the app's client,\n or to its native client (`values.nativeClientId`\
\ in GET /qqq/v1/metaData/authentication), for\n example from the device-authorization\
\ grant or auth-code + PKCE. It is verified against the\n provider's keys\
\ (issuer, audience, expiry, signature) and gives the same user and permissions\n\
\ as signing in on the web. When it expires (401), refresh it at the provider\
\ and retry; or\n- a session UUID returned by POST /qqq/v1/manageSession,\
\ which the server keeps refreshed."
scheme: "bearer"
type: "http"
oauth2:
description: "The app's OAuth2 / OpenID Connect provider. The real endpoints\
\ come from the provider's discovery\ndocument: `{values.baseUrl}.well-known/openid-configuration`,\
\ with values from\nGET /qqq/v1/metaData/authentication. Native clients use\
\ `values.nativeClientId` (a public client)\nwith auth-code + PKCE (loopback\
\ or custom-scheme redirect URI registered at the provider) or the\ndevice-authorization\
\ grant (`device_authorization_endpoint`, RFC 8628), then send the access\
\ token\nas `Authorization: Bearer`."
flows:
authorizationCode:
authorizationUrl: "/authorize"
scopes:
email: "Email address"
offline_access: "A refresh token, to renew the access token without signing\
\ in again"
openid: "OpenID Connect sign-in"
profile: "Name, username and groups (permissions come from the groups)"
tokenUrl: "/token"
type: "oauth2"
info:
contact:
email: "contact@kingsrook.com"
description: "Database performance, live activity and audited query plans"
title: "QQQ Middleware API extension: db-performance"
version: "1.0.0"
openapi: "3.0.3"
paths:
/qqq/v1/qbit/db-performance/overview:
get:
description: "Requires db_performance.view and uses the caller's session."
operationId: "dbPerformanceOverview"
parameters:
- description: "Configured monitored backend; defaults to the qbit backend."
in: "query"
name: "backend"
required: false
schema:
type: "string"
responses:
400:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Invalid request"
401:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Sign in required"
403:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Permission denied"
200:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceOverviewV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Database Performance overview"
tags:
- "db-performance"
/qqq/v1/qbit/db-performance/activity:
get:
description: "Requires db_performance.view and uses the caller's session."
operationId: "dbPerformanceActivity"
parameters:
- description: "Configured monitored backend; defaults to the qbit backend."
in: "query"
name: "backend"
required: false
schema:
type: "string"
responses:
400:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Invalid request"
401:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Sign in required"
403:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Permission denied"
200:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceActivityV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Database Performance activity"
tags:
- "db-performance"
/qqq/v1/qbit/db-performance/explain:
post:
description: "PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze.\
\ Values and ANALYZE plans are never persisted."
operationId: "dbPerformanceExplain"
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceExplainRequestV1"
required: true
responses:
400:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Invalid request"
401:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Sign in required"
403:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Permission denied"
200:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceExplainResultV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Explain a query (PostgreSQL JSON or H2 text plan)"
tags:
- "db-performance"
/qqq/v1/qbit/db-performance/statements/lookup:
get:
description: "Requires db_performance.view and uses the caller's session."
operationId: "dbPerformanceLookup"
parameters:
- description: "Configured monitored backend; defaults to the qbit backend."
in: "query"
name: "backend"
required: false
schema:
type: "string"
- in: "query"
name: "sql"
required: true
schema:
type: "string"
responses:
400:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Invalid request"
401:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Sign in required"
403:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceErrorV1"
description: "Permission denied"
200:
content:
application/json:
schema:
$ref: "#/components/schemas/DbPerformanceLookupV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Database Performance lookup"
tags:
- "db-performance"
servers:
- description: "The QQQ server that serves this document (paths include /qqq/v1)."
url: "/"
tags:
- description: "Operations of the db-performance extension."
name: "db-performance"
@@ -0,0 +1,239 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.RollupService;
import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Attribution detail grows by hundreds of thousands of rows a day. On
** 2026-10-16 the home widget loaded a whole day of it into the heap and ran
** the kof22 web pod out of memory, and the hourly analysis filled the worker
** heap the same way. These tests count every attribution detail record QQQ
** materializes, so the widget, the analysis and the daily rollup cannot go
** back to reading the table in one unbounded query.
*******************************************************************************/
class AttributionBoundedReadTest
{
private static final AtomicInteger LOADED = new AtomicInteger();
private static final AtomicInteger MAX_PAGE = new AtomicInteger();
private DbPerfRuntime runtime;
/*******************************************************************************
** Counts the detail records each QueryAction hands back.
*******************************************************************************/
public static class CountingCustomizer implements TableCustomizerInterface
{
/***************************************************************************
** Record the page size, change nothing.
***************************************************************************/
@Override
public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records)
{
LOADED.addAndGet(records.size());
MAX_PAGE.accumulateAndGet(records.size(), Math::max);
return (records);
}
}
/*******************************************************************************
** Fresh H2 instance with the counter on the detail table.
*******************************************************************************/
@BeforeEach
void setup() throws Exception
{
runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
runtime.getInstance().getTable("db_perf_attribution").withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CountingCustomizer.class));
LOADED.set(0);
MAX_PAGE.set(0);
}
/*******************************************************************************
** The widget sums in the database: no detail rows reach the heap, and the
** totals still match the detail.
*******************************************************************************/
@Test
void widgetSumsInTheDatabase() throws Exception
{
Instant now = Instant.now();
insert(3000, i -> row(now.minusSeconds(60), now, i));
LOADED.set(0);
MultiTableData attribution = new DbPerfWidgetRenderer().attribution();
assertThat(LOADED.get()).as("attribution detail rows loaded by the widget").isZero();
List<Map<String, Object>> units = attribution.getTableDataList().getFirst().getRows();
assertThat(units).hasSize(3);
// unit-0 holds i = 0, 3, 6 ... (1000 rows, 2 statements each), mostly against "patient"
assertThat(((Number) units.getFirst().get("Statements")).doubleValue() + ((Number) units.get(1).get("Statements")).doubleValue()
+ ((Number) units.get(2).get("Statements")).doubleValue()).isEqualTo(6000.0);
assertThat(units).allSatisfy(unit -> assertThat(((Number) unit.get("Statements")).doubleValue()).isEqualTo(2000.0));
assertThat(JsonUtils.toJson(units.getFirst())).contains("top table patient");
assertThat(units.stream().mapToDouble(unit -> ((Number) unit.get("Share %")).doubleValue()).sum()).isCloseTo(100.0, org.assertj.core.data.Offset.offset(0.001));
}
/*******************************************************************************
** Detail older than 24 hours is left out of the sums.
*******************************************************************************/
@Test
void widgetKeepsTheDayWindow() throws Exception
{
Instant now = Instant.now();
insert(3, i -> row(now.minusSeconds(60), now, i));
insert(3, i -> row(now.minusSeconds(3 * 86400), now.minusSeconds(2 * 86400), i));
List<Map<String, Object>> units = new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getRows();
assertThat(units).hasSize(3).allSatisfy(unit -> assertThat(((Number) unit.get("Statements")).doubleValue()).isEqualTo(2.0));
}
/*******************************************************************************
** The hourly analysis reads attribution as database sums per unit, table
** and operation: no detail rows, same totals.
*******************************************************************************/
@Test
void analysisSumsInTheDatabase() throws Exception
{
Instant now = Instant.now();
insert(3000, i -> row(now.minusSeconds(60), now, i));
LOADED.set(0);
RuleContext context = new RuleContext(runtime, runtime.getConfig().getBackendName(), new QRecord(), new QRecord(), now.plusSeconds(1));
List<QRecord> rows = context.window("attribution", 1);
assertThat(LOADED.get()).as("attribution detail rows loaded by the analysis").isZero();
assertThat(rows).hasSize(6);
assertThat(rows.stream().mapToDouble(row -> RuleContext.number(row, "statements")).sum()).isEqualTo(6000.0);
assertThat(rows.stream().mapToDouble(row -> RuleContext.number(row, "totalMs")).sum()).isEqualTo(1500 * 10.0 + 1500 * 1.0);
assertThat(rows).allSatisfy(row -> assertThat(row.getValueString("operation")).isEqualTo("SELECT"));
}
/*******************************************************************************
** Keyset pages never exceed the page size, and every row is seen once.
*******************************************************************************/
@Test
void forEachPageIsBounded() throws Exception
{
Instant now = Instant.now();
insert(1050, i -> row(now.minusSeconds(60), now, i));
LOADED.set(0);
List<Integer> sizes = new ArrayList<>();
List<Object> ids = new ArrayList<>();
DbPerfStore.forEachPage("db_perf_attribution", new QQueryFilter(), 100, page ->
{
sizes.add(page.size());
page.forEach(row -> ids.add(row.getValue("id")));
});
assertThat(sizes).hasSize(11).allSatisfy(size -> assertThat(size).isLessThanOrEqualTo(100));
assertThat(ids).hasSize(1050).doesNotHaveDuplicates();
assertThat(MAX_PAGE.get()).isLessThanOrEqualTo(100);
}
/*******************************************************************************
** The daily rollup reads the day in BATCH_SIZE pages and its sums still
** cover every detail row across page boundaries.
*******************************************************************************/
@Test
void rollupReadsTheDayInPages() throws Exception
{
ZoneId zone = ZoneId.of(runtime.getConfig().getTimeZone());
LocalDate day = LocalDate.now(zone).minusDays(1);
Instant start = day.atStartOfDay(zone).toInstant().plusSeconds(3600);
int rows = DbPerfStore.BATCH_SIZE * 2 + 500;
insert(rows, i -> row(start, start.plusSeconds(60), i));
MAX_PAGE.set(0);
LOADED.set(0);
new RollupService(runtime).rollupDay(day);
assertThat(MAX_PAGE.get()).as("largest page of attribution detail held at once").isLessThanOrEqualTo(DbPerfStore.BATCH_SIZE);
assertThat(LOADED.get()).isEqualTo(rows);
List<QRecord> daily = DbPerfStore.query("db_perf_attribution_daily", new QQueryFilter());
assertThat(daily).hasSize(6);
assertThat(daily.stream().mapToLong(row -> row.getValueLong("statements")).sum()).isEqualTo(2L * rows);
}
/*******************************************************************************
** Insert generated rows in batches.
*******************************************************************************/
private static void insert(int count, java.util.function.IntFunction<QRecord> factory) throws Exception
{
List<QRecord> rows = new ArrayList<>();
for(int i = 0; i < count; i++)
{
rows.add(factory.apply(i));
}
DbPerfStore.insert("db_perf_attribution", rows);
}
/*******************************************************************************
** Three units; each unit spends most of its time on "patient".
*******************************************************************************/
private static QRecord row(Instant windowStart, Instant windowEnd, int i)
{
boolean patient = i % 6 < 3;
return (new QRecord().withValue("windowStart", windowStart).withValue("windowEnd", windowEnd).withValue("podName", "pod-" + (i % 2))
.withValue("unitKind", "PROCESS").withValue("unitName", "unit-" + (i % 3)).withValue("tableName", patient ? "patient" : "visit")
.withValue("operation", "SELECT").withValue("fingerprint", "f".repeat(64)).withValue("sqlText", "select 1")
.withValue("units", 1L).withValue("statements", 2L).withValue("rowsCount", 1L).withValue("errors", 0L)
.withValue("totalMs", patient ? BigDecimal.TEN : BigDecimal.ONE).withValue("maxMs", BigDecimal.ONE));
}
}
@@ -0,0 +1,66 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.nio.file.Files;
import java.nio.file.Path;
import com.kingsrook.qbits.dbperformance.api.DbPerformanceUiApi;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData;
import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1;
import com.kingsrook.qqq.middleware.javalin.tools.UiApiContract;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The committed fragment is generated from exactly the registered UI specs.
*******************************************************************************/
class DbPerfApiContractTest
{
/*******************************************************************************
** Explicit regeneration flag; normal builds detect contract drift.
*******************************************************************************/
@Test
void contract() throws Exception
{
var extension = DbPerformanceUiApi.extension();
var version = new MiddlewareVersionV1();
String generated = UiApiContract.generateExtensionYaml(version, extension);
Path path = Path.of("src/main/resources/openapi/qqq-v1/db-performance.yaml");
if(Boolean.getBoolean("dbperf.openapi.update"))
{
Files.createDirectories(path.getParent());
Files.writeString(path, generated);
}
assertThat(UiApiContract.check(generated, Files.readString(path), null, 1, "mvnw -Dtest=DbPerfApiContractTest -Ddbperf.openapi.update=true test")).isEmpty();
var model = version.generateExtensionOpenAPIModel("qqq", extension);
assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> (name.startsWith("DbPerformance") || name.equals("BasicErrorResponseV1")) && name.endsWith("V1"));
assertThat(model.getPaths().keySet()).containsExactlyInAnyOrder("/qqq/v1/qbit/db-performance/overview", "/qqq/v1/qbit/db-performance/activity",
"/qqq/v1/qbit/db-performance/explain", "/qqq/v1/qbit/db-performance/statements/lookup");
assertThat(extension.getSemanticVersion()).isEqualTo("1.0.0");
assertThat(MiddlewareSpecExtensionsMetaData.of(DbPerfTestSupport.instance()).getExtensions()).containsOnlyKeys("db-performance");
assertThat(generated).contains("liveTextRedacted", "parameters", "assistantPrompt", "403");
}
}
@@ -47,6 +47,7 @@ public class DbPerfFixtureProvider extends UnsupportedStatsProvider
public Instant reset = START.minusSeconds(3600);
public boolean limited;
public boolean fail;
public Error error;
/*******************************************************************************
@@ -85,6 +86,10 @@ public class DbPerfFixtureProvider extends UnsupportedStatsProvider
@Override
public List<StatementStats> readStatements(Connection connection)
{
if(error != null)
{
throw (error);
}
return (List.of(new StatementStats(1, "app", true, "select * from public.accounts where id=$1", calls, BigDecimal.valueOf(calls * 10), BigDecimal.TEN, BigDecimal.valueOf(15), calls, calls * 9, calls, 0, reset, reset),
new StatementStats(2, "app", true, "select * from public.accounts where name='private'", calls, BigDecimal.valueOf(calls * 5), BigDecimal.valueOf(5), BigDecimal.TEN, calls, calls, calls, 2, reset, reset)));
}
@@ -76,7 +76,7 @@ class DbPerfHealthTest
DbPerfHealthChecks checks = checks(f, 0);
for(String access : List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "UNSUPPORTED", "UNKNOWN"))
{
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("statsAccess", access)));
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("provider", "POSTGRES").withValue("statsAccess", access)));
var outcome = checks.check("stats_access");
assertThat(outcome.getDetails().getValues()).containsEntry("statsAccess", access).hasSize(1);
assertThat(outcome.getStatus()).isEqualTo(switch(access)
@@ -91,6 +91,14 @@ class DbPerfHealthTest
assertThat(outcome.getMessage()).isEqualTo("grant pg_read_all_stats to sa");
}
}
for(String access : List.of("NO_QUERY_STATISTICS", "NOT_ADMIN"))
{
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("provider", "H2").withValue("statsAccess", access)));
var outcome = checks.check("stats_access");
assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED);
assertThat(outcome.getDetails().getValues()).containsEntry("statsAccess", access);
assertThat(outcome.getMessage()).doesNotContain("pg_").contains(access.equals("NOT_ADMIN") ? "ADMIN TRUE" : "QUERY_STATISTICS=TRUE");
}
for(int total : List.of(79, 80, 94, 95))
{
DbPerfStore.update("db_perf_snapshot", List.of(new QRecord().withValue("id", f.snapshot.getValueInteger("id")).withValue("connectionsTotal", total)));
@@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import com.kingsrook.qbits.dbperformance.metadata.DbPerfHelp;
import com.kingsrook.qbits.dbperformance.metadata.DbPerfMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Version, live references, summary quality and complete owned metadata coverage.
*******************************************************************************/
class DbPerfHelpBundleTest
{
/*******************************************************************************
** No host Help tables are needed for registration or empty-state guidance.
*******************************************************************************/
@Test
void bundleCoverage() throws Exception
{
var instance = DbPerfTestSupport.instance();
HelpBundleAssertions.forInstance(instance).bundles(DbPerfHelp.RESOURCE)
.owning(target -> target.name().startsWith("db_perf_") || target.name().startsWith("dbPerf")).assertComplete();
var bundle = HelpBundles.of(instance).getBundles().getFirst();
assertThat(bundle.getVersion()).isEqualTo(projectVersion());
var keys = bundle.getEntries().stream().map(entry -> entry.getKey()).toList();
instance.getWidgets().keySet().forEach(name -> assertThat(keys).contains("widget:" + name + ";slot:label"));
for(String table : DbPerfMetaData.TABLES)
{
assertThat(instance.getTable(table).getHelpContent()).containsKeys("help", "emptyState");
}
QHelpContent host = new QHelpContent().withContentAsMarkdown("Host-specific help");
instance.getTable("db_perf_statement").withHelpContent("help", host);
DbPerfHelp.register(instance, DbPerfRuntime.of(instance).getConfig());
assertThat(instance.getTable("db_perf_statement").getHelpContent().get("help")).containsExactly(host);
}
/*******************************************************************************
** Compare to the effective Maven project version supplied to this test's resource.
*******************************************************************************/
private static String projectVersion() throws Exception
{
try(var stream = DbPerfHelpBundleTest.class.getResourceAsStream("/help/project-version.txt"))
{
return (new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).trim());
}
}
}
@@ -0,0 +1,295 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dbperformance.api.ActivitySpec;
import com.kingsrook.qbits.dbperformance.api.DbPerformanceData;
import com.kingsrook.qbits.dbperformance.api.ExplainSpec;
import com.kingsrook.qbits.dbperformance.api.LookupSpec;
import com.kingsrook.qbits.dbperformance.api.OverviewSpec;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainResult;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer;
import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgets;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec.JsonInput;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** H2 fixtures exercise real QQQ reads, typed widgets and the actual spec handlers.
*******************************************************************************/
class DbPerfPresentationTest
{
private DbPerfRuntime runtime;
private DbPerformanceData data;
/*******************************************************************************
** Isolated migrated tables and production producer registration.
*******************************************************************************/
@BeforeEach
void setup() throws Exception
{
runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
data = new DbPerformanceData(runtime);
}
/*******************************************************************************
** Every home widget renders an honest guided empty state and requires view.
*******************************************************************************/
@Test
void emptyWidgetsAndPermissions() throws Exception
{
assertThat(runtime.getInstance().getApp("dbPerformance").getWidgets()).containsExactlyElementsOf(DbPerfWidgets.HOME);
for(String name : DbPerfWidgets.HOME)
{
var meta = runtime.getInstance().getWidget(name);
assertThat(meta.getGridColumns()).isEqualTo(12);
assertThat(meta.getPermissionRules().getPermissionBaseName()).isEqualTo(DbPerfPermissions.VIEW);
var output = new DbPerfWidgetRenderer().render(new RenderWidgetInput().withWidgetMetaData(meta));
assertThat(output.getWidgetData()).isNotNull();
if(output.getWidgetData() instanceof TableData table)
{
assertThat(table.getFixedHeight()).isNull();
assertThat(table.getRowsPerPage()).isEqualTo(10);
}
}
assertThat(new DbPerfWidgetRenderer().topQueries(data).getNoRowsFoundHTML()).contains("two snapshots");
assertThat(new DbPerfWidgetRenderer().liveActivity(data).getNoRowsFoundHTML()).isEqualTo("No long-running or blocked sessions right now");
assertThat(new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getNoRowsFoundHTML()).isEqualTo("Attribution starts after the next flush");
assertThat(new DbPerfWidgetRenderer().connectionStatistics(data).getType()).isEqualTo("multiStatistics");
assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isNotEmpty().allSatisfy(row -> assertThat(row).containsKeys("Application", "User", "State", "Connections"));
QContext.setQSession(new QSession());
for(String name : DbPerfWidgets.HOME)
{
assertThatThrownBy(() -> new DbPerfWidgetRenderer().render(new RenderWidgetInput().withWidgetMetaData(runtime.getInstance().getWidget(name)))).hasMessageContaining("permission");
}
assertThatThrownBy(() -> new OverviewSpec().handle(new JsonInput())).hasMessageContaining("permission");
assertThatThrownBy(() -> new ActivitySpec().handle(new JsonInput())).hasMessageContaining("permission");
assertThatThrownBy(() -> new LookupSpec().handle(new JsonInput())).hasMessageContaining("permission");
}
/*******************************************************************************
** Persisted rows change the API and widgets, with correct units and record links.
*******************************************************************************/
@Test
@SuppressWarnings("unchecked")
void fixtureRows() throws Exception
{
fixture();
Map<String, Object> overview = (Map<String, Object>) new OverviewSpec().handle(new JsonInput());
assertThat((List<?>) overview.get("topStatements")).hasSize(1);
assertThat((Map<String, Integer>) overview.get("findingsBySeverity")).containsEntry("HIGH", 1);
TableData queries = new DbPerfWidgetRenderer().topQueries(data);
assertThat(queries.getRows()).hasSize(1);
assertThat((BigDecimal) queries.getRows().getFirst().get("Share of DB time %")).isEqualByComparingTo("25");
assertThat(JsonUtils.toJson(queries)).contains("/app/dbPerformance/db_perf_statement/", "composite").doesNotContain("private-value");
ChartData chart = new DbPerfWidgetRenderer().databaseTime(data);
assertThat(chart.getChartData().getDatasets()).hasSize(2);
assertThat(chart.getChartData().getDatasets().getFirst().getData()).hasSize(1);
assertThat(new DbPerfWidgetRenderer().findings(data).getRows()).hasSize(1);
assertThat(new DbPerfWidgetRenderer().tables(data).getTableDataList().getFirst().getRows()).hasSize(1);
MultiTableData attribution = new DbPerfWidgetRenderer().attribution();
assertThat(JsonUtils.toJson(attribution.getTableDataList().getFirst().getRows().getFirst())).contains("top table patient");
assertThat(attribution.getTableDataList().get(1).getRows()).hasSize(1);
assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().connectionStatistics(data))).contains("Deadlocks", "Cache hit");
JsonInput lookup = new JsonInput();
lookup.getQueryParams().put("sql", "SELECT * FROM patient WHERE name = 'different'");
assertThat((List<?>) ((Map<?, ?>) new LookupSpec().handle(lookup)).get("matches")).hasSize(1);
runtime.getConfig().withAppName("custom");
assertThat(data.recordPath("db_perf_statement", 1)).isEqualTo("/app/custom/db_perf_statement/1");
JsonInput invalid = new JsonInput();
invalid.getQueryParams().put("backend", "other");
assertThatThrownBy(() -> new OverviewSpec().handle(invalid)).hasMessageContaining("monitored backend");
assertThatThrownBy(() -> new LookupSpec().handle(new JsonInput())).hasMessageContaining("Supply SQL");
}
/*******************************************************************************
** Raw text is permission gated; transitive blocking includes indirect descendants.
*******************************************************************************/
@Test
@SuppressWarnings("unchecked")
void liveTextAndBlocking() throws Exception
{
List<SessionInfo> sessions = List.of(session(10, List.of()), session(20, List.of(10)), session(30, List.of(20)));
QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW));
Map<String, Object> redacted = data.activity(sessions, true);
assertThat(JsonUtils.toJson(redacted)).doesNotContain("private-value").contains("liveTextRedacted");
List<Map<String, Object>> chains = (List<Map<String, Object>>) redacted.get("blockingChains");
assertThat(chains).hasSize(1);
assertThat(chains.getFirst()).containsEntry("rootPid", 10).containsEntry("blockedPids", List.of(20, 30));
QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.VIEW_LIVE_TEXT));
assertThat(JsonUtils.toJson(data.activity(sessions, true))).contains("private-value");
assertThat(JsonUtils.toJson(data.activity(sessions, false))).doesNotContain("private-value");
assertThat(((Map<?, ?>) new ActivitySpec().handle(new JsonInput())).get("sessions")).isEqualTo(List.of());
}
/*******************************************************************************
** The actual home renderer requests redacted live data, even for privileged viewers.
*******************************************************************************/
@Test
void liveWidgetRedaction() throws Exception
{
List<SessionInfo> sessions = List.of(session(10, List.of()), session(20, List.of(10)));
DbPerformanceData fixtureData = new DbPerformanceData(runtime)
{
@Override
public Map<String, Object> activity(String backend, boolean allowRawText) throws Exception
{
assertThat(allowRawText).isFalse();
return (activity(sessions, allowRawText));
}
};
QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW));
TableData table = new DbPerfWidgetRenderer().liveActivity(fixtureData);
assertThat(table.getRows()).hasSize(2);
assertThat(JsonUtils.toJson(table)).doesNotContain("private-value").contains("select * from patient where name = ?", "blocks 1", "blocked by");
QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.VIEW_LIVE_TEXT));
assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().liveActivity(fixtureData))).doesNotContain("private-value");
}
/*******************************************************************************
** The record action requires one statement and removes parameter text on every exit.
*******************************************************************************/
@Test
void recordAction() throws Exception
{
var process = runtime.getInstance().getProcess(QueryPlanProcess.NAME);
assertThat(process.getTableName()).isEqualTo("db_perf_statement");
assertThat(process.getMinInputRecords()).isEqualTo(1);
assertThat(process.getMaxInputRecords()).isEqualTo(1);
RunBackendStepInput input = new RunBackendStepInput();
input.addValue("parameterValues", "private-value");
assertThatThrownBy(() -> new QueryPlanProcess().run(input, new RunBackendStepOutput())).hasMessageContaining("exactly one");
assertThat(input.getValues()).doesNotContainKey("parameterValues");
input.withRecords(List.of(new QRecord().withValue("id", 1)));
input.addValue("mode", "ANALYZE");
input.addValue("parameterValues", "private-value\nsecond-value");
QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.EXPLAIN));
assertThatThrownBy(() -> new QueryPlanProcess().run(input, new RunBackendStepOutput())).hasMessageContaining(DbPerfPermissions.EXPLAIN_ANALYZE);
assertThat(input.getValues()).doesNotContainKey("parameterValues");
input.addValue("mode", "INVALID");
assertThatThrownBy(() -> new QueryPlanProcess().run(input, new RunBackendStepOutput())).hasMessageContaining("Choose PLAN");
assertThat(runtime.getInstance().getProcesses().values()).noneMatch(p -> "db_perf_explain".equals(p.getTableName()));
}
/*******************************************************************************
** Audit denied, invalid and unsupported-vendor paths using H2; no JDBC execution.
*******************************************************************************/
@Test
void explainAuditAndHandler() throws Exception
{
ExplainService service = new ExplainService(runtime);
ExplainResult h2 = service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, "select current_date", ExplainRequest.Mode.PLAN, List.of()));
assertThat(h2.planJson()).isNull();
assertThat(h2.planText()).containsIgnoringCase("CURRENT_DATE");
assertThatThrownBy(() -> service.explain(null)).hasMessageContaining("Choose PLAN");
assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, null, ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("either");
QContext.setQSession(new QSession());
JsonInput input = new JsonInput();
input.getBody().put("mode", "ANALYZE");
input.getBody().put("sql", "select 'private-value'");
input.getBody().put("parameters", List.of("private-value"));
assertThatThrownBy(() -> new ExplainSpec().handle(input)).hasMessageContaining(DbPerfPermissions.EXPLAIN_ANALYZE);
List<QRecord> rows = runtime.asSystem(() -> DbPerfStore.query("db_perf_explain", new QQueryFilter()));
assertThat(rows).hasSize(4);
assertThat(rows).anyMatch(row -> "DENIED".equals(row.getValueString("outcome")));
assertThat(JsonUtils.toJson(rows)).doesNotContain("private-value");
input.getBody().put("parameters", List.of(5));
assertThatThrownBy(() -> new ExplainSpec().handle(input)).hasMessageContaining("array of strings");
input.getBody().put("statementId", "bad");
assertThatThrownBy(() -> new ExplainSpec().handle(input)).hasMessageContaining("integer");
}
/*******************************************************************************
** A realistic session whose raw text must not reach a home widget or storage.
*******************************************************************************/
private static SessionInfo session(int pid, List<Integer> blockedBy)
{
return (new SessionInfo(pid, "app", "QQQ", null, "active", "Lock", "transactionid", "client backend", Instant.now(), Instant.now(), Instant.now(),
BigDecimal.valueOf(60), BigDecimal.valueOf(60), blockedBy, 1L, "select * from patient where name = 'private-value'"));
}
/*******************************************************************************
** One row in every presentation source proves the renderer consumes persisted data.
*******************************************************************************/
private void fixture() throws Exception
{
new DbPerfSettingsService(runtime).ensure(runtime.getConfig().getBackendName());
QRecord statement = DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("queryId", 1L)
.withValue("queryText", "select * from patient where name = ?").withValue("fingerprint", SqlNormalizer.fingerprint(SqlNormalizer.normalize("select * from patient where name = ?")))
.withValue("calls24h", 5L).withValue("totalMs24h", BigDecimal.valueOf(50)).withValue("meanMs24h", BigDecimal.TEN).withValue("shareOfDbTime24h", BigDecimal.valueOf(25)))).getFirst();
QRecord snapshot = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("capturedAt", Instant.now())
.withValue("status", "OK").withValue("statementTotalMs", BigDecimal.valueOf(200)).withValue("statementCalls", 20L).withValue("connectionsTotal", 5).withValue("maxConnections", 100)
.withValue("cacheHitRatio", new BigDecimal("0.95")).withValue("deadlocks", 2L).withValue("tempBytes", 1048576L))).getFirst();
DbPerfStore.insert("db_perf_query_snapshot", List.of(new QRecord().withValue("statementId", statement.getValueInteger("id")).withValue("snapshotId", snapshot.getValueInteger("id"))
.withValue("capturedAt", Instant.now()).withValue("sharedBlksHit", 90L).withValue("sharedBlksRead", 10L)));
DbPerfStore.insert("db_perf_finding", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("dedupeKey", "test").withValue("status", "OPEN")
.withValue("severity", "HIGH").withValue("title", "Slow query").withValue("ruleCode", "SLOW_QUERY").withValue("score", BigDecimal.TEN)));
DbPerfStore.insert("db_perf_table_stat", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("schemaName", "public").withValue("tableName", "patient").withValue("relid", 1L)));
DbPerfStore.insert("db_perf_attribution", List.of(new QRecord().withValue("windowStart", Instant.now().minusSeconds(60)).withValue("windowEnd", Instant.now()).withValue("unitKind", "PROCESS")
.withValue("unitName", "sync").withValue("tableName", "patient").withValue("statements", 3L).withValue("totalMs", BigDecimal.TEN)));
DbPerfStore.insert("db_perf_repeat_pattern", List.of(new QRecord().withValue("windowStart", Instant.now().minusSeconds(60)).withValue("windowEnd", Instant.now()).withValue("unitKind", "PROCESS")
.withValue("unitName", "sync").withValue("maxRepeats", 12L)));
}
}
@@ -0,0 +1,322 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainResult;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.postgresql.core.QueryExecutor;
import org.postgresql.jdbc.PgStatement;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.withSettings;
/*******************************************************************************
** Driver contract tests: cleanup, state restoration, unknown binding and audit isolation.
** These supplement, but do not replace, the real PostgreSQL acceptance tests.
*******************************************************************************/
class ExplainConnectionTest
{
private DbPerfRuntime runtime;
private final List<String> events = new ArrayList<>();
private int version = 17;
private String failureState;
private boolean autoCommit = true;
private boolean readOnly = false;
private String json = "[{\"Plan\":{\"Node Type\":\"Result\",\"Total Cost\":1.25,\"Actual Total Time\":0.1}}]";
/*******************************************************************************
** Real H2 storage for the audit and settings; only plan execution JDBC is simulated.
*******************************************************************************/
@BeforeEach
void setup() throws Exception
{
runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
}
/*******************************************************************************
** Parameters reach setObject as unknown types; success never commits the plan transaction.
*******************************************************************************/
@Test
void analyzeCleanupAndBinding() throws Exception
{
ExplainResult result = service().explain(request("select $1::int", ExplainRequest.Mode.ANALYZE, List.of("876543")));
assertThat(result.mode()).isEqualTo("ANALYZE");
assertThat(result.explainId()).isNotNull();
assertThat(result.actualTotalMs()).isEqualByComparingTo("0.1");
assertThat(events).anyMatch(event -> event.startsWith("PREPARE dbperf_") && event.endsWith("AS select $1::int"));
assertThat(events).anyMatch(event -> event.startsWith("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE dbperf_") && event.endsWith("(?)"));
assertThat(events).contains("bind:1:" + Types.OTHER).doesNotContain("commit");
assertThat(events).noneMatch(event -> event.contains("876543"));
assertThat(events.indexOf("rollback")).isLessThan(index("DEALLOCATE"));
assertThat(events.getLast()).isEqualTo("close");
assertThat(autoCommit).isTrue();
assertThat(readOnly).isFalse();
assertThat(audit().getValueString("planJson")).isNull();
assertThat(audit().getValueString("outcome")).isEqualTo("OK");
}
/*******************************************************************************
** Driver cancellation is classified by SQLSTATE, not a potentially sensitive message.
*******************************************************************************/
@Test
void timeoutAndErrorCleanup() throws Exception
{
for(String state : List.of("57014", "25006"))
{
failureState = state;
events.clear();
assertThatThrownBy(() -> service().explain(request("select $1::int", ExplainRequest.Mode.ANALYZE, List.of("876543"))))
.hasMessageContaining(state.equals("57014") ? "timed out" : "Writes are not allowed").hasNoCause();
assertThat(audit().getValueString("outcome")).isEqualTo(state.equals("57014") ? "TIMEOUT" : "ERROR");
assertThat(audit().getValueString("errorMessage")).doesNotContain("private");
assertThat(events.indexOf("rollback")).isLessThan(index("DEALLOCATE"));
assertThat(events).doesNotContain("commit");
assertThat(autoCommit).isTrue();
}
}
/*******************************************************************************
** Generic version guard, literal-free storage, and normalized statement-source linkage.
*******************************************************************************/
@Test
void planStorageAndVersion() throws Exception
{
assertThat(service().explain(request("select current_date", ExplainRequest.Mode.PLAN, List.of())).mode()).isEqualTo("PLAN");
assertThat(audit().getValueString("planJson")).isEqualTo(json);
assertThat(events).contains("SET TRANSACTION READ ONLY", "SET LOCAL statement_timeout = 10000", "rollback");
version = 15;
assertThatThrownBy(() -> service().explain(request("select $1::int", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("PostgreSQL 16");
version = 17;
QRecord statement = DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName())
.withValue("queryId", 42L).withValue("queryText", "select $1::int"))).getFirst();
ExplainResult generic = service().explain(new ExplainRequest(runtime.getConfig().getBackendName(), statement.getValueInteger("id"), null, ExplainRequest.Mode.PLAN, List.of()));
assertThat(generic.mode()).isEqualTo("GENERIC");
assertThat(audit().getValueInteger("statementId")).isEqualTo(statement.getValueInteger("id"));
assertThat(audit().getValueString("planJson")).isEqualTo(json);
assertThat(events).contains("EXPLAIN (GENERIC_PLAN, FORMAT JSON) select $1::int");
service().explain(request("SELECT 'private'", ExplainRequest.Mode.PLAN, List.of()));
assertThat(audit().getValueString("planJson")).isNull();
assertThat(audit().getValueString("sqlText")).isEqualTo("select ?");
}
/*******************************************************************************
** Pool state can start non-default; no result can escape without valid JSON.
*******************************************************************************/
@Test
void restoreNonDefaultAndBadPlan() throws Exception
{
autoCommit = false;
readOnly = true;
json = "{}";
assertThatThrownBy(() -> service().explain(request("select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("Query plan failed");
assertThat(autoCommit).isFalse();
assertThat(readOnly).isTrue();
assertThat(events).contains("rollback", "close");
assertThat(audit().getValueString("outcome")).isEqualTo("ERROR");
}
/*******************************************************************************
** A separate connection boundary keeps real QQQ settings and audits untouched.
*******************************************************************************/
private ExplainService service()
{
return (new ExplainService(runtime)
{
@Override
protected Connection openConnection()
{
return (connection());
}
});
}
/*******************************************************************************
** JDBC protocol recorder. Unexpected calls fail instead of silently succeeding.
*******************************************************************************/
private Connection connection()
{
return (proxy(Connection.class, (proxy, method, args) ->
{
switch(method.getName())
{
case "getMetaData":
return (proxy(DatabaseMetaData.class, (p, m, a) -> m.getName().equals("getDatabaseProductName") ? "PostgreSQL" : version));
case "getAutoCommit":
return (autoCommit);
case "isReadOnly":
return (readOnly);
case "setAutoCommit":
autoCommit = (Boolean) args[0];
events.add("autoCommit:" + autoCommit);
return (null);
case "setReadOnly":
readOnly = (Boolean) args[0];
events.add("readOnly:" + readOnly);
return (null);
case "createStatement":
return (statement(false));
case "prepareStatement":
events.add((String) args[0]);
return (statement(true));
case "rollback", "commit", "close":
events.add(method.getName());
return (null);
default:
throw (new AssertionError("Unexpected connection call: " + method.getName()));
}
}));
}
/*******************************************************************************
** Record only bind position and type, never the synthetic sensitive fixture value.
*******************************************************************************/
private Statement statement(boolean prepared)
{
InvocationHandler handler = (proxy, method, args) ->
{
switch(method.getName())
{
case "execute":
events.add((String) args[0]);
return (false);
case "unwrap":
return (proxy);
case "executeWithFlags":
assertThat(args[args.length - 1]).isEqualTo(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
if(args.length == 2)
{
events.add((String) args[0]);
}
if(failureState != null)
{
throw (new SQLException("private-value in driver error", failureState));
}
return (true);
case "getResultSet":
return (proxy(ResultSet.class, (p, m, a) -> switch(m.getName())
{
case "next" -> true;
case "getString" -> json;
case "close" -> null;
default -> throw new AssertionError("Unexpected result call: " + m.getName());
}));
case "setObject":
assertThat(args[1]).isEqualTo("876543");
events.add("bind:" + args[0] + ":" + args[2]);
return (null);
case "close":
return (null);
default:
throw (new AssertionError("Unexpected statement call: " + method.getName()));
}
};
var settings = withSettings().defaultAnswer(invocation -> handler.invoke(invocation.getMock(), invocation.getMethod(), invocation.getArguments()));
if(prepared)
{
settings.extraInterfaces(PreparedStatement.class);
}
return (mock(PgStatement.class, settings));
}
/*******************************************************************************
** Type-safe interface proxy for the small JDBC contract exercised above.
*******************************************************************************/
private static <T> T proxy(Class<T> type, InvocationHandler handler)
{
return (type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[] {type}, handler)));
}
/*******************************************************************************
** Position of a protocol event for ordering assertions.
*******************************************************************************/
private int index(String prefix)
{
for(int i = 0; i < events.size(); i++)
{
if(events.get(i).startsWith(prefix))
{
return (i);
}
}
throw (new AssertionError("Missing event " + prefix));
}
/*******************************************************************************
** Current backend request.
*******************************************************************************/
private ExplainRequest request(String sql, ExplainRequest.Mode mode, List<String> parameters)
{
return (new ExplainRequest(runtime.getConfig().getBackendName(), null, sql, mode, parameters));
}
/*******************************************************************************
** Most recent persisted audit.
*******************************************************************************/
private QRecord audit() throws Exception
{
return (DbPerfStore.first("db_perf_explain", new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false))));
}
}
@@ -0,0 +1,239 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.sql.Connection;
import java.sql.Statement;
import java.util.List;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainResult;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** PostgreSQL 17 acceptance tests for permission, PHI and READ ONLY boundaries.
*******************************************************************************/
@Testcontainers
class ExplainServiceTest
{
@Container
static final PostgreSQLContainer<?> POSTGRES = DbPerfPostgresSupport.container();
private DbPerfRuntime runtime;
private ExplainService service;
/*******************************************************************************
** A volatile function attempts a write even though its caller is a SELECT.
*******************************************************************************/
@BeforeAll
static void fixture() throws Exception
{
try(Connection connection = DbPerfPostgresSupport.connect(POSTGRES); Statement statement = connection.createStatement())
{
statement.execute("CREATE TABLE explain_probe (id integer)");
statement.execute("CREATE FUNCTION explain_write() RETURNS integer LANGUAGE plpgsql VOLATILE AS $$ BEGIN INSERT INTO explain_probe VALUES (1); RETURN 1; END $$");
}
}
/*******************************************************************************
** Fresh QQQ runtime, sharing the container's migrated schema.
*******************************************************************************/
@BeforeEach
void setup() throws Exception
{
runtime = DbPerfRuntime.of(DbPerfTestSupport.instance(DbPerfPostgresSupport.backend(POSTGRES)));
service = new ExplainService(runtime);
}
/*******************************************************************************
** Literal-free PLAN is persisted, while literals and ANALYZE are transient.
*******************************************************************************/
@Test
void planAndGeneric() throws Exception
{
ExplainResult plain = explain("select * from explain_probe", ExplainRequest.Mode.PLAN, List.of());
assertThat(plain.mode()).isEqualTo("PLAN");
assertThat(plain.explainId()).isNotNull();
assertThat(audit().getValueString("planJson")).isEqualTo(plain.planJson());
assertThat(plain.planText()).contains("Seq Scan", "explain_probe", "Total Cost", "Plan Rows");
assertThat(plain.sqlWorkbenchUrl()).contains("?sql=EXPLAIN+", "&backend=");
ExplainResult generic = explain("select * from explain_probe where id = $1", ExplainRequest.Mode.PLAN, List.of());
assertThat(generic.mode()).isEqualTo("GENERIC");
assertThat(explain("select * from explain_probe where id = ?", ExplainRequest.Mode.PLAN, List.of()).mode()).isEqualTo("GENERIC");
ExplainResult literal = explain("select * from explain_probe where id = 987654", ExplainRequest.Mode.PLAN, List.of());
assertThat(audit().getValueString("planJson")).isNull();
assertThat(audit().getValueString("sqlText")).doesNotContain("987654");
assertThat(literal.assistantPrompt()).doesNotContain("987654");
}
/*******************************************************************************
** Bind values as unknown JDBC values, never in SQL interpolation or persistent rows.
*******************************************************************************/
@Test
void analyze() throws Exception
{
ExplainResult result = explain("select * from explain_probe where id = $1", ExplainRequest.Mode.ANALYZE, List.of("987654"));
assertThat(result.actualTotalMs()).isNotNull();
assertThat(result.mode()).isEqualTo("ANALYZE");
assertThat(result.assistantPrompt()).doesNotContain("987654");
assertThat(audit().getValueString("outcome")).isEqualTo("OK");
assertThat(audit().getValueString("planJson")).isNull();
assertThat(audit().getValueString("planText")).isNull();
assertThat(audit().getValueInteger("parameterCount")).isEqualTo(1);
assertThat(explain("select $1::text", ExplainRequest.Mode.ANALYZE, List.of("'; DELETE FROM explain_probe; --")).actualTotalMs()).isNotNull();
assertThat(explain("select current_date", ExplainRequest.Mode.ANALYZE, List.of()).actualTotalMs()).isNotNull();
}
/*******************************************************************************
** Rejected requests still receive sanitized audit rows.
*******************************************************************************/
@Test
void rejectUnsafeSql() throws Exception
{
for(String sql : List.of("DELETE FROM explain_probe", "WITH d AS (DELETE FROM explain_probe RETURNING *) SELECT * FROM d",
"SELECT * FROM explain_probe; DELETE FROM explain_probe", "SELECT * INTO probe_copy FROM explain_probe", "SELECT * FROM explain_probe FOR UPDATE",
"SELECT * FROM explain_probe FOR SHARE", "WITH d AS (SELECT 1) DELETE FROM explain_probe"))
{
assertThatThrownBy(() -> explain(sql, ExplainRequest.Mode.ANALYZE, List.of())).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class).satisfies(e -> assertThat(e.getCause()).isNull());
assertThat(audit().getValueString("outcome")).isEqualTo("ERROR");
}
assertThatThrownBy(() -> explain("select $1::int", ExplainRequest.Mode.ANALYZE, List.of())).hasMessageContaining("exactly 1");
assertThatThrownBy(() -> explain("select current_date", ExplainRequest.Mode.PLAN, List.of("secret"))).hasMessageContaining("only for ANALYZE");
}
/*******************************************************************************
** A write hidden in a volatile function is stopped by the database itself.
*******************************************************************************/
@Test
void readOnlyAndTimeout() throws Exception
{
assertThatThrownBy(() -> explain("select explain_write()", ExplainRequest.Mode.ANALYZE, List.of())).hasMessageContaining("Writes are not allowed");
try(Connection connection = DbPerfPostgresSupport.connect(POSTGRES); Statement statement = connection.createStatement(); var rows = statement.executeQuery("SELECT count(*) FROM explain_probe"))
{
rows.next();
assertThat(rows.getInt(1)).isZero();
}
QRecord settings = new DbPerfSettingsService(runtime).ensure(runtime.getConfig().getBackendName());
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", settings.getValueInteger("id")).withValue("explainTimeoutMs", 10)));
assertThatThrownBy(() -> explain("select pg_sleep($1)", ExplainRequest.Mode.ANALYZE, List.of("1"))).hasMessageContaining("timed out");
assertThat(audit().getValueString("outcome")).isEqualTo("TIMEOUT");
assertThat(explain("select current_date", ExplainRequest.Mode.PLAN, List.of()).mode()).isEqualTo("PLAN");
}
/*******************************************************************************
** Denial is audited as system but caller identity and context are retained.
*******************************************************************************/
@Test
void deniedAndValidation() throws Exception
{
QSession caller = new QSession().withUser(new QUser().withIdReference("viewer"));
QContext.setQSession(caller);
assertThatThrownBy(() -> explain("select 'private'", ExplainRequest.Mode.PLAN, List.of())).hasMessageContaining(DbPerfPermissions.EXPLAIN);
assertThat(QContext.getQSession()).isSameAs(caller);
QRecord denied = runtime.asSystem(this::audit);
assertThat(denied.getValueString("outcome")).isEqualTo("DENIED");
assertThat(denied.getValueString("requestedBy")).isEqualTo("viewer");
assertThat(denied.getValueString("sqlText")).doesNotContain("private");
caller.withPermissions(DbPerfPermissions.EXPLAIN);
assertThatThrownBy(() -> explain("select current_date", ExplainRequest.Mode.ANALYZE, List.of())).hasMessageContaining(DbPerfPermissions.EXPLAIN_ANALYZE);
assertThatThrownBy(() -> service.explain(new ExplainRequest("wrong", null, "select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("monitored backend");
assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), 999999, null, ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("not found");
}
/*******************************************************************************
** The record action carries a real PostgreSQL plan into copyable result fields.
*******************************************************************************/
@Test
void recordActionResult() throws Exception
{
QRecord statement = DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName())
.withValue("queryId", 123456L).withValue("queryText", "select * from explain_probe where id = ?"))).getFirst();
RunBackendStepInput input = new RunBackendStepInput().withRecords(List.of(statement));
RunBackendStepOutput output = new RunBackendStepOutput();
new QueryPlanProcess().run(input, output);
assertThat(output.getValueString("planText")).contains("explain_probe");
assertThat(output.getValueBigDecimal("totalCost")).isNotNull();
assertThat(output.getValueString("sqlWorkbenchUrl")).contains("?sql=EXPLAIN");
assertThat(output.getValueString("assistantPrompt")).contains("Query:", "Plan:");
assertThat(audit().getValueString("planJson")).isNotNull();
input.addValue("mode", "ANALYZE");
input.addValue("parameterValues", "987654");
new QueryPlanProcess().run(input, output);
assertThat(input.getValues()).doesNotContainKey("parameterValues");
assertThat(audit().getValueString("planJson")).isNull();
assertThat(output.getValueString("assistantPrompt")).doesNotContain("987654");
}
/*******************************************************************************
** Use the configured backend for each direct SQL request.
*******************************************************************************/
private ExplainResult explain(String sql, ExplainRequest.Mode mode, List<String> parameters) throws Exception
{
return (service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, sql, mode, parameters)));
}
/*******************************************************************************
** Last audit across the shared container.
*******************************************************************************/
private QRecord audit() throws Exception
{
return (DbPerfStore.first("db_perf_explain", new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false))));
}
}
@@ -0,0 +1,86 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import com.kingsrook.qbits.dbperformance.explain.ExplainSql;
import com.kingsrook.qbits.dbperformance.explain.PlanTextRenderer;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Lexical security, bind-slot identity and readable plan rendering without JDBC.
*******************************************************************************/
class ExplainSqlTest
{
/*******************************************************************************
** Quotes, nested comments and repeated parameter references retain their meaning.
*******************************************************************************/
@Test
void lexicalBoundaries() throws Exception
{
var parsed = ExplainSql.parse("/* UPDATE /* DELETE */ */ SELECT ?, '?', E'foo\\'UPDATE', $$DELETE;$$, \"INTO\" -- DELETE\n ;");
parsed.requireSelect();
assertThat(parsed.parameterCount()).isEqualTo(1);
assertThat(parsed.sql()).contains("SELECT $1", "'?'").doesNotContain("/*");
assertThat(ExplainSql.parse("SELECT $1::int + $1, $2").parameterCount()).isEqualTo(2);
assertThat(ExplainSql.parse("SELECT col ?| array['a'], col ?& array['b'] FROM example").parameterCount()).isZero();
for(String sql : new String[] {"select $0", "select $999999999999999999", "select $2", "select ?, $1", "select 'bad", "select $$bad", "/*bad", "select 1; select 2", "", " ", "select 1;;"})
{
assertThatThrownBy(() -> ExplainSql.parse(sql)).isInstanceOf(Exception.class);
}
assertThatThrownBy(() -> ExplainSql.parse(null)).hasMessageContaining("Supply");
assertThatThrownBy(() -> ExplainSql.parse("x".repeat(100001))).hasMessageContaining("100000");
for(String sql : new String[] {"delete from x", "with x as (insert into t values (1)) select * from x", "select * from t for no key update", "select * from t for key share", "select * into t from x"})
{
assertThatThrownBy(() -> ExplainSql.parse(sql).requireSelect()).hasMessageContaining("single SELECT");
}
ExplainSql.parse("with x as (select 'UPDATE') select * from x").requireSelect();
}
/*******************************************************************************
** Render hierarchy and index detail; sanitize plan expressions in assistant prompts.
*******************************************************************************/
@Test
void planTree() throws Exception
{
var plan = PlanTextRenderer.render("""
[{"Plan":{"Node Type":"Nested Loop","Startup Cost":0.1,"Total Cost":42.5,"Plan Rows":2,
"Plans":[{"Node Type":"Index Scan","Relation Name":"patient","Index Name":"patient_pk",
"Index Cond":"(id = 987654)","Filter":"(name = 'private-name'::text)","Actual Rows":1}]},"Execution Time":1.25}]
""");
assertThat(plan.text()).contains("Nested Loop", "\n Index Scan", "patient_pk", "Total Cost: 42.5", "Plan Rows: 2", "private-name");
assertThat(plan.safeText()).doesNotContain("private-name", "987654").contains("id = ?", "name = ?");
assertThat(plan.totalCost()).isEqualByComparingTo("42.5");
assertThat(plan.actualTotalMs()).isEqualByComparingTo("1.25");
assertThat(PlanTextRenderer.render("{\"Plan\":{\"Node Type\":\"Result\"}}").actualTotalMs()).isNull();
assertThat(PlanTextRenderer.render("{\"Plan\":{\"Actual Total Time\":2}}").actualTotalMs()).isEqualByComparingTo("2");
assertThatThrownBy(() -> PlanTextRenderer.render("[]")).hasMessageContaining("plan tree");
assertThatThrownBy(() -> PlanTextRenderer.render("broken")).isInstanceOf(Exception.class);
}
}
@@ -0,0 +1,173 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.analyze.AnalyzeService;
import com.kingsrook.qbits.dbperformance.api.DbPerformanceData;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainResult;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qbits.dbperformance.processes.DbPerfEnableStatisticsStep;
import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** The qbit end to end on an H2 backend: collection, the enable action,
** feature-gated analysis, setup rows and text plans.
*******************************************************************************/
class H2CollectionTest
{
/*******************************************************************************
** Two collections store H2 statements, tables, indexes and connections.
*******************************************************************************/
@Test
void collectAnalyzeAndExplain() throws Exception
{
DbPerfRuntime runtime = runtime(";QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000");
String backend = runtime.getConfig().getBackendName();
workload(runtime);
QRecord first = new SnapshotCollector(runtime).collect(backend, true);
assertThat(first.getValueString("status")).as(first.getValueString("errorMessage")).isEqualTo("OK");
workload(runtime);
QRecord second = new SnapshotCollector(runtime).collect(backend, true);
assertThat(second.getValueString("status")).as(second.getValueString("errorMessage")).isEqualTo("OK");
assertThat(second.getValueInteger("intervalSeconds")).isNotNull();
assertThat(second.getValue("cacheHitRatio")).isNull();
assertThat(second.getValue("maxConnections")).isNull();
assertThat(second.getValueInteger("connectionsTotal")).isPositive();
QRecord settings = new DbPerfSettingsService(runtime).ensure(backend);
assertThat(settings.getValueString("provider")).isEqualTo("H2");
assertThat(settings.getValueString("statsAccess")).isEqualTo("FULL");
List<QRecord> statements = DbPerfStore.query("db_perf_statement", DbPerfStore.eq("backendName", backend));
assertThat(statements).anySatisfy(row ->
{
assertThat(row.getValueString("queryText")).containsIgnoringCase("h2_probe");
assertThat(row.getValueLong("lastCalls")).isGreaterThanOrEqualTo(6);
});
assertThat(DbPerfStore.query("db_perf_query_snapshot", new QQueryFilter())).anySatisfy(row -> assertThat(row.getValueLong("calls")).isEqualTo(3));
assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend)))
.anySatisfy(row -> assertThat(row.getValueString("tableName")).isEqualToIgnoringCase("h2_probe"));
assertThat(DbPerfStore.query("db_perf_index_stat", DbPerfStore.eq("backendName", backend)))
.anySatisfy(row -> assertThat(row.getValueString("indexName")).isEqualToIgnoringCase("h2_probe_name"));
Map<String, String> results = new AnalyzeService(runtime).run(backend).stream().collect(Collectors.toMap(AnalyzeService.Result::ruleCode, r -> String.valueOf(r.error())));
assertThat(results).containsKeys("SLOW_QUERY", "DUPLICATE_INDEX", "IDLE_IN_TRANSACTION", "N_PLUS_ONE")
.doesNotContainKeys("UNUSED_INDEX", "MISSING_INDEX", "CACHE_HIT_LOW", "BLOAT", "VACUUM_STALE", "DEADLOCKS", "TEMP_SPILL", "POOL_SIZING");
assertThat(results.values()).containsOnly("null");
assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isNotEmpty();
assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))).allSatisfy(row -> assertThat(row.getValue("seqScanShare")).isNull());
assertThat(new DbPerfWidgetRenderer().tables(new DbPerformanceData(runtime)).getTableDataList().getFirst().getColumns()).extracting(c -> c.getHeader())
.containsExactly("Table", "Rows (estimate)", "Size MB");
assertThat(new DbPerfWidgetRenderer().connectionStatistics(new DbPerformanceData(runtime)).getStatisticsGroupData().getFirst().getStatisticList())
.extracting(statistic -> statistic.getLabel()).contains("Total connections", "Idle in transaction").doesNotContain("Maximum connections", "Cache hit %", "Deadlocks (24h)");
assertThat(new DbPerfWidgetRenderer().setup(runtime, new DbPerformanceData(runtime)).getRows()).extracting(row -> row.get("Check"))
.contains("H2 QUERY_STATISTICS", "H2 admin user").doesNotContain("pg_read_all_stats");
ExplainService explain = new ExplainService(runtime);
ExplainResult plan = explain.explain(new ExplainRequest(backend, null, "select * from h2_probe where name = ?", ExplainRequest.Mode.PLAN, List.of()));
assertThat(plan.mode()).isEqualTo("GENERIC");
assertThat(plan.planJson()).isNull();
assertThat(plan.planText()).containsIgnoringCase("h2_probe_name");
assertThat(plan.assistantPrompt()).contains("database query plan");
assertThatThrownBy(() -> explain.explain(new ExplainRequest(backend, null, "select * from h2_probe where name = $1", ExplainRequest.Mode.ANALYZE, List.of("secret-value"))))
.hasMessageContaining("ANALYZE is not available");
assertThatThrownBy(() -> explain.explain(new ExplainRequest(backend, null, "select file_write('x', '/tmp/p') from h2_probe", ExplainRequest.Mode.PLAN, List.of())))
.hasMessageContaining("FILE_WRITE");
}
/*******************************************************************************
** Without QUERY_STATISTICS the snapshot is partial until the action runs.
*******************************************************************************/
@Test
void enableActionTurnsOnStatistics() throws Exception
{
DbPerfRuntime runtime = runtime("");
String backend = runtime.getConfig().getBackendName();
QRecord partial = new SnapshotCollector(runtime).collect(backend, true);
assertThat(partial.getValueString("status")).isEqualTo("PARTIAL");
assertThat(partial.getValueString("errorMessage")).contains("QUERY_STATISTICS");
assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))).isNotEmpty();
RunBackendStepOutput output = new RunBackendStepOutput();
new DbPerfEnableStatisticsStep().run(new RunBackendStepInput(), output);
assertThat(String.valueOf(output.getValue("processResults"))).contains("FULL");
assertThat(new DbPerfSettingsService(runtime).ensure(backend).getValueString("statsAccess")).isEqualTo("FULL");
workload(runtime);
assertThat(new SnapshotCollector(runtime).collect(backend, false).getValueString("status")).isEqualTo("OK");
}
/*******************************************************************************
** QQQ runtime on a fresh H2 backend with the given URL options.
*******************************************************************************/
private static DbPerfRuntime runtime(String options) throws Exception
{
String name = "dbperfh2" + UUID.randomUUID().toString().replace("-", "");
RDBMSBackendMetaData backend = new RDBMSBackendMetaData().withName(name).withVendor("h2").withHostName("mem").withDatabaseName(name).withUsername("sa");
backend.setJdbcUrl("jdbc:h2:mem:" + name + ";CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1;MODE=PostgreSQL" + options);
return (DbPerfRuntime.of(DbPerfTestSupport.instance(backend)));
}
/*******************************************************************************
** A table with a duplicate-prone index and three identical lookups.
*******************************************************************************/
private static void workload(DbPerfRuntime runtime) throws Exception
{
RDBMSBackendMetaData backend = (RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName());
try(Connection connection = DriverManager.getConnection(backend.getJdbcUrl(), "sa", ""); Statement statement = connection.createStatement())
{
statement.execute("CREATE TABLE IF NOT EXISTS h2_probe (id INT PRIMARY KEY, name VARCHAR(40))");
statement.execute("CREATE INDEX IF NOT EXISTS h2_probe_name ON h2_probe (name)");
statement.execute("MERGE INTO h2_probe KEY (id) SELECT x, 'n' || x FROM SYSTEM_RANGE(1, 20)");
for(int i = 0; i < 3; i++)
{
statement.executeQuery("SELECT * FROM h2_probe WHERE name = 'n7'").close();
}
}
}
}
@@ -0,0 +1,366 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import com.kingsrook.qbits.dbperformance.provider.ConnectionCount;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.H2StatsProvider;
import com.kingsrook.qbits.dbperformance.provider.IndexStats;
import com.kingsrook.qbits.dbperformance.provider.PostgresStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
import com.kingsrook.qbits.dbperformance.provider.StatementStats;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qbits.dbperformance.provider.TableStats;
import com.kingsrook.qbits.dbperformance.provider.UnsupportedStatsProvider;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** H2StatsProvider against real in-memory H2 databases: every read returns
** rows produced by activity in the test, never fixtures.
*******************************************************************************/
class H2StatsProviderTest
{
private final H2StatsProvider provider = new H2StatsProvider();
/*******************************************************************************
** The registry maps JDBC product names and saved names to providers.
*******************************************************************************/
@Test
void registryChoosesByProductName() throws Exception
{
assertThat(DbStatsProviders.forProductName("H2")).isInstanceOf(H2StatsProvider.class);
assertThat(DbStatsProviders.forProductName("PostgreSQL")).isInstanceOf(PostgresStatsProvider.class);
assertThat(DbStatsProviders.forProductName("MySQL")).isInstanceOf(UnsupportedStatsProvider.class);
assertThat(DbStatsProviders.forProductName(null)).isInstanceOf(UnsupportedStatsProvider.class);
assertThat(DbStatsProviders.forProviderName("H2")).isInstanceOf(H2StatsProvider.class);
assertThat(DbStatsProviders.forProviderName("POSTGRES")).isInstanceOf(PostgresStatsProvider.class);
assertThat(DbStatsProviders.forProviderName(null)).isInstanceOf(UnsupportedStatsProvider.class);
try(Connection connection = open(false, ""))
{
assertThat(DbStatsProviders.forConnection(connection)).isInstanceOf(H2StatsProvider.class);
}
assertThat(DbStatsProviders.forProductName("ExampleDB")).isInstanceOf(UnsupportedStatsProvider.class);
DbStatsProviders.register("ExampleDB", H2StatsProvider::new);
assertThat(DbStatsProviders.forProductName("exampledb")).isInstanceOf(H2StatsProvider.class);
assertThat(new UnsupportedStatsProvider().capabilities(null).features()).isEmpty();
}
/*******************************************************************************
** Every read returns real rows after activity, in default and MySQL mode.
*******************************************************************************/
@ParameterizedTest
@ValueSource(strings = {"", ";MODE=MySQL", ";MODE=PostgreSQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE"})
void readsReturnRealRowsAfterActivity(String mode) throws Exception
{
String url = url(true, mode);
try(Connection connection = DriverManager.getConnection(url, "sa", ""))
{
activity(connection);
ProviderCapabilities capabilities = provider.capabilities(connection);
assertThat(capabilities.supported()).isTrue();
assertThat(capabilities.statsAccess()).isEqualTo("FULL");
assertThat(capabilities.statements()).isTrue();
assertThat(capabilities.readAllStats()).isTrue();
assertThat(capabilities.version()).isGreaterThanOrEqualTo(2000000);
assertThat(capabilities.features()).contains(StatsFeature.STATEMENTS, StatsFeature.SESSIONS, StatsFeature.BLOCKING, StatsFeature.TABLE_ROWS, StatsFeature.EXPLAIN_TEXT)
.doesNotContain(StatsFeature.CACHE_COUNTERS, StatsFeature.DATABASE_COUNTERS, StatsFeature.INDEX_USAGE, StatsFeature.TABLE_MAINTENANCE, StatsFeature.MAX_CONNECTIONS, StatsFeature.EXPLAIN_JSON);
assertThat(provider.features()).containsAll(capabilities.features());
assertThat(provider.readDatabase(connection).counters()).isEmpty();
assertThat(provider.readDatabase(connection).maxConnections()).isZero();
assertThat(provider.readStatementResetInfo(connection).statsReset()).isNull();
provider.readTables(connection);
List<StatementStats> statements = provider.readStatements(connection);
StatementStats lookup = statements.stream().filter(s -> s.queryText().contains("WHERE customer_id = 3")).findFirst().orElseThrow();
assertThat(lookup.calls()).isEqualTo(5);
assertThat(lookup.rows()).isEqualTo(5 * 10);
assertThat(lookup.totalExecMs()).isGreaterThanOrEqualTo(lookup.maxExecMs());
assertThat(lookup.dbUser()).isNull();
assertThat(statements).noneMatch(s -> s.queryText().toUpperCase().contains("INFORMATION_SCHEMA") || s.queryText().contains("DB_PERF_PROBE")
|| s.queryText().toUpperCase().startsWith("SET ") || s.queryText().contains("DISK_SPACE_USED") || s.queryText().equalsIgnoreCase("CALL DATABASE()"));
assertThat(provider.readStatements(connection)).anyMatch(s -> s.queryId() == lookup.queryId() && s.queryText().equals(lookup.queryText()));
TableStats orders = provider.readTables(connection).stream().filter(t -> t.tableName().equalsIgnoreCase("orders")).findFirst().orElseThrow();
assertThat(orders.counters()).containsEntry("nLiveTup", 50L).containsKeys("tableBytes", "totalBytes").doesNotContainKeys("seqScan", "nDeadTup");
assertThat(orders.relid()).isPositive();
List<IndexStats> indexes = provider.readIndexes(connection).stream().filter(i -> i.tableName().equalsIgnoreCase("orders")).toList();
assertThat(indexes).anySatisfy(i -> assertThat(i.primary() && i.unique()).isTrue());
assertThat(indexes).anySatisfy(i ->
{
assertThat(i.indexName()).isEqualToIgnoringCase("orders_status");
assertThat(i.unique()).isFalse();
assertThat(i.columns()).isEqualToIgnoringCase("status, created DESC");
assertThat(i.definition()).contains("CREATE INDEX").doesNotContain("USING CONSTRAINT");
});
assertThat(indexes).anySatisfy(i -> assertThat(i.definition()).contains("USING CONSTRAINT"));
try(Connection holder = DriverManager.getConnection(url, "sa", ""); Connection waiter = DriverManager.getConnection(url + ";LOCK_TIMEOUT=10000", "sa", ""))
{
holder.setAutoCommit(false);
try(Statement statement = holder.createStatement())
{
statement.executeUpdate("UPDATE orders SET status = 'HELD' WHERE id = 1");
}
CompletableFuture<Void> blocked = CompletableFuture.runAsync(() -> update(waiter));
List<SessionInfo> sessions = waitForBlocked(connection);
SessionInfo idle = sessions.stream().filter(s -> s.state().equals("idle in transaction")).findFirst().orElseThrow();
assertThat(idle.stateChange()).isNotNull();
assertThat(idle.xactSeconds()).isNotNull();
SessionInfo waiting = sessions.stream().filter(s -> !s.blockedBy().isEmpty()).findFirst().orElseThrow();
assertThat(waiting.state()).isEqualTo("active");
assertThat(waiting.waitEventType()).isEqualTo("Lock");
assertThat(waiting.blockedBy()).containsExactly(idle.pid());
assertThat(waiting.queryText()).contains("UPDATE orders");
assertThat(waiting.queryId()).isNotNull();
assertThat(waiting.querySeconds()).isNotNull();
assertThat(provider.readConnections(connection).stream().mapToInt(ConnectionCount::connectionCount).sum()).isGreaterThanOrEqualTo(3);
assertThat(provider.readConnections(connection)).anyMatch(c -> "idle in transaction".equals(c.state()));
holder.rollback();
blocked.get();
}
Map<String, String> settings = provider.readSettings(connection);
assertThat(settings).containsEntry("QUERY_STATISTICS", "TRUE").containsKey("info.VERSION");
}
}
/*******************************************************************************
** Without QUERY_STATISTICS statements are unavailable until enabled.
*******************************************************************************/
@Test
void capabilitiesWithoutQueryStatisticsAndEnable() throws Exception
{
try(Connection connection = open(false, ""))
{
activity(connection);
ProviderCapabilities off = provider.capabilities(connection);
assertThat(off.supported()).isTrue();
assertThat(off.statements()).isFalse();
assertThat(off.statsAccess()).isEqualTo("NO_QUERY_STATISTICS");
assertThat(off.message()).contains("QUERY_STATISTICS=TRUE");
assertThat(off.has(StatsFeature.STATEMENTS)).isFalse();
assertThat(off.has(StatsFeature.SESSIONS)).isTrue();
assertThat(provider.readStatements(connection)).isEmpty();
assertThat(provider.readTables(connection)).isNotEmpty();
assertThat(provider.readSettings(connection)).containsEntry("QUERY_STATISTICS", "FALSE");
assertThat(provider.enableStatementStatistics(connection)).contains("JDBC URL");
activity(connection);
ProviderCapabilities on = provider.capabilities(connection);
assertThat(on.statsAccess()).isEqualTo("FULL");
assertThat(on.has(StatsFeature.STATEMENTS)).isTrue();
assertThat(provider.readStatements(connection)).anyMatch(s -> s.queryText().contains("WHERE customer_id = 3") && s.calls() == 5);
assertThat(provider.enableStatementStatistics(connection)).contains("already on");
}
}
/*******************************************************************************
** A non-admin user sees only its own session and cannot enable statistics.
*******************************************************************************/
@Test
void nonAdminUser() throws Exception
{
String url = url(true, "");
try(Connection admin = DriverManager.getConnection(url, "sa", ""))
{
try(Statement statement = admin.createStatement())
{
statement.execute("CREATE USER app PASSWORD 'secret'");
statement.execute("CREATE TABLE probe (id INT PRIMARY KEY)");
statement.execute("GRANT SELECT ON probe TO app");
}
try(Connection app = DriverManager.getConnection(url.replace(";DB_CLOSE_DELAY=-1", "").replace(";QUERY_STATISTICS=TRUE", ""), "app", "secret"))
{
ProviderCapabilities capabilities = provider.capabilities(app);
assertThat(capabilities.statsAccess()).isEqualTo("NOT_ADMIN");
assertThat(capabilities.readAllStats()).isFalse();
assertThat(capabilities.statements()).isTrue();
assertThat(capabilities.has(StatsFeature.BLOCKING)).isFalse();
assertThat(provider.readSessions(app)).isEmpty();
assertThat(provider.remedy("NOT_ADMIN", "APP")).contains("ALTER USER APP ADMIN TRUE");
try(Statement statement = admin.createStatement())
{
statement.execute("SET QUERY_STATISTICS FALSE");
}
assertThatThrownBy(() -> provider.enableStatementStatistics(app)).isInstanceOf(SQLException.class);
}
}
}
/*******************************************************************************
** EXPLAIN plans without executing; ANALYZE only for a non-admin SELECT;
** side-effecting functions, aliases, DDL and multiple statements refused.
*******************************************************************************/
@Test
void explainText() throws Exception
{
String url = url(true, ";MODE=MySQL");
try(Connection connection = DriverManager.getConnection(url, "sa", ""))
{
activity(connection);
assertThat(provider.capabilities(connection).has(StatsFeature.EXPLAIN_ANALYZE)).isFalse();
assertThat(provider.capabilities(connection).has(StatsFeature.EXPLAIN_TEXT)).isTrue();
assertThat(provider.explainText(connection, "SELECT * FROM orders WHERE status = 'NEW'", List.of(), false, 5000)).containsIgnoringCase("orders_status").doesNotContain("scanCount");
assertThat(provider.explainText(connection, "SELECT * FROM orders WHERE customer_id = ?1", List.of(), false, 5000)).contains("?1");
assertThat(connection.getAutoCommit()).isTrue();
assertThat(connection.isReadOnly()).isFalse();
assertThatThrownBy(() -> provider.explainText(connection, "SELECT * FROM orders WHERE customer_id = ?1", List.of("3"), true, 5000)).hasMessageContaining("admin");
assertThatThrownBy(() -> provider.explainText(connection, "SELECT FILE_WRITE('x', '/tmp/dbperf-h2-explain-probe') FROM orders", List.of(), false, 5000)).hasMessageContaining("FILE_WRITE");
assertThatThrownBy(() -> provider.explainText(connection, "SELECT \"file_write\"('x', '/tmp/p')", List.of(), false, 5000)).hasMessageContaining("FILE_WRITE");
assertThatThrownBy(() -> provider.explainText(connection, "DROP TABLE orders", List.of(), false, 5000)).hasMessageContaining("DROP");
assertThatThrownBy(() -> provider.explainText(connection, "SELECT 1; SELECT 2", List.of(), false, 5000)).hasMessageContaining("one statement");
assertThat(provider.explainText(connection, "SELECT 'drop; file_write' AS label FROM orders", List.of(), false, 5000)).containsIgnoringCase("orders");
try(Statement statement = connection.createStatement())
{
statement.execute("CREATE ALIAS my_side_effect FOR \"java.lang.System.nanoTime\"");
statement.execute("CREATE USER reader PASSWORD 'secret'");
statement.execute("GRANT SELECT ON orders TO reader");
}
assertThatThrownBy(() -> provider.explainText(connection, "SELECT my_side_effect() FROM orders", List.of(), false, 5000)).hasMessageContaining("MY_SIDE_EFFECT");
try(Connection reader = DriverManager.getConnection(url.replace(";DB_CLOSE_DELAY=-1", "").replace(";QUERY_STATISTICS=TRUE", ""), "reader", "secret"))
{
assertThat(provider.capabilities(reader).has(StatsFeature.EXPLAIN_ANALYZE)).isTrue();
assertThat(provider.explainText(reader, "SELECT * FROM orders WHERE customer_id = ?1", List.of("3"), true, 5000)).contains("scanCount");
assertThatThrownBy(() -> provider.explainText(reader, "VALUES (1)", List.of(), true, 5000)).hasMessageContaining("single SELECT");
}
}
assertThat(java.nio.file.Files.exists(java.nio.file.Path.of("/tmp/dbperf-h2-explain-probe"))).isFalse();
}
/*******************************************************************************
** H2 session states use PostgreSQL's names.
*******************************************************************************/
@Test
void stateMapping() throws Exception
{
assertThat(provider.setupChecks("NO_QUERY_STATISTICS", "m")).extracting(c -> c.check()).containsExactly("H2 QUERY_STATISTICS", "H2 admin user");
assertThat(provider.remedy("FULL", "sa")).isEqualTo("Full statistics access");
assertThat(provider.remedy("UNSUPPORTED", "sa")).contains("H2 2.0");
assertThat(new PostgresStatsProvider().remedy("NO_READ_ALL_STATS", "sa")).isEqualTo("grant pg_read_all_stats to sa");
assertThat(new PostgresStatsProvider().setupChecks("FULL", null)).extracting(c -> c.check()).containsExactly("pg_stat_statements", "pg_read_all_stats");
}
/*******************************************************************************
** Tables, a secondary index, a foreign key and repeated queries.
*******************************************************************************/
private static void activity(Connection connection) throws SQLException
{
try(Statement statement = connection.createStatement())
{
statement.execute("CREATE TABLE IF NOT EXISTS customer (id INT PRIMARY KEY, name VARCHAR(50))");
statement.execute("CREATE TABLE IF NOT EXISTS orders (id INT PRIMARY KEY, customer_id INT REFERENCES customer(id), status VARCHAR(20), created TIMESTAMP)");
statement.execute("CREATE INDEX IF NOT EXISTS orders_status ON orders (status, created DESC)");
statement.execute("MERGE INTO customer KEY (id) SELECT x, 'c' || x FROM SYSTEM_RANGE(1, 5)");
statement.execute("MERGE INTO orders KEY (id) SELECT x, MOD(x, 5) + 1, 'NEW', CURRENT_TIMESTAMP FROM SYSTEM_RANGE(1, 50)");
for(int i = 0; i < 5; i++)
{
statement.executeQuery("SELECT * FROM orders WHERE customer_id = 3").close();
}
}
}
/*******************************************************************************
** Poll until the waiter shows as blocked.
*******************************************************************************/
private List<SessionInfo> waitForBlocked(Connection connection) throws Exception
{
for(int i = 0; i < 100; i++)
{
List<SessionInfo> sessions = provider.readSessions(connection);
if(sessions.stream().anyMatch(s -> !s.blockedBy().isEmpty()))
{
return (sessions);
}
Thread.sleep(50);
}
throw (new AssertionError("no blocked session observed"));
}
/*******************************************************************************
** Blocking update for the waiter thread.
*******************************************************************************/
private static void update(Connection waiter)
{
try(Statement statement = waiter.createStatement())
{
statement.executeUpdate("UPDATE orders SET status = 'WAIT' WHERE id = 1");
}
catch(SQLException e)
{
throw (new IllegalStateException(e));
}
}
/*******************************************************************************
** A fresh in-memory database.
*******************************************************************************/
private static Connection open(boolean statistics, String mode) throws SQLException
{
return (DriverManager.getConnection(url(statistics, mode), "sa", ""));
}
/*******************************************************************************
** URL of a fresh in-memory database.
*******************************************************************************/
private static String url(boolean statistics, String mode)
{
return ("jdbc:h2:mem:h2stats" + UUID.randomUUID().toString().replace("-", "") + ";DB_CLOSE_DELAY=-1" + (statistics ? ";QUERY_STATISTICS=TRUE" : "") + mode);
}
}
@@ -112,6 +112,9 @@ class RollupRetentionTest
}
rows.add(new QRecord().withValue("windowStart", now.minusSeconds(2 * 86400)).withValue("unitName", "boundary"));
DbPerfStore.insert("db_perf_attribution", rows);
DbPerfStore.insert("db_perf_repeat_pattern", List.of(
new QRecord().withValue("windowStart", now.minusSeconds(3 * 86400)),
new QRecord().withValue("windowStart", now.minusSeconds(2 * 86400))));
DbPerfStore.insert("db_perf_attribution_daily", List.of(new QRecord().withValue("day", LocalDate.of(2026, 9, 20)).withValue("rollupKey", "old")));
int rule = DbPerfStore.query("db_perf_rule", new QQueryFilter()).getFirst().getValueInteger("id");
List<QRecord> findings = new ArrayList<>();
@@ -121,8 +124,9 @@ class RollupRetentionTest
}
DbPerfStore.insert("db_perf_finding", findings);
RetentionService retention = new RetentionService(runtime, Clock.fixed(now, ZoneOffset.UTC));
assertThat(retention.purge()).isEqualTo(1008);
assertThat(retention.purge()).isEqualTo(1009);
assertThat(retention.purge()).isZero();
assertThat(DbPerfStore.query("db_perf_repeat_pattern", new QQueryFilter())).hasSize(1);
assertThat(DbPerfStore.query("db_perf_attribution", new QQueryFilter())).singleElement().satisfies(row -> assertThat(row.getValueString("unitName")).isEqualTo("boundary"));
assertThat(DbPerfStore.query("db_perf_finding", new QQueryFilter())).extracting(row -> row.getValueString("status")).containsExactlyInAnyOrder("OPEN", "ACKNOWLEDGED");
}
@@ -28,6 +28,7 @@ import java.util.List;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.provider.UnsupportedStatsProvider;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -123,6 +124,37 @@ class SnapshotCollectorTest
}
/*******************************************************************************
** An Error (not just an Exception) still closes the run as FAILED, and runs
** left RUNNING by an earlier crash are closed once they are stale.
*******************************************************************************/
@Test
void errorsAndAbandonedRuns() throws Exception
{
QInstance instance = DbPerfTestSupport.instance();
DbPerfRuntime runtime = DbPerfRuntime.of(instance);
DbPerfFixtureProvider provider = new DbPerfFixtureProvider();
runtime.asSystem(() -> DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName())
.withValue("capturedAt", DbPerfFixtureProvider.START.minusSeconds(3600)).withValue("status", "RUNNING"))));
provider.error = new StackOverflowError();
QRecord failed = collect(runtime, provider, 0, true);
assertThat(failed.getValueString("status")).isEqualTo("FAILED");
assertThat(failed.getValueString("errorMessage")).contains("StackOverflowError");
runtime.asSystem(() ->
{
List<QRecord> rows = DbPerfStore.query("db_perf_snapshot", new QQueryFilter());
assertThat(rows).hasSize(2).allSatisfy(row -> assertThat(row.getValueString("status")).isEqualTo("FAILED"));
assertThat(rows).anySatisfy(row -> assertThat(row.getValueString("errorMessage")).contains("did not finish"));
return (null);
});
provider.error = null;
assertThat(collect(runtime, provider, 300, true).getValueString("status")).isEqualTo("OK");
}
/*******************************************************************************
** H2 remains usable and explicitly unsupported by the production selector.
*******************************************************************************/
@@ -130,9 +162,17 @@ class SnapshotCollectorTest
void unsupportedVendor() throws Exception
{
DbPerfRuntime runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
QRecord snapshot = new SnapshotCollector(runtime).collect(runtime.getConfig().getBackendName(), false);
QRecord snapshot = new SnapshotCollector(runtime, Clock.systemUTC(), new UnsupportedStatsProvider()).collect(runtime.getConfig().getBackendName(), false);
assertThat(snapshot.getValueString("status")).isEqualTo("PARTIAL");
assertThat(snapshot.getValueString("errorMessage")).contains("not supported");
//////////////////////////////////////////////////////////////////////
// the H2 test backend itself is supported, partially without //
// QUERY_STATISTICS //
//////////////////////////////////////////////////////////////////////
QRecord h2 = new SnapshotCollector(runtime).collect(runtime.getConfig().getBackendName(), false);
assertThat(h2.getValueString("status")).isEqualTo("PARTIAL");
assertThat(h2.getValueString("errorMessage")).contains("QUERY_STATISTICS");
}
@@ -86,6 +86,14 @@ class SqlNormalizerTest
assertThat(SqlNormalizer.mainTable("select 1")).isNull();
assertThat(SqlNormalizer.normalize("select \"in ( ? , ? )\" from t")).isEqualTo("select \"in ( ? , ? )\" from t");
assertThat(SqlNormalizer.normalize(null)).isEmpty();
///////////////////////////////////////////////////////////////////////
// huge lists (production had 60 KB statements) must not overflow //
///////////////////////////////////////////////////////////////////////
String hugeIn = "select * from t where id in (" + "$1, ".repeat(20000) + "$1)";
assertThat(SqlNormalizer.normalize(hugeIn)).isEqualTo("select * from t where id in ( ? )");
String hugeValues = "insert into t (a, b) values " + "($1, $2), ".repeat(20000) + "($1, $2)";
assertThat(SqlNormalizer.normalize(hugeValues)).isEqualTo("insert into t ( a , b ) values ( ? , ? )");
assertThat(SqlNormalizer.normalize("select * from t where id in (1, x)")).isEqualTo("select * from t where id in ( ? , x )");
assertThat(SqlNormalizer.normalize(" ")).isEmpty();
assertThat(SqlNormalizer.normalize("select " + "field, ".repeat(5000))).hasSize(10000);
}
@@ -0,0 +1,135 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.alerts;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.DbPerfTestSupport;
import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Spike detection reads seven days of query snapshots (~230,000 rows in kof22)
** on every evaluation. It must page them and keep only per-statement numbers.
*******************************************************************************/
class SpikeHistoryBoundedTest
{
private static final AtomicInteger MAX_PAGE = new AtomicInteger();
private static final AtomicInteger LOADED = new AtomicInteger();
/*******************************************************************************
** Records the largest page QQQ hands back.
*******************************************************************************/
public static class CountingCustomizer implements TableCustomizerInterface
{
/***************************************************************************
** Count, change nothing.
***************************************************************************/
@Override
public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records)
{
LOADED.addAndGet(records.size());
MAX_PAGE.accumulateAndGet(records.size(), Math::max);
return (records);
}
}
/*******************************************************************************
** 2,500 samples across page boundaries: pages stay at BATCH_SIZE, invalid
** snapshots and the window are honored, the current interval is found.
*******************************************************************************/
@Test
void historyIsPagedAndExact() throws Exception
{
DbPerfRuntime runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
String backend = runtime.getConfig().getBackendName();
runtime.getInstance().getTable("db_perf_query_snapshot").withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CountingCustomizer.class));
Instant now = Instant.parse("2026-10-16T20:00:00Z");
QRecord ok = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now.minusSeconds(3600)).withValue("status", "OK"))).getFirst();
QRecord failed = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now.minusSeconds(1800)).withValue("status", "FAILED"))).getFirst();
QRecord current = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now).withValue("status", "OK"))).getFirst();
int one = statement(backend, 1L);
int two = statement(backend, 2L);
List<QRecord> rows = new ArrayList<>();
for(int i = 0; i < 2500; i++)
{
rows.add(sample(ok, i % 2 == 0 ? one : two, now.minusSeconds(3600 + i), i));
}
rows.add(sample(failed, one, now.minusSeconds(1800), 999999));
rows.add(sample(ok, one, now.minusSeconds(8 * 86400L), 999999));
rows.add(sample(current, one, now, 42));
DbPerfStore.insert("db_perf_query_snapshot", rows);
MAX_PAGE.set(0);
LOADED.set(0);
RuleContext ctx = new RuleContext(runtime, backend, new QRecord(), DbPerfStore.first("db_perf_settings", new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()), now);
Map<Integer, SnapshotEvaluator.StatementHistory> history = SnapshotEvaluator.history(ctx, current.getValueInteger("id"));
assertThat(MAX_PAGE.get()).as("largest page of query snapshots held at once").isLessThanOrEqualTo(DbPerfStore.BATCH_SIZE);
assertThat(history.get(one).sortedPrevious()).hasSize(1250).doesNotContain(999999.0);
assertThat(history.get(two).sortedPrevious()).hasSize(1250);
assertThat(history.get(one).sortedPrevious()[0]).isEqualTo(0.0);
assertThat(history.get(one)).extracting("current").isEqualTo(42.0);
assertThat(history.get(two)).extracting("current").isNull();
}
/*******************************************************************************
** One statement identity; returns its id.
*******************************************************************************/
private static int statement(String backend, long queryId) throws Exception
{
return (DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", backend).withValue("queryId", queryId)
.withValue("queryText", "select " + queryId).withValue("fingerprint", String.valueOf(queryId).repeat(64).substring(0, 64)))).getFirst().getValueInteger("id"));
}
/*******************************************************************************
** One query snapshot row.
*******************************************************************************/
private static QRecord sample(QRecord snapshot, int statementId, Instant at, int totalMs)
{
return (new QRecord().withValue("snapshotId", snapshot.getValueInteger("id")).withValue("statementId", statementId)
.withValue("capturedAt", at).withValue("totalMs", BigDecimal.valueOf(totalMs)));
}
}
@@ -0,0 +1,664 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.attribution;
import java.math.BigDecimal;
import java.sql.DriverManager;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.DbPerfTestSupport;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.RollupService;
import com.kingsrook.qbits.dbperformance.metadata.DbPerfWriteGuard;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instrumentation.StatementExecution;
import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListener;
import com.kingsrook.qqq.backend.core.instrumentation.StatementExecutionListeners;
import com.kingsrook.qqq.backend.core.instrumentation.StatementOperation;
import com.kingsrook.qqq.backend.core.instrumentation.UnitOfWork;
import com.kingsrook.qqq.backend.core.instrumentation.UnitOfWorkKind;
import com.kingsrook.qqq.backend.core.instrumentation.UnitsOfWork;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Real QQQ RDBMS hooks, persistence, bounded counters, and lifecycle contracts.
*******************************************************************************/
class AttributionTest
{
private DbPerfRuntime runtime;
private AttributionFlusher flusher;
private AttributionListener listener;
private QRecord settings;
/*******************************************************************************
** A real business table, separate from the self-suppressed db_perf tables.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
QInstance instance = DbPerfTestSupport.instance();
runtime = DbPerfRuntime.of(instance);
RDBMSBackendMetaData backend = (RDBMSBackendMetaData) instance.getBackend(runtime.getConfig().getBackendName());
instance.addTable(new QTableMetaData().withName("business").withBackendName(backend.getName()).withPrimaryKeyField("id")
.withBackendDetails(new RDBMSTableBackendDetails().withTableName("business"))
.withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("name", QFieldType.STRING)));
try(var connection = DriverManager.getConnection(backend.getJdbcUrl(), backend.getUsername(), backend.getPassword()); var statement = connection.createStatement())
{
statement.execute("CREATE TABLE business (id INT PRIMARY KEY, name VARCHAR(100))");
statement.execute("INSERT INTO business VALUES (1, 'example')");
}
flusher = new AttributionFlusher(runtime);
flusher.refreshSettings();
listener = flusher.getListener();
settings = DbPerfStore.query("db_perf_settings", new QQueryFilter()).getFirst();
StatementExecutionListeners.register(listener);
}
/*******************************************************************************
** No listener, scope, suppression or runtime may escape a test.
*******************************************************************************/
@AfterEach
void tearDown()
{
StatementExecutionListeners.unregister(listener);
flusher.close();
runtime.stopAttribution();
UnitsOfWork.reset();
AttributionListener.suppress(false);
QContext.clear();
}
/*******************************************************************************
** The core hook attributes real queries and reads final HTTP route names.
*******************************************************************************/
@Test
void realQueriesAndFinalNames() throws Exception
{
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.SCHEDULED_JOB, "job"))
{
query(2);
}
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.HTTP_REQUEST, "unresolved"))
{
query(1);
scope.getUnitOfWork().setName("GET /things/{id}");
}
query(1);
flusher.flushNow();
List<QRecord> rows = rows("db_perf_attribution");
assertThat(rows).hasSize(3);
assertThat(rows).extracting(row -> row.getValueString("unitName")).containsExactlyInAnyOrder("job", "GET /things/{id}", "(none)");
QRecord job = named(rows, "job");
assertThat(job.getValueString("unitKind")).isEqualTo("SCHEDULED_JOB");
assertThat(job.getValueLong("units")).isEqualTo(1);
assertThat(job.getValueLong("statements")).isEqualTo(2);
assertThat(job.getValueLong("rowsCount")).isEqualTo(2);
assertThat(job.getValueString("operation")).isEqualTo("QUERY");
assertThat(job.getValueString("tableName")).isEqualTo("business");
assertThat(job.getValueString("fingerprint")).isEqualTo(SqlNormalizer.fingerprint(job.getValueString("sqlText")));
assertThat(named(rows, "(none)").getValueString("unitKind")).isEqualTo("OTHER");
}
/*******************************************************************************
** The threshold applies to each unit, and each window uses one batch/table.
*******************************************************************************/
@Test
void repeatPatternsBatchesAndSelfSuppression() throws Exception
{
AtomicInteger attributionInserts = new AtomicInteger();
AtomicInteger repeatInserts = new AtomicInteger();
StatementExecutionListener observer = execution ->
{
if(execution.operation() == StatementOperation.INSERT)
{
if("db_perf_attribution".equals(execution.tableName()))
{
attributionInserts.incrementAndGet();
}
if("db_perf_repeat_pattern".equals(execution.tableName()))
{
repeatInserts.incrementAndGet();
}
}
};
StatementExecutionListeners.register(observer);
try
{
for(int count : List.of(12, 5))
{
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.SCHEDULED_JOB, "job"))
{
query(count);
}
}
rows("db_perf_settings");
flusher.flushNow();
QRecord repeated = rows("db_perf_repeat_pattern").getFirst();
assertThat(repeated.getValueLong("unitsAffected")).isEqualTo(1);
assertThat(repeated.getValueInteger("maxRepeats")).isEqualTo(12);
assertThat(repeated.getValueLong("totalRepeats")).isEqualTo(12);
assertThat(repeated.getValueBigDecimal("sampleRate")).isEqualByComparingTo("1");
assertThat(rows("db_perf_repeat_pattern")).hasSize(1);
assertThat(rows("db_perf_attribution")).singleElement().satisfies(row ->
{
assertThat(row.getValueLong("units")).isEqualTo(2);
assertThat(row.getValueLong("statements")).isEqualTo(17);
});
flusher.flushNow();
assertThat(attributionInserts).hasValue(1);
assertThat(repeatInserts).hasValue(1);
assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
}
finally
{
StatementExecutionListeners.unregister(observer);
}
}
/*******************************************************************************
** Both runtime switches and sample zero take effect through cached settings.
*******************************************************************************/
@Test
void disabledSamplingAndFilters() throws Exception
{
settings.withValue("attributionSampleRate", BigDecimal.ZERO);
refresh();
query(1);
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "zero"))
{
query(1);
}
assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
settings.withValue("attributionSampleRate", BigDecimal.ONE).withValue("attributionEnabled", false);
refresh();
query(1);
assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
settings.withValue("attributionEnabled", true);
refresh();
runtime.getConfig().setAttributionEnabled(false);
query(1);
runtime.getConfig().setAttributionEnabled(true);
listener.onStatement(new StatementExecution("wrong", "business", StatementOperation.QUERY, "select ?", 1, 1, true, null, null));
AttributionListener.suppress(true);
query(1);
AttributionListener.suppress(false);
rows("db_perf_settings");
assertThat(listener.drain(Instant.now()).attribution()).isEmpty();
query(1);
assertThat(listener.drain(Instant.now()).attribution()).hasSize(1);
}
/*******************************************************************************
** The cap includes the overflow bucket; a unit counts only once in it.
*******************************************************************************/
@Test
void overflowAndRepeatBounds() throws Exception
{
settings.withValue("attributionMaxKeys", 2).withValue("nPlusOneThreshold", 2);
refresh();
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "many-shapes"))
{
for(int i = 0; i < 8; i++)
{
synthetic("select column_" + i + " from business", 1, true, scope.getUnitOfWork());
}
}
for(int i = 0; i < 8; i++)
{
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "job_" + i))
{
synthetic("select column_" + i + " from business", 1, true, scope.getUnitOfWork());
synthetic("select column_" + i + " from business", 1, true, scope.getUnitOfWork());
}
}
flusher.flushNow();
assertThat(rows("db_perf_attribution")).hasSize(2);
assertThat(named(rows("db_perf_attribution"), "(other)").getValueLong("units")).isEqualTo(9);
assertThat(rows("db_perf_attribution").stream().mapToLong(row -> row.getValueLong("statements")).sum()).isEqualTo(24);
assertThat(rows("db_perf_repeat_pattern")).hasSize(2);
assertThat(named(rows("db_perf_repeat_pattern"), "(other)").getValueString("fingerprint")).isEqualTo(SqlNormalizer.fingerprint("(other)"));
}
/*******************************************************************************
** Counters, normalization, null rows, process fallback, and text bounds.
*******************************************************************************/
@Test
void syntheticCountersAndSqlSafety()
{
String sql = "select 'secret', 123 from business where id in (?, ?, ?)";
synthetic(sql, 2000000, true, null);
synthetic(sql, 5000000, false, null);
listener.onStatement(new StatementExecution(runtime.getConfig().getBackendName(), null, null, null, 10, null, true, null, "process"));
flusher.flushNow();
List<QRecord> rows = rows("db_perf_attribution");
QRecord row = named(rows, "(none)");
assertThat(row.getValueBigDecimal("totalMs")).isEqualByComparingTo("7");
assertThat(row.getValueBigDecimal("maxMs")).isEqualByComparingTo("5");
assertThat(row.getValueLong("errors")).isEqualTo(1);
assertThat(row.getValueLong("rowsCount")).isEqualTo(2);
assertThat(row.getValueString("sqlText")).doesNotContain("secret", "123");
assertThat(named(rows, "process").getValueString("unitKind")).isEqualTo("OTHER");
synthetic("select " + "identifier".repeat(600) + " from business", 1, true, null);
flusher.flushNow();
assertThat(rows("db_perf_attribution")).anySatisfy(value -> assertThat(value.getValueString("sqlText")).hasSize(4000));
}
/*******************************************************************************
** An abandoned unit is merged when another unit arrives on that worker.
*******************************************************************************/
@Test
void defensiveUnitTransitionAndWindowBoundary() throws Exception
{
var abandoned = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "abandoned");
query(1);
UnitsOfWork.reset();
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.HTTP_REQUEST, "initial"))
{
query(1);
flusher.flushNow();
assertThat(rows("db_perf_attribution")).singleElement().satisfies(row -> assertThat(row.getValueString("unitName")).isEqualTo("abandoned"));
scope.getUnitOfWork().setName("final");
query(1);
}
abandoned.close();
flusher.flushNow();
assertThat(named(rows("db_perf_attribution"), "final").getValueLong("statements")).isEqualTo(2);
}
/*******************************************************************************
** Each sampled unit contributes all statements, including across a refresh.
*******************************************************************************/
@Test
void samplingIsPerUnit() throws Exception
{
settings.withValue("attributionSampleRate", new BigDecimal("0.5"));
refresh();
for(int i = 0; i < 100; i++)
{
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "sample_" + i))
{
for(int j = 0; j < 3; j++)
{
synthetic("select ?", 1, true, scope.getUnitOfWork());
}
}
}
var window = listener.drain(Instant.now());
assertThat(window.attribution().size()).isBetween(20, 80);
assertThat(window.attribution().values()).allSatisfy(counts -> assertThat(counts.statements.sum()).isEqualTo(3));
}
/*******************************************************************************
** An in-flight unit keeps its sampling decision and original rate annotation.
*******************************************************************************/
@Test
void samplingRefreshDuringUnit() throws Exception
{
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "in-flight"))
{
query(1);
settings.withValue("attributionSampleRate", BigDecimal.ZERO);
refresh();
query(1);
}
query(1);
flusher.flushNow();
assertThat(rows("db_perf_attribution")).singleElement().satisfies(row ->
{
assertThat(row.getValueLong("statements")).isEqualTo(2);
assertThat(row.getValueBigDecimal("sampleRate")).isEqualByComparingTo("1");
});
}
/*******************************************************************************
** Atomic drains cannot lose or double-count concurrent statement delivery.
*******************************************************************************/
@Test
void concurrentDrains() throws Exception
{
var pool = Executors.newFixedThreadPool(4);
try
{
List<java.util.concurrent.Future<?>> tasks = new ArrayList<>();
for(int i = 0; i < 4; i++)
{
tasks.add(pool.submit(() ->
{
for(int j = 0; j < 5000; j++)
{
synthetic("select ?", 1, true, null);
}
}));
}
long total = 0;
while(tasks.stream().anyMatch(task -> !task.isDone()))
{
total += listener.drain(Instant.now()).attribution().values().stream().mapToLong(counts -> counts.statements.sum()).sum();
Thread.yield();
}
for(var task : tasks)
{
task.get();
}
total += listener.drain(Instant.now()).attribution().values().stream().mapToLong(counts -> counts.statements.sum()).sum();
assertThat(total).isEqualTo(20000);
}
finally
{
pool.shutdownNow();
}
}
/*******************************************************************************
** Cached hot-path overhead stays below a generous two-second budget.
*******************************************************************************/
@Test
void overhead()
{
StatementExecution execution = new StatementExecution(runtime.getConfig().getBackendName(), "business", StatementOperation.QUERY, "select ?", 1, 1, true, null, null);
listener.onStatement(execution);
long start = System.nanoTime();
for(int i = 0; i < 100000; i++)
{
listener.onStatement(execution);
}
assertThat(System.nanoTime() - start).isLessThan(TimeUnit.SECONDS.toNanos(2));
assertThat(listener.drain(Instant.now()).attribution().values().iterator().next().statements.sum()).isEqualTo(100001);
}
/*******************************************************************************
** Launcher lifecycle registers once, flushes on stop, and respects config.
*******************************************************************************/
@Test
void runtimeLifecycle() throws Exception
{
flusher.close();
assertThat(StatementExecutionListeners.isActive()).isFalse();
AttributionRuntimeService service = new AttributionRuntimeService();
assertThat(service.getName()).isEqualTo("dbPerformanceAttribution");
service.stop();
service.start(runtime.getInstance());
runtime.startAttribution();
assertThat(StatementExecutionListeners.isActive()).isTrue();
query(1);
service.stop();
service.stop();
assertThat(StatementExecutionListeners.isActive()).isFalse();
assertThat(rows("db_perf_attribution")).hasSize(1);
runtime.getConfig().setAttributionEnabled(false);
service.start(runtime.getInstance());
assertThat(StatementExecutionListeners.isActive()).isFalse();
service.stop();
}
/*******************************************************************************
** Real flusher rows roll up across pods, preserving max and additive totals.
*******************************************************************************/
@Test
void rollupMergesPods() throws Exception
{
for(int i = 1; i <= 2; i++)
{
runtime.getConfig().setPodName("pod_" + i);
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "job"))
{
synthetic("select ?", i * 1000000L, true, scope.getUnitOfWork());
synthetic("select ?", i * 1000000L, false, scope.getUnitOfWork());
}
flusher.flushNow();
}
new RollupService(runtime).rollupDay(LocalDate.now(ZoneOffset.UTC));
assertThat(rows("db_perf_attribution_daily")).singleElement().satisfies(row ->
{
assertThat(row.getValueLong("units")).isEqualTo(2);
assertThat(row.getValueLong("statements")).isEqualTo(4);
assertThat(row.getValueBigDecimal("totalMs")).isEqualByComparingTo("6");
assertThat(row.getValueBigDecimal("maxMs")).isEqualByComparingTo("2");
assertThat(row.getValueLong("rowsCount")).isEqualTo(4);
assertThat(row.getValueLong("errors")).isEqualTo(2);
});
}
/*******************************************************************************
** Large windows use one QQQ insert action even when the backend pages SQL.
*******************************************************************************/
@Test
void largeWindowUsesOneInsertAction() throws Exception
{
CountingInsert.inputs.clear();
for(String table : List.of("db_perf_attribution", "db_perf_repeat_pattern"))
{
runtime.getInstance().getTable(table).getCustomizers().put(TableCustomizers.PRE_INSERT_RECORD.getRole(), new QCodeReference(CountingInsert.class));
}
settings.withValue("nPlusOneThreshold", 2);
refresh();
for(int i = 0; i < 1005; i++)
{
try(var scope = UnitsOfWork.begin(UnitOfWorkKind.PROCESS, "job_" + i))
{
synthetic("select ?", 1, true, scope.getUnitOfWork());
synthetic("select ?", 1, true, scope.getUnitOfWork());
}
}
flusher.flushNow();
assertThat(rows("db_perf_attribution")).hasSize(1005);
assertThat(rows("db_perf_repeat_pattern")).hasSize(1005);
assertThat(CountingInsert.inputs).extracting(InsertInput::getTableName).containsExactlyInAnyOrder("db_perf_attribution", "db_perf_repeat_pattern");
assertThat(CountingInsert.inputs).allSatisfy(input -> assertThat(input.getRecords()).hasSize(1005));
}
/*******************************************************************************
** An insert failure drops the window and never poisons a later good window.
*******************************************************************************/
@Test
void failedWindowIsDropped() throws Exception
{
RDBMSTableBackendDetails details = (RDBMSTableBackendDetails) runtime.getInstance().getTable("db_perf_attribution").getBackendDetails();
query(1);
details.setTableName("missing_attribution_table");
flusher.flushNow();
details.setTableName("db_perf_attribution");
flusher.flushNow();
assertThat(rows("db_perf_attribution")).isEmpty();
query(1);
flusher.flushNow();
assertThat(rows("db_perf_attribution")).singleElement().satisfies(row -> assertThat(row.getValueLong("statements")).isEqualTo(1));
}
/*******************************************************************************
** Cold-cache overflow remains bounded and later hits keep the right shape.
*******************************************************************************/
@Test
void normalizationCacheOverflow()
{
for(int i = 0; i < 5010; i++)
{
synthetic("select column_" + i + " from business", 1, true, null);
}
synthetic("select column_0 from business", 1, true, null);
var window = listener.drain(Instant.now());
assertThat(window.attribution()).hasSize(5010);
String fingerprint = SqlNormalizer.fingerprint(SqlNormalizer.normalize("select column_0 from business"));
assertThat(window.attribution().entrySet().stream().filter(entry -> entry.getKey().fingerprint().equals(fingerprint)).findFirst().orElseThrow().getValue().statements.sum()).isEqualTo(2);
}
/*******************************************************************************
** The daemon executes on its own system context and honors a short interval.
*******************************************************************************/
@Test
void scheduledFlush() throws Exception
{
settings.withValue("attributionFlushSeconds", 1);
refresh();
flusher.start();
flusher.start();
query(1);
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
while(rows("db_perf_attribution").isEmpty() && System.nanoTime() < deadline)
{
Thread.sleep(25);
}
assertThat(rows("db_perf_attribution")).hasSize(1);
assertThat(QContext.getQInstance()).isSameAs(runtime.getInstance());
flusher.close();
flusher.start();
assertThat(StatementExecutionListeners.isActive()).isFalse();
}
/*******************************************************************************
** Only one runtime attributes per JVM: starting another stops the first, so
** repeated server starts (tests, hot swaps) never pile up listeners.
*******************************************************************************/
@Test
void oneActiveRuntime() throws Exception
{
StatementExecutionListeners.unregister(listener);
runtime.startAttribution();
DbPerfRuntime other = new DbPerfRuntime(runtime.getInstance(), runtime.getConfig());
other.startAttribution();
runtime.stopAttribution();
assertThat(StatementExecutionListeners.isActive()).isTrue();
other.stopAttribution();
assertThat(StatementExecutionListeners.isActive()).isFalse();
}
/*******************************************************************************
** Deliver a statement with exact counter values.
*******************************************************************************/
private void synthetic(String sql, long nanos, boolean succeeded, UnitOfWork unit)
{
listener.onStatement(new StatementExecution(runtime.getConfig().getBackendName(), "business", StatementOperation.QUERY, sql, nanos, 1, succeeded, unit, null));
}
/*******************************************************************************
** Execute real RDBMS-backed QQQ queries.
*******************************************************************************/
private void query(int count) throws Exception
{
for(int i = 0; i < count; i++)
{
new QueryAction().execute(new QueryInput("business").withFilter(new QQueryFilter()));
}
}
/*******************************************************************************
** Persist administrator settings, then explicitly refresh the cached copy.
*******************************************************************************/
private void refresh() throws Exception
{
DbPerfStore.update("db_perf_settings", List.of(settings));
flusher.refreshSettings();
}
/*******************************************************************************
** Read persisted evidence with a concise test failure on action errors.
*******************************************************************************/
private List<QRecord> rows(String table)
{
try
{
return (DbPerfStore.query(table, new QQueryFilter()));
}
catch(Exception e)
{
throw (new AssertionError(e));
}
}
/*******************************************************************************
** Find one named aggregate.
*******************************************************************************/
private static QRecord named(List<QRecord> rows, String name)
{
return (rows.stream().filter(row -> name.equals(row.getValueString("unitName"))).findFirst().orElseThrow());
}
/*******************************************************************************
** Observe whole QQQ actions while preserving the production write guard.
*******************************************************************************/
public static class CountingInsert extends DbPerfWriteGuard
{
private static final List<InsertInput> inputs = new ArrayList<>();
/*******************************************************************************
** Capture each action's original input before the backend pages its SQL.
*******************************************************************************/
@Override
public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
{
inputs.add(input);
return (super.preInsert(input, records, preview));
}
}
}