Compare commits

..
Author SHA1 Message Date
james.maes 7c5e7a9320 Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#39) from deps/align-gitea into develop
build / maven (push) Successful in 5m20s
maven / build (push) Successful in 5m20s
2026-11-07 18:37:05 +00:00
james.maesandClaude Opus 5.5 f0ecacc3e9 test: regenerate OpenAPI contract for QQQ 4.1.0-RC.26
build / maven (push) Successful in 4m8s
maven / build (push) Successful in 4m8s
QQQ RC.26 adds the shared BasicErrorResponseV1 schema to extension models; regenerate the committed fragment and allow that core schema.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 12:30:25 -06:00
james.maesandClaude Opus 5.5 725446709b build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)
build / maven (push) Canceled after 0s
maven / build (push) Canceled after 0s
pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.13 -> 4.1.0-RC.26
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 17:59:16 +00:00
james.maes d578b5180a Merge pull request 'ci: pin shared workflow to v1' (#38) from ci/pin-v1-develop into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (push) Successful in 16s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 12s
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Successful in 12s
build / maven (push) Successful in 4m53s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 4m53s
2026-11-06 16:06:22 +00:00
james.maesandClaude Opus 5.5 cf106d4301 ci: pin shared workflow to v1
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 7s
build / maven (push) Successful in 5m7s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 5m7s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 14:08:36 +00:00
james.maes a26a175aa8 Merge pull request 'ci: use shared ci-workflows Maven workflow' (#37) from ci/ci-workflows into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 7s
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Successful in 5s
build / maven (push) Successful in 3m50s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow d20a578c7d818bc7ca32a2ac2c675da4e4f27385] (push) Successful in 10s
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 3m50s
2026-11-05 23:16:16 +00:00
CI aff84d5add ci: rerun shared workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Failing after 3s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 2s
build / maven (push) Successful in 3m43s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Failing after 2s
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 3m44s
2026-11-05 16:24:46 -06:00
james.maesandClaude Opus 5.5 171b0d3fad ci: use shared ci-workflows Maven workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Successful in 8s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
build / maven (push) Failing after 3m54s
QRun-IO/ci-policy: ci / ci-result (pull_request) Canceled after 0s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Failing after 3m55s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 3m42s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Failing after 1s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Canceled after 12m41s
Replace the platform-ci release callers with one caller of
KofTwentyTwo/ci-workflows maven.yaml (qqq-orb port).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:04:26 +00:00
forge-recovery b70c234c79 ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / ci-result (push) Successful in 4s
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 25m25s
qrun-release / postpublish (push) Failing after 10m34s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 14s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 13s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 6s
Shared Java Maven CI with Docker / verify (push) Successful in 3m56s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 22s
Shared Java Maven CI with Docker / gate (push) Successful in 20s
qrun-release / release (push) Successful in 25m48s
Shared QRun source version / version (push) Failing after 10m34s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 3m18s
2026-11-02 22:15:48 +00:00
forge-recovery ef58b9d8ad ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 40s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 12s
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 11m7s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 7s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 3m30s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 20s
2026-11-02 21:33:08 +00:00
forge-recoveryandCodex 5d82358d2b ci: enroll shared Gitea release-candidate lifecycle
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 4s
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 12s
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 2m49s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 17s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 59m32s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 20:16:11 +00:00
3468453335 ci: add the isolated postpublish job to the QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 11s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
Shared Java Maven CI with Docker / verify (push) Successful in 3m2s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
qrun-release / postpublish (push) Skipped
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 11s
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 2m22s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 9s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 2h51m11s
Shared Java Maven CI with Docker / gate (push) Successful in 2m25s
qrun-release / release (push) Successful in 2h52m39s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 15:48:51 +00:00
612f474e91 ci: move the QRun release caller to the thin v2 contract
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-9 (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 5s
Shared Java Maven CI with Docker / secrets (push) Successful in 22s
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Canceled after 1h7m43s
Shared Java Maven CI with Docker / verify (push) Canceled after 0s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 21s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Canceled after 0s
Shared Java Maven CI with Docker / gate (push) Canceled after 0s
qrun-release / release (push) Canceled after 1h7m51s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 06:17:47 +00:00
forge-recoveryandCodex f005c40083 ci: add the identical QRun release caller
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Successful in 52m15s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 3s
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-7 (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 2m39s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 10s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 16s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 04:10:32 +00:00
james.maesandClaude Fable 5.1 d46b6ca222 feat(provider): H2 statistics provider and per-area capabilities (#34)
QRun-IO/ci-policy: ci / gradle-0 (push) Skipped
QRun-IO/ci-policy: ci / node-3 (push) Skipped
QBit CodeQL / Analyze branch source (push) Canceled after 0s
CodeQL / analyze (push) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 11s
QRun-IO/ci-policy: ci / java-docker-1 (push) Successful in 40m46s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 4s
QRun-IO/ci-policy: ci / source-config-5 (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / java-maven-2 (push) Skipped
QRun-IO/ci-policy: ci / node-4 (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 3m47s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 9s
H2StatsProvider over INFORMATION_SCHEMA (statements via QUERY_STATISTICS, sessions, blocking, connections, indexes, size, settings, approximated table rows and sizes, text plans); per-area provider capabilities with rules skipped where a provider lacks the data; a vendor registry with a seam for MySQL/MariaDB; provider-specific setup and health checks; an Enable statement statistics action; and the H2 plan guard (single SELECT/WITH only, side-effecting functions refused, EXPLAIN ANALYZE only for non-admin users, read-only connection, always rolled back).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 06:14:44 -06:00
james.maesandCodex 4fd2c30b07 ci: adopt managed orb 0.8 patch track (#28)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 15:26:36 -05:00
james.maesandCodex 65e63551f9 ci: adopt shared orb 0.7.18 (#27)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:43:20 -05:00
james.maesandCodex ccc0cc2842 ci: adopt shared orb 0.7.17 (#26)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:15:18 -05:00
james.maesandCodex 9178646423 ci: centralize branch verification and source scanning (#23)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 12:17:56 -05:00
james.maesandCodex 681598be57 ci: adopt central orb 0.7.12 and stable gates (#21)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 08:56:36 -05:00
james.maesandCodex ea9036509c ci: centralize rc release completion in qqq orb (#20)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 02:25:42 -05:00
james.maesandCodex 2a2ba58b06 ci: adopt shared orb 0.7.11 (#19)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 00:57:50 -05:00
james.maes 61e9c22563 Merge pull request #18 from QRun-IO/feature/central-ci
ci: adopt shared QQQ orb standard
2026-10-20 18:57:48 -05:00
james.maesandCodex 65922dd413 ci: adopt verified central QQQ orb 0.7.3
Co-Authored-By: Codex <noreply@openai.com>
2026-10-20 18:52:28 -05:00
james.maes bdaf35fc3e Merge pull request #17 from QRun-IO/feature/bounded-spike-history
fix(alerts): page spike history instead of holding 7 days of query snapshots
2026-10-16 15:54:56 -05:00
james.maesandClaude Opus 5.5 c49b479620 fix(alerts): page spike history instead of holding 7 days of query snapshots
SnapshotEvaluator.spikes loaded RuleContext.window("query_snapshot", 7)
as QRecords on every snapshot evaluation (every collection interval on
the worker). kof22 writes ~33,000 query snapshots a day, so by day 7 that
is ~230,000 records per evaluation.

history() now reads the window in BATCH_SIZE keyset pages (only OK and
PARTIAL snapshots, as before) and keeps per statement only the current
interval time, the latest capture time and earlier interval times in a
primitive double array. Median, twelve-sample baseline and the one-hour
absence rule are unchanged (SnapshotEvaluatorTest still passes).

SpikeHistoryBoundedTest counts every query snapshot QQQ materializes:
2,500 samples never exceed one 1,000-row page, failed snapshots and
samples outside the window are excluded, the current interval is found.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 15:24:44 -05:00
james.maes de991509ca Merge pull request #16 from QRun-IO/feature/bounded-attribution-reads
fix(attribution): sum attribution in the database, page the daily rollup (prod OOM)
2026-10-16 15:15:33 -05:00
42 changed files with 2788 additions and 251 deletions

No files matched your search

+11 -57
View File
@@ -1,60 +1,14 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
qqq-orb: kingsrook/qqq-orb@0.8
workflows:
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
+7
View File
@@ -0,0 +1,7 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
+23
View File
@@ -0,0 +1,23 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+62 -14
View File
@@ -1,26 +1,74 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
on:
'on':
workflow_dispatch:
inputs:
dry_run:
description: Print the plan without merging or creating branches
description: Plan a cut without changing release state
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types: [closed]
branches: [develop]
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
secrets: inherit
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
+16
View File
@@ -0,0 +1,16 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
+11
View File
@@ -4,7 +4,18 @@ All notable changes are documented here. This project follows Keep a Changelog a
## [Unreleased]
### Added
- H2 2.x support (`H2StatsProvider`) from `INFORMATION_SCHEMA`: statement statistics from `QUERY_STATISTICS` (when the database runs with `QUERY_STATISTICS=TRUE`), sessions with blocking and idle-in-transaction detection, connection counts, table row estimates and sizes, index definitions, settings, and text query plans (`EXPLAIN`; `EXPLAIN ANALYZE` only for non-admin H2 users, since H2 has no read-only transaction: plans refuse side-effecting functions such as `FILE_WRITE`, DDL/DML keywords, user-defined aliases and multiple statements). The enterprise test server showed UNSUPPORTED on H2; it now collects.
- Per-area capabilities (`StatsFeature`): providers declare which statistics areas they can supply and `ProviderCapabilities.features()` reports what is available on a connection. Snapshot cache-hit ratio and connection limit stay empty when the vendor has none, and analysis rules declare the areas they need (`DbPerfRule.requires()`) and are skipped on providers that cannot supply them, so H2 never gets "unused index" or "low cache hit" findings from missing counters.
- Provider registry keyed by JDBC product name (`DbStatsProviders.register`), with `docs/PROVIDERS.md` describing what MySQL/MariaDB and SQLite would map to.
- **Enable statement statistics** action on Settings (`dbPerfEnableStatementStatistics`, manage settings): on H2 runs `SET QUERY_STATISTICS_MAX_ENTRIES 1000` and `SET QUERY_STATISTICS TRUE` as an admin; on PostgreSQL explains the manual extension setup. It is the qbit's only change to the monitored database and is explicit.
### Changed
- Health and the Setup panel take their prerequisite checks and remedies from the backend's provider: PostgreSQL still names `pg_stat_statements` and `pg_read_all_stats`; H2 names `QUERY_STATISTICS` and the admin role (new access codes `NO_QUERY_STATISTICS` and `NOT_ADMIN`).
- The previous-snapshot baseline no longer depends on a PostgreSQL-only counter (it uses the database size every supported provider records).
### Fixed
- Query-spike alerts no longer hold seven days of query snapshots as records on every snapshot evaluation (~230,000 rows at kof22's rate). `SnapshotEvaluator` reads the window in 1,000-row keyset pages and keeps only each statement's current interval time, latest capture time and earlier interval times as primitive doubles; the spike rule (twelve prior samples, median, absence after an hour) is unchanged.
- The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. The hourly analysis (`dbPerfAnalyze`) reads attribution the same way: `RuleContext.window("attribution", ...)` returns database sums per unit, table and operation instead of every detail row (it filled the kof22 worker heap to ~1.8 GB every hour).
- Collection no longer stops at RUNNING on large production statement sets: list folding in the SQL normalizer is a linear token scan (the nested-repetition regular expressions overflowed the stack on 60 KB statements with long IN/VALUES lists). Any failure, including an `Error`, now closes the run as FAILED with the exception class, logs it at ERROR (class and frame only, never SQL text) and raises the collection alert; runs left RUNNING by a crash are closed as FAILED after three intervals.
- Only one runtime attributes per JVM: starting attribution on a newer instance (a hot swap, or repeated server starts in tests) stops the previous one, so old instances are never pinned by a forgotten listener and flusher.
+15 -2
View File
@@ -1,6 +1,6 @@
# QBit DB Performance
Database performance for QQQ apps: PostgreSQL statement deltas, table and index statistics, live sessions and blocking, connection capacity, audited query plans, and typed home dashboards in standard secured QQQ tables. Approved design: [docs/SPEC.md](docs/SPEC.md). This branch implements collection and the presentation/plan-viewer phase; attribution listeners, findings evaluation, alerts and health are developed in a separate phase. No release has been published yet.
Database performance for QQQ apps: PostgreSQL and H2 statement deltas, table and index statistics, live sessions and blocking, connection capacity, audited query plans, and typed home dashboards in standard secured QQQ tables. Approved design: [docs/SPEC.md](docs/SPEC.md). This branch implements collection and the presentation/plan-viewer phase; attribution listeners, findings evaluation, alerts and health are developed in a separate phase. No release has been published yet.
## Requirements
@@ -8,6 +8,7 @@ Database performance for QQQ apps: PostgreSQL statement deltas, table and index
- PostgreSQL **13+**; **16+** for parameterized generic plans, with the pgJDBC driver (built against 42.7.14).
- `pg_stat_statements` preloaded, created in the monitored database, and readable by the application role.
- `pg_read_all_stats` granted to that role for complete statistics and other sessions' visibility.
- Or H2 **2.x** (embedded or server), with `QUERY_STATISTICS=TRUE` for statement statistics (see [H2](#h2)).
- A host scheduler for automatic collection and rollups. Docker is required for the PostgreSQL acceptance tests.
## Install
@@ -47,6 +48,16 @@ GRANT pg_read_all_stats TO your_application_role;
Configure `shared_preload_libraries` with `pg_stat_statements` and create the extension in the monitored database through your normal deployment process. Missing access produces reduced-capability status and setup guidance.
### H2
The same qbit runs on an H2 backend; the provider is chosen from the JDBC product name, so no configuration is needed. Add query statistics to the H2 JDBC URL (or run **Enable statement statistics** on Settings as an H2 admin user; H2 forgets it when the database closes):
```
jdbc:h2:file:/data/app;MODE=MySQL;DB_CLOSE_DELAY=-1;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000
```
The cap matters: H2 keeps 100 statements by default and evicts a third of them, oldest first, whenever it is passed. Use an admin user (the default `sa` is one) to see other sessions and blocking. H2 keeps no transaction, cache-hit, scan, vacuum or index-usage counters, so those dashboard figures stay empty and the rules that need them (missing/unused index, bloat, vacuum, cache hit, temp spill, deadlocks, pool sizing) are skipped rather than reporting zeros. Plans are H2 text (index choice and scan counts), not a cost tree. [docs/PROVIDERS.md](docs/PROVIDERS.md) has the full capability table and what other vendors would need.
With `schedulerName`, `dbPerfCollectSnapshot` runs every five minutes and `dbPerfRollupAndPurge` daily at 03:15 (UTC by default). Omitting it registers no schedules. Use **Collect now** on Snapshots for the first baseline and a second collection for deltas. Table and index statistics normally update hourly. The scheduled processes are durable-eligible when the host has durable jobs enabled.
UI hosts must include `qqq-middleware-javalin` (an optional dependency of this qbit). The producer registers UI API extension `db-performance` version `1.0.0` under `/qqq/v1/qbit/db-performance`. Core-only hosts can use tables and collection without middleware.
@@ -80,7 +91,7 @@ Snapshot totals include all observed statement deltas. Only the top configured d
| `db_performance.view_live_text` | Raw SQL in the transient activity API only; also requires view |
| `db_performance.explain` | PLAN/GENERIC requests and Query plan record action |
| `db_performance.explain_analyze` | Execute the guarded ANALYZE service; the record action also needs explain |
| `db_performance.manage_settings` | Settings/rule edits and Collect now |
| `db_performance.manage_settings` | Settings/rule edits, Collect now and Enable statement statistics |
| `db_performance.manage_findings` | Finding status actions supplied by the findings phase |
Hosts can change `permissionBaseName`; grant the corresponding complete names. Persistent collector and explain-audit writes run as the system user, retaining the requesting session's user ID for plans. Dashboard and API reads retain the viewer's session.
@@ -116,6 +127,8 @@ Paths above are relative to `/qqq/v1/qbit/db-performance`. The [committed OpenAP
Stored SQL is normalized and fingerprinted. Raw `pg_stat_activity` text is never persisted and appears only in the activity API with `view_live_text`; home widgets always use normalized text. SQL literals and parameter values must not be used as identifiers: normalization preserves quoted identifiers. Settings and rules are excluded from AI exposure.
On H2, PLAN runs `EXPLAIN` (not executed; `?n` slots stay unbound for generic plans) and ANALYZE runs `EXPLAIN ANALYZE` with the values bound. H2 has no READ ONLY transaction and a rollback does not undo file, session, sequence or DDL side effects, so on H2 both PLAN and ANALYZE refuse more than one statement and any SQL that names a side-effecting function (`FILE_WRITE`, `FILE_READ`, `CSVWRITE`, `CSVREAD`, `LINK_SCHEMA`, `SCRIPT`, `RUNSCRIPT`, `DISK_SPACE_USED`, `CANCEL_SESSION`, `ABORT_SESSION`, `SLEEP`, sequences), a DDL/DML or session keyword, or any user-defined alias; quoted names count. ANALYZE executes the query, so it is offered only to non-admin H2 users (an admin can call file functions) and only for a single SELECT/WITH; capabilities report it as unavailable otherwise. The connection is marked read-only (a hint in H2) and the transaction always rolls back.
PLAN executes EXPLAIN in a read-only transaction with a local timeout and rollback. Parameterized PLAN uses PostgreSQL 16+ GENERIC_PLAN. Only eligible literal-free PLAN/GENERIC plans are stored. A user's SQL must equal its own normalized form to qualify; SQL loaded from the normalized statement table also qualifies.
ANALYZE accepts one SELECT or a non-modifying WITH, rejects INTO and row locks, binds supplied strings as unknown JDBC parameters, and executes inside a read-only transaction. Functions are still constrained by PostgreSQL READ ONLY. The EXPLAIN command uses pgJDBC simple-query execution with JDBC-managed parameter quoting, so PostgreSQL can coerce unknown-typed EXECUTE arguments. The transaction always rolls back and the prepared statement is explicitly deallocated before the connection returns to its pool. SQLSTATE 57014 records TIMEOUT. The service stores only a sanitized audit, never ANALYZE plans or parameter values. Returned ANALYZE plans are transient and can contain values; the assistant prompt redacts literals in plan expressions. Review transient plans before copying them elsewhere.
+73
View File
@@ -0,0 +1,73 @@
# Statistics providers
The qbit reads database statistics through `DbStatsProvider`. `DbStatsProviders` picks the provider from the JDBC product name (`DatabaseMetaData.getDatabaseProductName()`); anything unregistered gets `UnsupportedStatsProvider`, which shows a setup message instead of failing. Each provider declares the statistics areas (`StatsFeature`) its vendor can supply; `ProviderCapabilities.features()` says which are available on the current connection. Dashboards leave unavailable figures empty and analysis rules that need a missing area (`DbPerfRule.requires()`) are skipped, so a vendor with fewer counters produces fewer findings, never false ones.
| JDBC product name | Provider | Saved name | Status |
|---|---|---|---|
| `PostgreSQL` | `PostgresStatsProvider` | `POSTGRES` | Supported (13+) |
| `H2` | `H2StatsProvider` | `H2` | Supported (2.x) |
| `MySQL`, `MariaDB` | none yet | `MYSQL` (reserved) | Unsupported |
| `SQLite` | none yet | | Unsupported |
## Areas by provider
S = supported, A = approximated, - = not available from the vendor.
| Area (`StatsFeature`) | PostgreSQL | H2 | H2 source and notes |
|---|---|---|---|
| Database counters (commits, rollbacks, temp, deadlocks) | S | - | H2 keeps no transaction or temp-file counters |
| Cache hit ratio | S | - | `info.CACHE_HIT_RATIO` exists but is not a cumulative hit/read pair; shown in settings only |
| Database size | S | S | `SETTINGS` `info.FILE_SIZE` (0 for in-memory databases) |
| Connection limit | S | - | H2 has no connection limit |
| Statement statistics | S | S | `QUERY_STATISTICS` with `QUERY_STATISTICS=TRUE`: calls, total/mean/max ms, rows. No query id or user: the id is a stable SHA-256 hash of the text and the user is empty. The collector's own `INFORMATION_SCHEMA` and `DISK_SPACE_USED` reads and connection setup (`SET ...`, which H2 replays for every JDBC URL setting on each new connection, and `CALL DATABASE()`) are excluded. H2 keeps `QUERY_STATISTICS_MAX_ENTRIES` statements (default 100, set 1000) and evicts the oldest third past the cap |
| Statement I/O (blocks, temp) | S | - | |
| Statement reset time | S (14+) | - | A restart or eviction shows as falling call counts, which the delta logic treats as a reset |
| Table row estimates | S | A | `TABLES.ROW_COUNT_ESTIMATE` |
| Table sizes | S | A | `DISK_SPACE_USED(table)`: table data only; H2 cannot size indexes, so total = table bytes |
| Table access counters (scans, tuple writes) | S | - | |
| Table maintenance (dead rows, vacuum, freeze) | S | - | MVStore has no vacuum |
| Index definitions | S | S | `INDEXES` + `INDEX_COLUMNS`; constraint-generated indexes are marked `USING CONSTRAINT` and never proposed for removal |
| Index usage and size | S | - | |
| Live sessions | S | S | `SESSIONS`: RUNNING = active, SLEEP with uncommitted changes = idle in transaction. Non-admin users see only their own session |
| Blocking | S | S | `SESSIONS.BLOCKER_ID` (admin users) |
| Wait events | S | A | Blocked sessions report wait type `Lock` |
| Transaction age | S | A | No transaction start in H2: an idle transaction's age is the time since it went idle (a lower bound) |
| Connection counts | S | S | `SESSIONS` grouped by user, client info and state |
| Plans | JSON tree with costs | Text | `EXPLAIN` / `EXPLAIN ANALYZE` (index choice, scan counts); generic plans keep `?n` unbound. ANALYZE only for non-admin users (see below) |
Access codes saved on settings: PostgreSQL `FULL`, `NO_READ_ALL_STATS`, `NO_PG_STAT_STATEMENTS`, `UNSUPPORTED`; H2 `FULL`, `NOT_ADMIN`, `NO_QUERY_STATISTICS`, `UNSUPPORTED`. Health and the Setup panel ask the provider for the remedy text (`remedy`, `setupChecks`). **Enable statement statistics** on Settings calls `enableStatementStatistics`: H2 runs `SET QUERY_STATISTICS_MAX_ENTRIES 1000` and `SET QUERY_STATISTICS TRUE` (admin, until the database closes); PostgreSQL cannot do it at runtime.
## H2 query plan rule
H2 has no READ ONLY transaction and a rollback does not undo file, session, sequence or DDL side effects, so on H2 both PLAN and ANALYZE refuse more than one statement and any SQL that names a side-effecting function (`FILE_WRITE`, `FILE_READ`, `CSVWRITE`, `CSVREAD`, `LINK_SCHEMA`, `SCRIPT`, `RUNSCRIPT`, `DISK_SPACE_USED`, `CANCEL_SESSION`, `ABORT_SESSION`, `SLEEP`, sequences), a DDL/DML or session keyword, or any user-defined alias; quoted names count. ANALYZE executes the query, so it is offered only to non-admin H2 users (an admin can call file functions) and only for a single SELECT/WITH; capabilities report it as unavailable otherwise. The connection is marked read-only (a hint in H2) and the transaction always rolls back. A host whose H2 user is `sa` (an admin, as in the enterprise test server) therefore gets PLAN only.
## Adding a vendor
1. Implement `DbStatsProvider`: the reads, `capabilities`, `features`, `remedy`, `setupChecks`, and optionally `enableStatementStatistics` and `explainText`. Map session states to `active` / `idle` / `idle in transaction` and leave unavailable counters out of the maps rather than writing zeros.
2. Register it under the JDBC product name: add it to the table in `DbStatsProviders`, or call `DbStatsProviders.register("ProductName", MyProvider::new)` from the host before collection.
3. Add its saved name to `DbPerfPossibleValues.Provider` and any new access codes to `StatsAccess` and the health check.
4. Make sure the qbit's Liquibase changelog runs on the vendor (the qbit stores its tables in the monitored backend).
5. Test every read against a real database (Testcontainers or embedded), with and without the vendor's statistics prerequisites.
## MySQL and MariaDB
What a provider would map (MySQL 8.0+, MariaDB 10.6+ with `performance_schema=ON`, which needs a server restart):
| Area | Source | Notes |
|---|---|---|
| Statements | `performance_schema.events_statements_summary_by_digest` | Digest id and normalized text, calls, timer (picoseconds), rows examined/sent, temp tables, `FIRST_SEEN`; `TRUNCATE` resets. Needs `SELECT` on `performance_schema` |
| Database counters | `SHOW GLOBAL STATUS` (`Com_commit`, `Com_rollback`, `Created_tmp_disk_tables`, `Innodb_deadlocks`) | Server-wide, not per schema |
| Cache hit ratio | `Innodb_buffer_pool_read_requests` / `Innodb_buffer_pool_reads` | |
| Connection limit and counts | `max_connections`, `information_schema.PROCESSLIST` / `performance_schema.threads` | Needs `PROCESS` to see other users |
| Sessions and blocking | `PROCESSLIST`, `sys.innodb_lock_waits` (MySQL) / `information_schema.INNODB_LOCK_WAITS` (MariaDB) | |
| Tables | `information_schema.TABLES` (`TABLE_ROWS` estimate, `DATA_LENGTH`, `INDEX_LENGTH`) | Sizes are estimates |
| Table access | `performance_schema.table_io_waits_summary_by_table` | Read/write counts, no seq-vs-index split |
| Index usage | `performance_schema.table_io_waits_summary_by_index_usage`, `sys.schema_unused_indexes` | |
| Maintenance | - | InnoDB purge is automatic; no vacuum or dead-row counters |
| Plans | `EXPLAIN FORMAT=JSON`, `EXPLAIN ANALYZE` (MySQL 8.0.18+, tree text) | MariaDB uses `ANALYZE FORMAT=JSON`; a JSON renderer would be needed |
Beyond the provider class, MySQL support also needs: the MySQL/MariaDB Connector/J on the host classpath; a provider-neutral `ExplainService` path for JSON plans (today JSON plans are PostgreSQL's format and the renderer is PostgreSQL-specific); MySQL-compatible SQL in the changelog (verify it runs, since the qbit stores its own tables there) and in the migration snippets the findings suggest (`CREATE INDEX CONCURRENTLY` is PostgreSQL syntax; MySQL needs `ALGORITHM=INPLACE, LOCK=NONE`); help text that currently names PostgreSQL views; and Testcontainers tests for both MySQL and MariaDB, since their lock views differ.
## SQLite
SQLite has no server, sessions, statement statistics or users. A provider could supply only: database size (`PRAGMA page_count` x `page_size`), tables and row counts (`sqlite_schema` plus `COUNT(*)` or `sqlite_stat1` after `ANALYZE`), table and index sizes (the `dbstat` virtual table, when compiled in), index definitions (`sqlite_schema.sql`, `PRAGMA index_list` / `index_info`), and text plans (`EXPLAIN QUERY PLAN`). Statement statistics would have to come from QQQ attribution (the qbit's own statement listener), not the database. It is useful only as an inventory and plan viewer.
+3 -1
View File
@@ -47,7 +47,9 @@ actions └► Attribution aggregator (memory) │ │ Analyze → f
`explain(...)`. `PostgresStatsProvider` is the v1 implementation (PostgreSQL 13+, features that
need 16+/17 such as `EXPLAIN (GENERIC_PLAN)` and per-entry `stats_since` are detected).
The provider is chosen per monitored backend from the JDBC vendor; an unsupported vendor is
reported in the Setup card, never a crash.
reported in the Setup card, never a crash. `H2StatsProvider` (H2 2.x, `INFORMATION_SCHEMA`)
followed; providers declare per-area `StatsFeature`s and rules are skipped where an area is
missing. See [PROVIDERS.md](PROVIDERS.md).
- **Connections** are the backend's own (QQQ `ConnectionManager`), so they run as the app's
database user. That user needs the built-in `pg_read_all_stats` role to see other sessions'
query text and all statements; without it the qbit runs in a reduced mode and the Setup card and
+3 -2
View File
@@ -18,6 +18,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<revision>0.1.0-SNAPSHOT</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
@@ -28,7 +29,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.0.0</version>
<version>2.2.1</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -248,7 +249,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.13</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -22,9 +22,11 @@ package com.kingsrook.qbits.dbperformance.alerts;
import java.time.Instant;
import java.util.Comparator;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
@@ -38,6 +40,9 @@ import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -146,7 +151,7 @@ public class SnapshotEvaluator
long blocked = activity.stream().filter(r -> "BLOCKED".equals(r.getValueString("reason")))
.filter(r -> Math.max(RuleContext.number(r, "querySeconds"), RuleContext.number(r, "xactSeconds")) >= RuleContext.number(settings, "blockedAlertSeconds")).count();
signal(BLOCKED_CHAIN, backend, null, blocked > 0, "Database blocking chain", blocked + " blocked sessions exceed the duration threshold", snapshot, "db_perf_snapshot");
if("OK".equals(status) || "NO_READ_ALL_STATS".equals(settings.getValueString("statsAccess")))
if("OK".equals(status) || List.of("NO_READ_ALL_STATS", "NOT_ADMIN").contains(settings.getValueString("statsAccess")))
{
spikes(ctx, snapshot);
}
@@ -159,33 +164,31 @@ public class SnapshotEvaluator
*******************************************************************************/
private void spikes(RuleContext ctx, QRecord snapshot) throws Exception
{
Map<Integer, List<QRecord>> history = ctx.window("query_snapshot", 7).stream().collect(Collectors.groupingBy(r -> r.getValueInteger("statementId")));
Map<Integer, StatementHistory> history = history(ctx, snapshot.getValueInteger("id"));
for(QRecord statement : ctx.identities("statement"))
{
List<QRecord> samples = history.getOrDefault(statement.getValueInteger("id"), List.of());
QRecord current = samples.stream().filter(r -> snapshot.getValueInteger("id").equals(r.getValueInteger("snapshotId"))).findFirst().orElse(null);
if(current == null)
StatementHistory samples = history.getOrDefault(statement.getValueInteger("id"), new StatementHistory());
if(samples.current == null)
{
Instant last = samples.stream().map(r -> r.getValueInstant("capturedAt")).max(Comparator.naturalOrder()).orElse(Instant.EPOCH);
Instant last = samples.last == null ? Instant.EPOCH : samples.last;
if(!last.plusSeconds(3600).isAfter(ctx.now))
{
signal(QUERY_SPIKE, ctx.backend, statement.getValueString("id"), false, "Query absent", "Statement absent from snapshots for at least one hour", statement, "db_perf_statement");
}
continue;
}
List<Double> previous = samples.stream().filter(r -> r.getValueInstant("capturedAt").isBefore(ctx.now))
.map(r -> RuleContext.number(r, "totalMs")).sorted().toList();
if(previous.size() < 12)
double[] previous = samples.sortedPrevious();
if(previous.length < 12)
{
if(RuleContext.number(current, "totalMs") < RuleContext.number(ctx.settings, "spikeMinTotalMs"))
if(samples.current < RuleContext.number(ctx.settings, "spikeMinTotalMs"))
{
signal(QUERY_SPIKE, ctx.backend, statement.getValueString("id"), false, "Query recovered", "Interval time is below the configured floor", statement, "db_perf_statement");
}
continue;
}
int middle = previous.size() / 2;
double median = previous.size() % 2 == 0 ? (previous.get(middle - 1) + previous.get(middle)) / 2 : previous.get(middle);
double time = RuleContext.number(current, "totalMs");
int middle = previous.length / 2;
double median = previous.length % 2 == 0 ? (previous[middle - 1] + previous[middle]) / 2 : previous[middle];
double time = samples.current;
boolean spike = time >= RuleContext.number(ctx.settings, "spikeMinTotalMs") && time >= median * RuleContext.number(ctx.settings, "spikeFactor");
String normalized = SqlNormalizer.normalize(statement.getValueString("queryText"));
String text = normalized.substring(0, Math.min(200, normalized.length()));
@@ -195,6 +198,90 @@ public class SnapshotEvaluator
}
/*******************************************************************************
** Seven days of per-statement interval times, read in keyset pages and kept
** as primitive doubles. kof22 writes ~33,000 query snapshots a day, so the
** window is ~230,000 rows; holding them as QRecords on every snapshot
** evaluation cost hundreds of MB of worker heap.
*******************************************************************************/
static Map<Integer, StatementHistory> history(RuleContext ctx, Integer currentSnapshotId) throws Exception
{
Set<Integer> valid = ctx.window("snapshot", 7).stream().filter(r -> List.of("OK", "PARTIAL").contains(r.getValueString("status")))
.map(r -> r.getValueInteger("id")).collect(Collectors.toSet());
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN, ctx.now.minusSeconds(7 * 86400L)),
new QFilterCriteria("capturedAt", QCriteriaOperator.LESS_THAN_OR_EQUALS, ctx.now));
Map<Integer, StatementHistory> history = new HashMap<>();
DbPerfStore.forEachPage("db_perf_query_snapshot", filter, DbPerfStore.BATCH_SIZE, page ->
{
for(QRecord row : page)
{
Integer snapshotId = row.getValueInteger("snapshotId");
if(!valid.contains(snapshotId))
{
continue;
}
StatementHistory statement = history.computeIfAbsent(row.getValueInteger("statementId"), ignored -> new StatementHistory());
Instant at = row.getValueInstant("capturedAt");
double totalMs = RuleContext.number(row, "totalMs");
if(statement.last == null || at.isAfter(statement.last))
{
statement.last = at;
}
if(snapshotId.equals(currentSnapshotId) && statement.current == null)
{
statement.current = totalMs;
}
if(at.isBefore(ctx.now))
{
statement.add(totalMs);
}
}
});
return (history);
}
/*******************************************************************************
** One statement's samples: the current interval, the latest capture time
** and earlier interval times.
*******************************************************************************/
static final class StatementHistory
{
private Double current;
private Instant last;
private double[] previous = new double[16];
private int size;
/***************************************************************************
** Append an earlier interval time.
***************************************************************************/
void add(double value)
{
if(size == previous.length)
{
previous = Arrays.copyOf(previous, size * 2);
}
previous[size++] = value;
}
/***************************************************************************
** Earlier interval times, ascending.
***************************************************************************/
double[] sortedPrevious()
{
double[] sorted = Arrays.copyOf(previous, size);
Arrays.sort(sorted);
return (sorted);
}
}
/*******************************************************************************
** Stable backend/subject keys make raises and resolutions restart-safe.
*******************************************************************************/
@@ -25,9 +25,12 @@ import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -70,7 +73,8 @@ public class AnalyzeService
/*******************************************************************************
** Disabled rules are neither evaluated nor reconciled.
** Disabled rules, and rules whose evidence the backend's provider cannot
** supply (unknown before the first collection), are neither evaluated nor reconciled.
*******************************************************************************/
public List<Result> run(String backend) throws Exception
{
@@ -79,9 +83,15 @@ public class AnalyzeService
throw (new IllegalArgumentException("Backend is not configured for this runtime"));
}
QRecord settings = new DbPerfSettingsService(runtime).ensure(backend);
String provider = settings.getValueString("provider");
Set<StatsFeature> available = provider == null ? null : DbStatsProviders.forProviderName(provider).features();
List<Result> results = new ArrayList<>();
for(DbPerfRule rule : rules)
{
if(available != null && !available.containsAll(rule.requires()))
{
continue;
}
try
{
QRecord row = runtime.asSystem(() -> DbPerfStore.first("db_perf_rule", DbPerfStore.eq("code", rule.code())));
@@ -22,6 +22,8 @@ package com.kingsrook.qbits.dbperformance.analyze;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
/*******************************************************************************
@@ -39,4 +41,15 @@ public interface DbPerfRule
** Evaluate the configured thresholds against stored observations.
*******************************************************************************/
List<FindingDraft> evaluate(RuleContext ctx) throws Exception;
/*******************************************************************************
** Statistics areas the evidence comes from. A rule is skipped (neither
** evaluated nor reconciled) when the backend's provider cannot supply them.
*******************************************************************************/
default Set<StatsFeature> requires()
{
return (Set.of());
}
}
@@ -36,6 +36,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -60,6 +61,30 @@ public enum StoredRule implements DbPerfRule
}
/*******************************************************************************
** Provider areas behind each rule's evidence; attribution rules need none.
*******************************************************************************/
@Override
public Set<StatsFeature> requires()
{
return (switch(this)
{
case SLOW_QUERY -> Set.of(StatsFeature.STATEMENTS);
case MISSING_INDEX, OVER_AUDITED -> Set.of(StatsFeature.TABLE_ACCESS);
case UNUSED_INDEX -> Set.of(StatsFeature.INDEX_USAGE);
case DUPLICATE_INDEX -> Set.of(StatsFeature.INDEX_DEFINITIONS);
case BLOAT, VACUUM_STALE -> Set.of(StatsFeature.TABLE_MAINTENANCE);
case POOL_SIZING -> Set.of(StatsFeature.MAX_CONNECTIONS, StatsFeature.CONNECTIONS);
case IDLE_IN_TRANSACTION -> Set.of(StatsFeature.SESSIONS);
case CACHE_HIT_LOW -> Set.of(StatsFeature.CACHE_COUNTERS);
case TEMP_SPILL -> Set.of(StatsFeature.STATEMENT_IO);
case DEADLOCKS -> Set.of(StatsFeature.DATABASE_COUNTERS);
case N_PLUS_ONE, HOT_JOB -> Set.of();
});
}
/*******************************************************************************
** Disabled evaluation is inert even when called outside AnalyzeService.
*******************************************************************************/
@@ -50,7 +50,7 @@ public class ExplainSpec extends DbPerformanceJsonSpec
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/db-performance/explain")
.withShortSummary("Explain a PostgreSQL query").withLongDescription("PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze. Values and ANALYZE plans are never persisted."));
.withShortSummary("Explain a query (PostgreSQL JSON or H2 text plan)").withLongDescription("PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze. Values and ANALYZE plans are never persisted."));
}
@@ -47,6 +47,7 @@ import com.kingsrook.qbits.dbperformance.provider.IndexStats;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
import com.kingsrook.qbits.dbperformance.provider.StatementStats;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qbits.dbperformance.provider.TableStats;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -217,7 +218,7 @@ public class SnapshotCollector
Instant now = snapshot.getValueInstant("capturedAt");
QQueryFilter filter = DbPerfStore.eq("backendName", backendName)
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("OK", "PARTIAL")))
.withCriteria(new QFilterCriteria("rawXactCommit", QCriteriaOperator.IS_NOT_BLANK))
.withCriteria(new QFilterCriteria("databaseSizeBytes", QCriteriaOperator.IS_NOT_BLANK))
.withOrderBy(new QFilterOrderBy("capturedAt", false));
QRecord previous = DbPerfStore.first("db_perf_snapshot", filter);
Instant previousCapture = previous == null ? null : previous.getValueInstant("capturedAt");
@@ -256,8 +257,9 @@ public class SnapshotCollector
}
}
snapshot.withValue("rawStatsReset", database.statsReset()).withValue("statsResetDetected", databaseReset)
.withValue("databaseSizeBytes", database.sizeBytes()).withValue("maxConnections", database.maxConnections())
.withValue("cacheHitRatio", ratio(number(snapshot, "blksHit"), number(snapshot, "blksHit") + number(snapshot, "blksRead")));
.withValue("databaseSizeBytes", database.sizeBytes())
.withValue("cacheHitRatio", capability.has(StatsFeature.CACHE_COUNTERS) ? ratio(number(snapshot, "blksHit"), number(snapshot, "blksHit") + number(snapshot, "blksRead")) : null)
.withValue("maxConnections", capability.has(StatsFeature.MAX_CONNECTIONS) ? database.maxConnections() : null);
if(capability.statements())
{
Instant reset = provider.readStatementResetInfo(connection).statsReset();
@@ -441,8 +443,8 @@ public class SnapshotCollector
QRecord row = new QRecord().withValue("backendName", backend).withValue("schemaName", entry.schemaName()).withValue("tableName", entry.tableName()).withValue("relid", entry.relid()).withValue("lastSeenAt", now);
entry.counters().forEach(row::setValue);
entry.maintenance().forEach(row::setValue);
row.withValue("deadRatio", ratio(number(row, "nDeadTup"), number(row, "nLiveTup") + number(row, "nDeadTup")))
.withValue("seqScanShare", ratio(number(row, "seqScan"), number(row, "seqScan") + number(row, "idxScan")));
row.withValue("deadRatio", entry.counters().containsKey("nDeadTup") ? ratio(number(row, "nDeadTup"), number(row, "nLiveTup") + number(row, "nDeadTup")) : null)
.withValue("seqScanShare", entry.counters().containsKey("seqScan") ? ratio(number(row, "seqScan"), number(row, "seqScan") + number(row, "idxScan")) : null);
if(old == null)
{
inserts.add(row);
@@ -38,6 +38,10 @@ import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -135,8 +139,9 @@ public class ExplainService
boolean generic = !analyze && parsed.parameterCount() > 0;
audit.setValue("mode", analyze ? "ANALYZE" : generic ? "GENERIC" : "PLAN");
QRecord settings = new DbPerfSettingsService(runtime).ensure(request.backendName());
json = execute(parsed.sql(), parameters, analyze, generic, SnapshotCollector.positive(settings, "explainTimeoutMs", 10000));
plan = PlanTextRenderer.render(json);
Explained explained = execute(parsed, parameters, analyze, generic, SnapshotCollector.positive(settings, "explainTimeoutMs", 10000));
json = explained.json();
plan = explained.plan();
if(!analyze && (storedStatement || sql.equals(SqlNormalizer.normalize(sql))))
{
audit.setValue("planJson", json);
@@ -181,7 +186,7 @@ public class ExplainService
}
return (new ExplainResult(audit.getValueString("mode"), json, plan.text(), plan.totalCost(), plan.actualTotalMs(), warnings,
audit.getValueInteger("id"), runtime.getConfig().getSqlWorkbenchRunnerPath() + "?sql=" + encode("EXPLAIN " + sql) + "&backend=" + encode(request.backendName()),
"Explain this PostgreSQL query plan and suggest improvements.\n\nQuery:\n" + SqlNormalizer.normalize(sql) + "\n\nPlan:\n" + plan.safeText()));
"Explain this " + (json == null ? "database" : "PostgreSQL") + " query plan and suggest improvements.\n\nQuery:\n" + SqlNormalizer.normalize(sql) + "\n\nPlan:\n" + plan.safeText()));
}
@@ -190,89 +195,141 @@ public class ExplainService
** Roll back even on parse, execution or timeout failure, deallocate after rollback,
** and restore pool state before releasing this dedicated connection.
*******************************************************************************/
private String execute(String sql, List<String> parameters, boolean analyze, boolean generic, int timeout) throws Exception
private Explained execute(ExplainSql.Parsed parsed, List<String> parameters, boolean analyze, boolean generic, int timeout) throws Exception
{
try(Connection connection = openConnection())
{
if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName()))
{
throw (new QBadRequestException("Query plans currently require PostgreSQL."));
}
if(generic && connection.getMetaData().getDatabaseMajorVersion() < 16)
{
throw (new QBadRequestException("Generic plans require PostgreSQL 16 or later. Use ANALYZE with parameter values if authorized."));
}
boolean autoCommit = connection.getAutoCommit();
boolean readOnly = connection.isReadOnly();
String preparedName = "dbperf_" + UUID.randomUUID().toString().replace("-", "");
boolean prepared = false;
try
{
connection.setAutoCommit(false);
try(Statement statement = connection.createStatement())
//////////////////////////////////////////////////////////////////
// other vendors plan through their provider as text (no JSON) //
//////////////////////////////////////////////////////////////////
DbStatsProvider provider = DbStatsProviders.forConnection(connection);
ProviderCapabilities capability = provider.capabilities(connection);
if(!capability.has(StatsFeature.EXPLAIN_TEXT) || generic && !capability.has(StatsFeature.EXPLAIN_GENERIC))
{
statement.execute("SET TRANSACTION READ ONLY");
statement.execute("SET LOCAL statement_timeout = " + timeout);
statement.execute("SET LOCAL standard_conforming_strings = on");
if(!analyze)
{
// GENERIC_PLAN must not be parsed as an external JDBC bind request.
statement.unwrap(PgStatement.class).executeWithFlags("EXPLAIN (" + (generic ? "GENERIC_PLAN, " : "") + "FORMAT JSON) " + sql, QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
try(ResultSet rows = statement.getResultSet())
{
rows.next();
return (rows.getString(1));
}
}
statement.execute("PREPARE " + preparedName + " AS " + sql);
prepared = true;
throw (new QBadRequestException("Query plans are not available for this database (" + provider.getName() + ")."));
}
String arguments = parameters.isEmpty() ? "" : "(" + String.join(", ", Collections.nCopies(parameters.size(), "?")) + ")";
try(PreparedStatement statement = connection.prepareStatement("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE " + preparedName + arguments))
if(analyze && !capability.has(StatsFeature.EXPLAIN_ANALYZE))
{
for(int i = 0; i < parameters.size(); i++)
throw (new QBadRequestException("ANALYZE is not available on this " + provider.getName() + " connection (it would execute the query without a read-only transaction). Use PLAN."));
}
String text;
try
{
text = provider.explainText(connection, parsed.indexedSql(), parameters, analyze, timeout);
}
catch(java.sql.SQLException e)
{
if("42000".equals(e.getSQLState()) && e.getMessage() != null && e.getMessage().contains("H2"))
{
statement.setObject(i + 1, parameters.get(i), Types.OTHER);
throw (new QBadRequestException(e.getMessage()));
}
// EXECUTE's arguments are utility expressions: unknown bind types cannot
// be inferred in extended Parse. Keep JDBC binding/escaping, but send this
// one command in simple mode so PostgreSQL coerces the quoted strings.
statement.unwrap(PgStatement.class).executeWithFlags(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
throw (e);
}
return (new Explained(null, new PlanTextRenderer.Plan(text, PlanTextRenderer.redactText(text), null, null)));
}
return (new Explained(postgres(connection, parsed.sql(), parameters, analyze, generic, timeout), null).rendered());
}
}
/*******************************************************************************
** PostgreSQL JSON plan; see {@link #execute}.
*******************************************************************************/
private String postgres(Connection connection, String sql, List<String> parameters, boolean analyze, boolean generic, int timeout) throws Exception
{
if(generic && connection.getMetaData().getDatabaseMajorVersion() < 16)
{
throw (new QBadRequestException("Generic plans require PostgreSQL 16 or later. Use ANALYZE with parameter values if authorized."));
}
boolean autoCommit = connection.getAutoCommit();
boolean readOnly = connection.isReadOnly();
String preparedName = "dbperf_" + UUID.randomUUID().toString().replace("-", "");
boolean prepared = false;
try
{
connection.setAutoCommit(false);
try(Statement statement = connection.createStatement())
{
statement.execute("SET TRANSACTION READ ONLY");
statement.execute("SET LOCAL statement_timeout = " + timeout);
statement.execute("SET LOCAL standard_conforming_strings = on");
if(!analyze)
{
// GENERIC_PLAN must not be parsed as an external JDBC bind request.
statement.unwrap(PgStatement.class).executeWithFlags("EXPLAIN (" + (generic ? "GENERIC_PLAN, " : "") + "FORMAT JSON) " + sql, QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
try(ResultSet rows = statement.getResultSet())
{
rows.next();
return (rows.getString(1));
}
}
statement.execute("PREPARE " + preparedName + " AS " + sql);
prepared = true;
}
String arguments = parameters.isEmpty() ? "" : "(" + String.join(", ", Collections.nCopies(parameters.size(), "?")) + ")";
try(PreparedStatement statement = connection.prepareStatement("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE " + preparedName + arguments))
{
for(int i = 0; i < parameters.size(); i++)
{
statement.setObject(i + 1, parameters.get(i), Types.OTHER);
}
// EXECUTE's arguments are utility expressions: unknown bind types cannot
// be inferred in extended Parse. Keep JDBC binding/escaping, but send this
// one command in simple mode so PostgreSQL coerces the quoted strings.
statement.unwrap(PgStatement.class).executeWithFlags(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE);
try(ResultSet rows = statement.getResultSet())
{
rows.next();
return (rows.getString(1));
}
}
}
finally
{
try
{
connection.rollback();
if(prepared)
{
try(Statement cleanup = connection.createStatement())
{
cleanup.execute("DEALLOCATE " + preparedName);
}
finally
{
connection.rollback();
}
}
}
finally
{
try
{
connection.rollback();
if(prepared)
{
try(Statement cleanup = connection.createStatement())
{
cleanup.execute("DEALLOCATE " + preparedName);
}
finally
{
connection.rollback();
}
}
}
finally
{
connection.setReadOnly(readOnly);
connection.setAutoCommit(autoCommit);
}
connection.setReadOnly(readOnly);
connection.setAutoCommit(autoCommit);
}
}
}
/*******************************************************************************
** Raw plan (JSON for PostgreSQL, null for text-plan vendors) and its rendering.
*******************************************************************************/
private record Explained(String json, PlanTextRenderer.Plan plan)
{
/*******************************************************************************
** Render a PostgreSQL JSON plan.
*******************************************************************************/
Explained rendered() throws Exception
{
return (new Explained(json, PlanTextRenderer.render(json)));
}
}
/*******************************************************************************
** Always obtain a dedicated connection, independent of any caller QQQ transaction.
** Protected for contract tests that verify pool state restoration on driver errors.
@@ -51,10 +51,21 @@ public final class ExplainSql
/*******************************************************************************
** Executable SQL with contiguous PostgreSQL bind slots and unquoted tokens.
** Executable SQL with contiguous PostgreSQL bind slots and unquoted tokens,
** plus the same SQL with JDBC-indexed ?n slots for vendors such as H2.
*******************************************************************************/
public record Parsed(String sql, List<String> tokens, int parameterCount)
public record Parsed(String sql, List<String> tokens, int parameterCount, String indexedSql)
{
/*******************************************************************************
** PostgreSQL-only form (no JDBC-indexed variant).
*******************************************************************************/
public Parsed(String sql, List<String> tokens, int parameterCount)
{
this(sql, tokens, parameterCount, sql);
}
/*******************************************************************************
** Reject writes, SELECT INTO and every locking SELECT, including nested CTEs.
*******************************************************************************/
@@ -83,6 +94,7 @@ public final class ExplainSql
StringBuilder result = new StringBuilder();
List<String> tokens = new ArrayList<>();
Set<Integer> positions = new HashSet<>();
List<int[]> slots = new ArrayList<>();
int jdbc = 0;
int i = 0;
boolean ended = false;
@@ -186,6 +198,10 @@ public final class ExplainSql
throw (new QBadRequestException("SQL parameter positions must be between 1 and 65535."));
}
}
if(i > start + 1)
{
slots.add(new int[] {result.length(), result.length() + (i - start), Integer.parseInt(sql.substring(start + 1, i))});
}
result.append(sql, start, i);
continue;
}
@@ -199,7 +215,9 @@ public final class ExplainSql
}
else
{
result.append('$').append(++jdbc);
String slot = "$" + (++jdbc);
slots.add(new int[] {result.length(), result.length() + slot.length(), jdbc});
result.append(slot);
i++;
}
continue;
@@ -223,7 +241,12 @@ public final class ExplainSql
{
throw (new QBadRequestException("Use either ? parameters or contiguous $1 through $n parameters."));
}
return (new Parsed(result.toString().trim(), List.copyOf(tokens), Math.max(jdbc, maximum)));
StringBuilder indexed = new StringBuilder(result);
for(int slot = slots.size() - 1; slot >= 0; slot--)
{
indexed.replace(slots.get(slot)[0], slots.get(slot)[1], "?" + slots.get(slot)[2]);
}
return (new Parsed(result.toString().trim(), List.copyOf(tokens), Math.max(jdbc, maximum), indexed.toString().trim()));
}
@@ -55,6 +55,41 @@ public final class PlanTextRenderer
/*******************************************************************************
** Literal-free form of a vendor text plan for prompts: quoted strings are
** redacted everywhere, numbers outside comments (H2 keeps index names and
** scan counts inside comments).
*******************************************************************************/
public static String redactText(String text)
{
if(text == null)
{
return (null);
}
StringBuilder result = new StringBuilder();
java.util.regex.Matcher comment = java.util.regex.Pattern.compile("/\\*.*?\\*/", java.util.regex.Pattern.DOTALL).matcher(text);
int last = 0;
while(comment.find())
{
result.append(redactLiterals(text.substring(last, comment.start()), true)).append(redactLiterals(comment.group(), false));
last = comment.end();
}
return (result.append(redactLiterals(text.substring(last), true)).toString());
}
/*******************************************************************************
** Quoted strings, and optionally standalone numbers, become ?.
*******************************************************************************/
private static String redactLiterals(String text, boolean numbers)
{
String result = text.replaceAll("'(?:[^']|'')*'", "?");
return (numbers ? result.replaceAll("(?<![\\w?$.\"])\\d+(?:\\.\\d+)?(?![\\w\"])", "?") : result);
}
/*******************************************************************************
** PostgreSQL emits an array containing the plan and optional execution timing.
*******************************************************************************/
@@ -27,6 +27,7 @@ import java.util.List;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
@@ -111,15 +112,15 @@ public class DbPerfHealthChecks
return (HealthCheckOutcome.unknown("No statistics access observation yet"));
}
String user = ((RDBMSBackendMetaData) runtime.getInstance().getBackend(backend)).getUsername();
String remedy = DbStatsProviders.forProviderName(settings.getValueString("provider")).remedy(access, user);
HealthCheckOutcome outcome = switch(access)
{
case "FULL" -> HealthCheckOutcome.up("Full statistics access");
case "NO_READ_ALL_STATS" -> HealthCheckOutcome.degraded("grant pg_read_all_stats to " + user);
case "NO_PG_STAT_STATEMENTS" -> HealthCheckOutcome.down("Enable pg_stat_statements in shared_preload_libraries and create the extension");
case "UNSUPPORTED" -> HealthCheckOutcome.down("Use a supported PostgreSQL version (13 or newer) and statistics provider");
case "NO_READ_ALL_STATS", "NOT_ADMIN", "NO_QUERY_STATISTICS" -> HealthCheckOutcome.degraded(remedy);
case "NO_PG_STAT_STATEMENTS", "UNSUPPORTED" -> HealthCheckOutcome.down(remedy);
default -> HealthCheckOutcome.unknown("Statistics access has not been classified");
};
String safe = List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "UNSUPPORTED").contains(access) ? access : "UNKNOWN";
String safe = List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "NO_QUERY_STATISTICS", "NOT_ADMIN", "UNSUPPORTED").contains(access) ? access : "UNKNOWN";
return (outcome.withDetails(new HealthDetails().withValue("statsAccess", safe)));
}
var filter = DbPerfStore.eq("backendName", backend).withOrderBy(new QFilterOrderBy("capturedAt", false));
@@ -43,7 +43,7 @@ public final class DbPerfPossibleValues
*******************************************************************************/
public enum Provider implements PossibleValueEnum<String>
{
POSTGRES, MYSQL, UNSUPPORTED;
POSTGRES, H2, MYSQL, UNSUPPORTED;
/*******************************************************************************
@@ -73,7 +73,7 @@ public final class DbPerfPossibleValues
*******************************************************************************/
public enum StatsAccess implements PossibleValueEnum<String>
{
FULL, NO_READ_ALL_STATS, NO_PG_STAT_STATEMENTS, UNSUPPORTED;
FULL, NO_READ_ALL_STATS, NO_PG_STAT_STATEMENTS, NO_QUERY_STATISTICS, NOT_ADMIN, UNSUPPORTED;
/*******************************************************************************
@@ -0,0 +1,99 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.processes;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
/*******************************************************************************
** Settings action: ask the backend's provider to turn on statement statistics
** (H2: SET QUERY_STATISTICS TRUE as an admin), then record the new access.
** This is the qbit's only change to the monitored database and it is explicit.
*******************************************************************************/
public class DbPerfEnableStatisticsStep implements BackendStep
{
/*******************************************************************************
** Requires manage_settings; vendor errors become result lines.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance());
if(QContext.getQSession() == null || !QContext.getQSession().hasPermission(runtime.getConfig().getPermissionBaseName() + ".manage_settings"))
{
throw (new QException("Managing settings permission is required"));
}
List<ProcessSummaryLineInterface> lines = new ArrayList<>();
String backend = runtime.getConfig().getBackendName();
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(backend)))
{
DbStatsProvider provider = DbStatsProviders.forConnection(connection);
try
{
lines.add(new ProcessSummaryLine(Status.OK, 1, provider.enableStatementStatistics(connection)));
}
catch(SQLFeatureNotSupportedException e)
{
lines.add(new ProcessSummaryLine(Status.ERROR, 0, e.getMessage()));
}
catch(SQLException e)
{
lines.add(new ProcessSummaryLine(Status.ERROR, 0, "The database refused the change (SQLSTATE " + e.getSQLState() + "); an admin database user is required."));
}
ProviderCapabilities capability = provider.capabilities(connection);
QRecord settings = new DbPerfSettingsService(runtime).ensure(backend);
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", settings.getValueInteger("id")).withValue("provider", provider.getName())
.withValue("statsAccess", capability.statsAccess()).withValue("statsAccessMessage", capability.message())));
lines.add(new ProcessSummaryLine(Status.INFO, 0, "statistics access: " + capability.statsAccess()));
}
catch(QException e)
{
throw (e);
}
catch(Exception e)
{
throw (new QException("Unable to enable statement statistics", e));
}
output.addValue("processResults", new ArrayList<>(lines));
}
}
@@ -54,6 +54,7 @@ public final class DbPerfProcesses
public static final String COLLECT = "dbPerfCollectSnapshot";
public static final String ROLLUP = "dbPerfRollupAndPurge";
public static final String COLLECT_NOW = "dbPerfCollectNow";
public static final String ENABLE_STATISTICS = "dbPerfEnableStatementStatistics";
/*******************************************************************************
@@ -100,6 +101,10 @@ public final class DbPerfProcesses
.withTableName("db_perf_snapshot")
.withStep(new QFrontendStepMetaData().withName("result").withLabel("Result")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))));
instance.addProcess(base(config, ENABLE_STATISTICS, "Enable statement statistics", DbPerfEnableStatisticsStep.class, false)
.withTableName("db_perf_settings")
.withStep(new QFrontendStepMetaData().withName("result").withLabel("Result")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))));
}
@@ -23,12 +23,17 @@ package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.List;
import java.util.Map;
import java.util.Set;
/*******************************************************************************
** Read-only database statistics SPI. Implementations never change the database.
** Database statistics SPI. Every read is read-only and never changes the
** database; {@link #enableStatementStatistics} is the one explicit, opt-in
** change, run only from the settings action by a user who manages settings.
** A provider is chosen per JDBC product name by {@link DbStatsProviders}.
*******************************************************************************/
public interface DbStatsProvider
{
@@ -90,4 +95,57 @@ public interface DbStatsProvider
** readSettings using the supplied backend connection.
*******************************************************************************/
Map<String, String> readSettings(Connection connection) throws SQLException;
/*******************************************************************************
** Statistics areas this vendor can supply at best, independent of a
** connection. Analysis rules that need an absent area are skipped.
*******************************************************************************/
default Set<StatsFeature> features()
{
return (Set.of());
}
/*******************************************************************************
** Operator next step for a persisted statsAccess code (health and setup).
*******************************************************************************/
default String remedy(String statsAccess, String databaseUser)
{
return ("FULL".equals(statsAccess) ? "Full statistics access" : "Use a supported database vendor and statistics provider");
}
/*******************************************************************************
** Vendor prerequisite rows for the guided setup panel, from saved settings.
*******************************************************************************/
default List<SetupCheck> setupChecks(String statsAccess, String message)
{
return (List.of(new SetupCheck("Statistics access", statsAccess, message == null ? remedy(statsAccess, null) : message)));
}
/*******************************************************************************
** Turn on statement statistics where the vendor allows it at runtime, and
** return what was done. The default explains why it is not possible.
*******************************************************************************/
default String enableStatementStatistics(Connection connection) throws SQLException
{
throw (new SQLFeatureNotSupportedException("Statement statistics cannot be enabled at runtime for this database; see the setup panel."));
}
/*******************************************************************************
** Vendor text plan for providers without structured JSON plans. The SQL uses
** JDBC ?n slots; parameters are bound only for ANALYZE.
*******************************************************************************/
default String explainText(Connection connection, String sql, List<String> parameters, boolean analyze, int timeoutMs) throws SQLException
{
throw (new SQLFeatureNotSupportedException("Query plans are not available for this database."));
}
}
@@ -23,13 +23,31 @@ package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
/*******************************************************************************
** Select a provider from JDBC product identity.
** Provider registry keyed by JDBC {@code DatabaseMetaData.getDatabaseProductName()}.
** Adding a vendor is one implementation of {@link DbStatsProvider} plus one
** {@link #register} call (see docs/PROVIDERS.md); anything unregistered gets
** {@link UnsupportedStatsProvider}.
*******************************************************************************/
public final class DbStatsProviders
{
//////////////////////////////////////////////////////////////////////////
// JDBC product name (lower case) -> provider. MySQL reports "MySQL", //
// MariaDB "MariaDB" (Connector/J 3) and SQLite "SQLite"; none are //
// registered yet: docs/PROVIDERS.md says what each could supply. //
//////////////////////////////////////////////////////////////////////////
private static final Map<String, Supplier<DbStatsProvider>> BY_PRODUCT = new ConcurrentHashMap<>(Map.of(
"postgresql", PostgresStatsProvider::new,
"h2", H2StatsProvider::new));
/*******************************************************************************
** Utility class.
*******************************************************************************/
@@ -38,11 +56,53 @@ public final class DbStatsProviders
}
/*******************************************************************************
** PostgreSQL first; other vendors are explicitly unsupported.
** Provider for the connection's JDBC product.
*******************************************************************************/
public static DbStatsProvider forConnection(Connection connection) throws SQLException
{
return ("PostgreSQL".equalsIgnoreCase(connection.getMetaData().getDatabaseProductName()) ? new PostgresStatsProvider() : new UnsupportedStatsProvider());
return (forProductName(connection.getMetaData().getDatabaseProductName()));
}
/*******************************************************************************
** Provider for a JDBC product name; unsupported when not registered.
*******************************************************************************/
public static DbStatsProvider forProductName(String productName)
{
Supplier<DbStatsProvider> supplier = productName == null ? null : BY_PRODUCT.get(productName.toLowerCase(Locale.ROOT));
return (supplier == null ? new UnsupportedStatsProvider() : supplier.get());
}
/*******************************************************************************
** Provider by its saved name (db_perf_settings.provider), for code that has
** settings but no connection; unsupported for null or unknown names.
*******************************************************************************/
public static DbStatsProvider forProviderName(String providerName)
{
for(Supplier<DbStatsProvider> supplier : BY_PRODUCT.values())
{
DbStatsProvider provider = supplier.get();
if(provider.getName().equalsIgnoreCase(providerName))
{
return (provider);
}
}
return (new UnsupportedStatsProvider());
}
/*******************************************************************************
** Register (or replace) the provider for a JDBC product name. Hosts may add
** vendors before the qbit collects; the name must be a Provider possible value.
*******************************************************************************/
public static void register(String productName, Supplier<DbStatsProvider> provider)
{
BY_PRODUCT.put(productName.toLowerCase(Locale.ROOT), provider);
}
}
@@ -0,0 +1,691 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.instant;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.read;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.scalar;
/*******************************************************************************
** H2 2.x statistics from INFORMATION_SCHEMA. Statement statistics need the
** database setting QUERY_STATISTICS (URL {@code ;QUERY_STATISTICS=TRUE} or
** {@code SET QUERY_STATISTICS TRUE} as an admin); every other area works
** without it. H2 keeps no transaction, cache, scan or vacuum counters, so
** those areas are reported as unavailable rather than as zeros. See
** docs/PROVIDERS.md for what is exact, approximated and unsupported.
*******************************************************************************/
public class H2StatsProvider implements DbStatsProvider
{
public static final String NAME = "H2";
private static final String NO_QUERY_STATISTICS = "NO_QUERY_STATISTICS";
private static final String NOT_ADMIN = "NOT_ADMIN";
private static final int MAX_ENTRIES = 1000;
private static final Pattern WORD = Pattern.compile("[A-Za-z_][A-Za-z0-9_$]*");
//////////////////////////////////////////////////////////////////////////
// H2 has no READ ONLY transaction and a rollback does not undo file, //
// session, sequence or DDL side effects, so plans refuse SQL naming //
// these (and any user-defined alias) outright //
//////////////////////////////////////////////////////////////////////////
private static final Set<String> DENIED = Set.of("FILE_WRITE", "FILE_READ", "CSVWRITE", "CSVREAD", "LINK_SCHEMA", "SCRIPT", "RUNSCRIPT",
"DISK_SPACE_USED", "CANCEL_SESSION", "ABORT_SESSION", "BACKUP", "SHUTDOWN", "CALL", "SET", "DROP", "CREATE", "ALTER", "TRUNCATE", "GRANT",
"REVOKE", "COMMIT", "ROLLBACK", "SAVEPOINT", "CHECKPOINT", "NEXTVAL", "SETVAL", "CURRVAL", "LOCK", "EXECUTE", "INSERT", "UPDATE",
"DELETE", "MERGE", "UPSERT", "REPLACE", "INTO", "FOR", "JAVA_OBJECT", "SLEEP");
///////////////////////////////////////////////////////////////////////////
// statement that guarantees QUERY_STATISTICS has a row when it is on; //
// it and INFORMATION_SCHEMA reads (the collector's own) are not stored //
///////////////////////////////////////////////////////////////////////////
private static final String PROBE = "SELECT 1 AS DB_PERF_PROBE";
private static final String DISK_SPACE = "SELECT DISK_SPACE_USED(?)";
private static final Set<StatsFeature> FEATURES = StatsFeature.of(StatsFeature.DATABASE_SIZE, StatsFeature.STATEMENTS,
StatsFeature.TABLE_ROWS, StatsFeature.TABLE_SIZES, StatsFeature.INDEX_DEFINITIONS, StatsFeature.SESSIONS, StatsFeature.BLOCKING,
StatsFeature.CONNECTIONS, StatsFeature.EXPLAIN_TEXT, StatsFeature.EXPLAIN_GENERIC, StatsFeature.EXPLAIN_ANALYZE);
private static final List<String> SETTINGS = List.of("info.VERSION", "MODE", "MV_STORE", "info.FILE_SIZE", "info.PAGE_SIZE",
"info.CACHE_MAX_SIZE", "info.CACHE_SIZE", "info.CACHE_HIT_RATIO", "info.FILE_READ", "info.FILE_WRITE", "MAX_QUERY_TIMEOUT", "QUERY_TIMEOUT");
/*******************************************************************************
** Provider name.
*******************************************************************************/
@Override
public String getName()
{
return (NAME);
}
/*******************************************************************************
** Version, admin rights and whether QUERY_STATISTICS is collecting.
*******************************************************************************/
@Override
public ProviderCapabilities capabilities(Connection connection) throws SQLException
{
int major = connection.getMetaData().getDatabaseMajorVersion();
int version = version(connection);
boolean supported = major >= 2;
if(!supported)
{
return (new ProviderCapabilities(false, version, false, false, "UNSUPPORTED", "H2 2.0 or newer is required.", Set.of()));
}
boolean statistics = queryStatistics(connection);
boolean admin = admin(connection);
String access = !statistics ? NO_QUERY_STATISTICS : !admin ? NOT_ADMIN : "FULL";
String message = switch(access)
{
case NO_QUERY_STATISTICS -> "H2 query statistics are off. Run Enable statement statistics on Settings as an H2 admin user, or add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 to the JDBC URL.";
case NOT_ADMIN -> "The H2 user is not an admin, so only its own session is visible. Use an admin user (ALTER USER ... ADMIN TRUE) to see every session and blocking.";
default -> "Full statistics access (H2 does not keep transaction, cache, scan or vacuum counters).";
};
EnumSet<StatsFeature> available = EnumSet.copyOf(FEATURES);
if(!statistics)
{
available.remove(StatsFeature.STATEMENTS);
}
if(!admin)
{
available.remove(StatsFeature.BLOCKING);
}
else
{
////////////////////////////////////////////////////////////////////
// an admin may call file and session functions; ANALYZE executes //
// the query, so it is offered only to non-admin users //
////////////////////////////////////////////////////////////////////
available.remove(StatsFeature.EXPLAIN_ANALYZE);
}
return (new ProviderCapabilities(true, version, statistics, admin, access, message, available));
}
/*******************************************************************************
** Areas H2 can supply at best.
*******************************************************************************/
@Override
public Set<StatsFeature> features()
{
return (FEATURES);
}
/*******************************************************************************
** Size from the store; H2 has no cumulative counters, reset time or limit.
*******************************************************************************/
@Override
public DatabaseStats readDatabase(Connection connection) throws SQLException
{
List<Long> size = read(connection, "SELECT SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS WHERE SETTING_NAME = 'info.FILE_SIZE'", row -> parseLong(row.getString(1)));
return (new DatabaseStats(Map.of(), null, 0, size.isEmpty() || size.getFirst() == null ? 0 : size.getFirst()));
}
/*******************************************************************************
** QUERY_STATISTICS rows (times in milliseconds), without connection setup
** (see {@link #excluded}). H2 has no query id or user, so the id is a stable
** hash of the statement text and the user is null.
*******************************************************************************/
@Override
public List<StatementStats> readStatements(Connection connection) throws SQLException
{
List<StatementStats> result = new ArrayList<>();
read(connection, "SELECT SQL_STATEMENT, EXECUTION_COUNT, CUMULATIVE_EXECUTION_TIME, AVERAGE_EXECUTION_TIME, MAX_EXECUTION_TIME, CUMULATIVE_ROW_COUNT FROM INFORMATION_SCHEMA.QUERY_STATISTICS", row ->
{
String sql = row.getString("SQL_STATEMENT");
if(sql != null && !excluded(sql))
{
result.add(new StatementStats(hash(sql), null, true, sql, row.getLong("EXECUTION_COUNT"), millis(row.getDouble("CUMULATIVE_EXECUTION_TIME")),
millis(row.getDouble("AVERAGE_EXECUTION_TIME")), millis(row.getDouble("MAX_EXECUTION_TIME")), row.getLong("CUMULATIVE_ROW_COUNT"), 0, 0, 0, null, null));
}
return (null);
});
return (result);
}
/*******************************************************************************
** Not application statements: the collector's own probe, table sizing and
** INFORMATION_SCHEMA reads, and connection setup. H2 replays every JDBC URL setting as a SET on
** each new connection and drivers call DATABASE() for the catalog, so with
** unpooled connections these dominate the call counts without being work.
*******************************************************************************/
static boolean excluded(String sql)
{
String upper = sql.strip().toUpperCase(Locale.ROOT);
return (PROBE.equals(sql) || DISK_SPACE.equals(sql) || upper.contains("INFORMATION_SCHEMA.") || upper.startsWith("SET ") || upper.equals("CALL DATABASE()"));
}
/*******************************************************************************
** H2 keeps no reset time; a reset shows as falling call counts instead.
*******************************************************************************/
@Override
public StatementResetInfo readStatementResetInfo(Connection connection)
{
return (new StatementResetInfo(null, 0));
}
/*******************************************************************************
** Base tables with H2's row-count estimate and DISK_SPACE_USED (table data
** only: H2 cannot size indexes, so totalBytes equals tableBytes).
*******************************************************************************/
@Override
public List<TableStats> readTables(Connection connection) throws SQLException
{
List<TableStats> result = new ArrayList<>();
List<String[]> tables = read(connection, "SELECT TABLE_SCHEMA, TABLE_NAME, ROW_COUNT_ESTIMATE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA <> 'INFORMATION_SCHEMA' ORDER BY TABLE_SCHEMA, TABLE_NAME",
row -> new String[] {row.getString(1), row.getString(2), row.getString(3)});
try(PreparedStatement size = connection.prepareStatement(DISK_SPACE))
{
size.setQueryTimeout(30);
for(String[] table : tables)
{
Map<String, Long> counters = new LinkedHashMap<>();
counters.put("nLiveTup", table[2] == null ? 0L : Long.parseLong(table[2]));
Long bytes = diskSpace(size, quote(table[0]) + "." + quote(table[1]));
if(bytes != null)
{
counters.put("tableBytes", bytes);
counters.put("totalBytes", bytes);
}
result.add(new TableStats(table[0], table[1], hash("TABLE:" + table[0] + "." + table[1]) & Long.MAX_VALUE, counters, Map.of()));
}
}
return (result);
}
/*******************************************************************************
** Index definitions and ordered key columns. H2 has no index usage or size
** statistics; constraint-generated indexes are marked USING CONSTRAINT so
** they are never proposed for removal.
*******************************************************************************/
@Override
public List<IndexStats> readIndexes(Connection connection) throws SQLException
{
Map<String, IndexBuilder> indexes = new LinkedHashMap<>();
String sql = "SELECT i.INDEX_SCHEMA, i.TABLE_NAME, i.INDEX_NAME, i.INDEX_TYPE_NAME, i.IS_GENERATED, c.COLUMN_NAME, c.ORDERING_SPECIFICATION FROM INFORMATION_SCHEMA.INDEXES i "
+ "JOIN INFORMATION_SCHEMA.INDEX_COLUMNS c ON c.INDEX_SCHEMA = i.INDEX_SCHEMA AND c.TABLE_NAME = i.TABLE_NAME AND c.INDEX_NAME = i.INDEX_NAME "
+ "WHERE i.INDEX_SCHEMA <> 'INFORMATION_SCHEMA' ORDER BY i.INDEX_SCHEMA, i.TABLE_NAME, i.INDEX_NAME, c.ORDINAL_POSITION";
read(connection, sql, row ->
{
IndexBuilder index = indexes.computeIfAbsent(row.getString(1) + "." + row.getString(3), key -> new IndexBuilder());
index.schema = row.getString(1);
index.table = row.getString(2);
index.name = row.getString(3);
index.type = row.getString(4) == null ? "" : row.getString(4).toUpperCase(Locale.ROOT);
index.generated = row.getBoolean(5);
String order = "DESC".equalsIgnoreCase(row.getString(7)) ? " DESC" : "";
index.columns.add(row.getString(6) + order);
index.quoted.add(quote(row.getString(6)) + order);
return (null);
});
List<IndexStats> result = new ArrayList<>();
for(IndexBuilder index : indexes.values())
{
boolean primary = index.type.contains("PRIMARY");
boolean unique = primary || index.type.contains("UNIQUE");
String columns = String.join(", ", index.columns);
String keys = String.join(", ", index.quoted);
String definition = primary ? "ALTER TABLE " + quote(index.schema) + "." + quote(index.table) + " ADD PRIMARY KEY (" + keys + ")"
: "CREATE " + (unique ? "UNIQUE " : "") + "INDEX " + quote(index.name) + " ON " + quote(index.schema) + "." + quote(index.table) + (index.generated ? " USING CONSTRAINT" : "") + " (" + keys + ")";
result.add(new IndexStats(index.schema, index.table, index.name, hash("INDEX:" + index.schema + "." + index.name) & Long.MAX_VALUE, definition, columns, unique, primary, true, 0, 0, 0, 0, 0));
}
return (result);
}
/*******************************************************************************
** Other sessions in PostgreSQL's state vocabulary: RUNNING is active, BLOCKED
** is active waiting on a Lock, SLEEP with uncommitted changes is idle in
** transaction. H2 has no transaction start, so an idle transaction's age is
** the time since it went idle (a lower bound).
*******************************************************************************/
@Override
public List<SessionInfo> readSessions(Connection connection) throws SQLException
{
Instant now = Instant.now();
String sql = "SELECT SESSION_ID, USER_NAME, CLIENT_INFO, CLIENT_ADDR, SERVER, SESSION_STATE, CONTAINS_UNCOMMITTED, BLOCKER_ID, EXECUTING_STATEMENT, EXECUTING_STATEMENT_START, SLEEP_SINCE "
+ "FROM INFORMATION_SCHEMA.SESSIONS WHERE SESSION_ID <> SESSION_ID() ORDER BY SESSION_ID";
return (read(connection, sql, row ->
{
String state = state(row.getString("SESSION_STATE"), row.getBoolean("CONTAINS_UNCOMMITTED"));
boolean blocked = "BLOCKED".equalsIgnoreCase(row.getString("SESSION_STATE"));
Instant queryStart = instant(row, "EXECUTING_STATEMENT_START");
Instant sleepSince = instant(row, "SLEEP_SINCE");
boolean active = "active".equals(state);
String query = row.getString("EXECUTING_STATEMENT");
Integer blocker = row.getObject("BLOCKER_ID") == null ? null : row.getInt("BLOCKER_ID");
return (new SessionInfo(row.getInt("SESSION_ID"), row.getString("USER_NAME"), row.getString("CLIENT_INFO"), row.getString("CLIENT_ADDR"), state,
blocked ? "Lock" : null, blocked ? "row lock" : null, row.getString("SERVER") == null ? "embedded" : row.getString("SERVER"),
null, queryStart, active ? queryStart : sleepSince, active ? seconds(queryStart, now) : null,
state.startsWith("idle in transaction") ? seconds(sleepSince, now) : null, blocker == null ? List.of() : List.of(blocker), query == null ? null : hash(query), query));
}));
}
/*******************************************************************************
** Every visible session, including this one, by user, client and state.
*******************************************************************************/
@Override
public List<ConnectionCount> readConnections(Connection connection) throws SQLException
{
Map<List<String>, Integer> counts = new LinkedHashMap<>();
read(connection, "SELECT USER_NAME, CLIENT_INFO, SESSION_STATE, CONTAINS_UNCOMMITTED FROM INFORMATION_SCHEMA.SESSIONS ORDER BY SESSION_ID", row ->
counts.merge(Arrays.asList(row.getString(1), row.getString(2), state(row.getString(3), row.getBoolean(4))), 1, Integer::sum));
List<ConnectionCount> result = new ArrayList<>();
counts.forEach((key, count) -> result.add(new ConnectionCount(key.get(0), key.get(1), key.get(2), count)));
return (result);
}
/*******************************************************************************
** Operational, non-secret settings plus the QUERY_STATISTICS state.
*******************************************************************************/
@Override
public Map<String, String> readSettings(Connection connection) throws SQLException
{
Map<String, String> result = new LinkedHashMap<>();
read(connection, "SELECT SETTING_NAME, SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS", row -> SETTINGS.contains(row.getString(1)) ? result.put(row.getString(1), row.getString(2)) : null);
result.put("QUERY_STATISTICS", queryStatistics(connection) ? "TRUE" : "FALSE");
return (result);
}
/*******************************************************************************
** Health and setup next steps for H2's access codes.
*******************************************************************************/
@Override
public String remedy(String statsAccess, String databaseUser)
{
return (switch(statsAccess == null ? "" : statsAccess)
{
case "FULL" -> "Full statistics access";
case NO_QUERY_STATISTICS -> "Turn on H2 query statistics: run Enable statement statistics on Settings, or add ;QUERY_STATISTICS=TRUE to the JDBC URL";
case NOT_ADMIN -> "Make " + databaseUser + " an H2 admin (ALTER USER " + databaseUser + " ADMIN TRUE) to see every session";
default -> "Use H2 2.0 or newer";
});
}
/*******************************************************************************
** Guided setup rows: query statistics and the admin role.
*******************************************************************************/
@Override
public List<SetupCheck> setupChecks(String statsAccess, String message)
{
return (List.of(new SetupCheck("H2 QUERY_STATISTICS", statsAccess, "Run Enable statement statistics on Settings (admin user), or add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 to the JDBC URL; then run Collect now."),
new SetupCheck("H2 admin user", statsAccess, message == null ? "Use an H2 admin user to see every session and blocking." : message)));
}
/*******************************************************************************
** SET QUERY_STATISTICS TRUE (admin only), raising H2's default of 100 kept
** statements: H2 evicts a third of them, oldest first, whenever the cap is
** passed, and an application easily runs more distinct statements. H2 does
** not persist either setting: they last until the database closes, so also
** put them in the JDBC URL. An existing setup is left untouched.
*******************************************************************************/
@Override
public String enableStatementStatistics(Connection connection) throws SQLException
{
if(queryStatistics(connection))
{
return ("H2 query statistics are already on.");
}
try(Statement statement = connection.createStatement())
{
statement.execute("SET QUERY_STATISTICS_MAX_ENTRIES " + MAX_ENTRIES);
statement.execute("SET QUERY_STATISTICS TRUE");
}
return ("H2 query statistics are on (up to " + MAX_ENTRIES + " statements) until the database closes. Add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=" + MAX_ENTRIES + " to the JDBC URL to keep them on after a restart.");
}
/*******************************************************************************
** EXPLAIN (not executed) or EXPLAIN ANALYZE (executed with bound values).
** H2 has no READ ONLY transaction, so both refuse more than one statement
** and any SQL naming a side-effecting function, DDL/DML keyword or
** user-defined alias; ANALYZE additionally requires a single SELECT/WITH
** and a non-admin user. The connection is marked read-only (a hint in H2)
** and the transaction always rolls back.
*******************************************************************************/
@Override
public String explainText(Connection connection, String sql, List<String> parameters, boolean analyze, int timeoutMs) throws SQLException
{
List<String> words = words(sql);
if(statements(sql) > 1)
{
throw (new SQLException("Query plans on H2 accept exactly one statement.", "42000"));
}
List<String> aliases = read(connection, "SELECT UPPER(ROUTINE_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA <> 'INFORMATION_SCHEMA'", row -> row.getString(1));
for(String word : words)
{
if(DENIED.contains(word) || aliases.contains(word))
{
throw (new SQLException("Query plans on H2 refuse " + word + ": H2 cannot run it read-only.", "42000"));
}
}
if(analyze)
{
if(words.isEmpty() || !Set.of("SELECT", "WITH").contains(words.getFirst()))
{
throw (new SQLException("EXPLAIN ANALYZE on H2 accepts a single SELECT only.", "42000"));
}
if(admin(connection))
{
throw (new SQLException("EXPLAIN ANALYZE on H2 is disabled for admin users (it executes the query and H2 has no read-only transactions). Use PLAN, or connect as a non-admin user.", "42000"));
}
}
boolean autoCommit = connection.getAutoCommit();
boolean readOnly = connection.isReadOnly();
try
{
connection.setReadOnly(true);
connection.setAutoCommit(false);
if(!analyze)
{
////////////////////////////////////////////////////////////////
// a plain statement plans ?n slots without values (generic) //
////////////////////////////////////////////////////////////////
try(Statement statement = connection.createStatement())
{
statement.setQueryTimeout(Math.max(1, (timeoutMs + 999) / 1000));
return (plan(statement.executeQuery("EXPLAIN " + sql)));
}
}
try(PreparedStatement statement = connection.prepareStatement("EXPLAIN ANALYZE " + sql))
{
statement.setQueryTimeout(Math.max(1, (timeoutMs + 999) / 1000));
for(int i = 0; i < parameters.size(); i++)
{
statement.setString(i + 1, parameters.get(i));
}
return (plan(statement.executeQuery()));
}
}
finally
{
try
{
connection.rollback();
}
finally
{
connection.setAutoCommit(autoCommit);
connection.setReadOnly(readOnly);
}
}
}
/*******************************************************************************
** Upper-case words outside string literals and comments; double-quoted
** identifiers count as words, since H2 resolves quoted function names.
*******************************************************************************/
static List<String> words(String sql)
{
String code = code(sql);
List<String> words = new ArrayList<>();
Matcher matcher = WORD.matcher(code);
while(matcher.find())
{
words.add(matcher.group().toUpperCase(Locale.ROOT));
}
return (words);
}
/*******************************************************************************
** Statements separated by semicolons outside literals and comments.
*******************************************************************************/
static int statements(String sql)
{
int count = 0;
for(String part : code(sql).split(";"))
{
count += part.isBlank() ? 0 : 1;
}
return (count);
}
/*******************************************************************************
** SQL with string literals and comments blanked, quoted identifiers unquoted.
*******************************************************************************/
private static String code(String sql)
{
return (sql.replaceAll("(?s)/\\*.*?\\*/", " ").replaceAll("--[^\n]*", " ").replaceAll("'(?:[^']|'')*'", " ").replace('"', ' '));
}
/*******************************************************************************
** Plan rows joined as text; closes the result.
*******************************************************************************/
private static String plan(ResultSet rows) throws SQLException
{
try(rows)
{
StringBuilder plan = new StringBuilder();
while(rows.next())
{
plan.append(plan.isEmpty() ? "" : "\n").append(rows.getString(1));
}
return (plan.toString());
}
}
/*******************************************************************************
** QUERY_STATISTICS is not exposed in SETTINGS; when it is on, the probe just
** run is always recorded, and when it is off the view is empty.
*******************************************************************************/
boolean queryStatistics(Connection connection) throws SQLException
{
read(connection, PROBE, row -> null);
return (scalar(connection, "SELECT COUNT(*) FROM INFORMATION_SCHEMA.QUERY_STATISTICS") > 0);
}
/*******************************************************************************
** Non-admin users see only their own USERS row and session.
*******************************************************************************/
private boolean admin(Connection connection) throws SQLException
{
return (!read(connection, "SELECT IS_ADMIN FROM INFORMATION_SCHEMA.USERS WHERE USER_NAME = CURRENT_USER", row -> row.getBoolean(1)).stream().filter(Boolean::booleanValue).toList().isEmpty());
}
/*******************************************************************************
** major*1000000 + minor*1000 + build, e.g. 2004240 for 2.4.240.
*******************************************************************************/
private int version(Connection connection) throws SQLException
{
String[] parts = read(connection, "SELECT H2VERSION()", row -> row.getString(1)).getFirst().split("[.\\s]");
int version = 0;
for(int i = 0; i < 3; i++)
{
version = version * 1000 + (i < parts.length && parts[i].matches("\\d+") ? Integer.parseInt(parts[i]) : 0);
}
return (version);
}
/*******************************************************************************
** PostgreSQL-compatible state names used by collection and presentation.
*******************************************************************************/
static String state(String h2State, boolean uncommitted)
{
String value = h2State == null ? "" : h2State.toUpperCase(Locale.ROOT);
return (switch(value)
{
case "RUNNING", "BLOCKED" -> "active";
case "SLEEP" -> uncommitted ? "idle in transaction" : "idle";
default -> value.toLowerCase(Locale.ROOT);
});
}
/*******************************************************************************
** Stable 64-bit identity for text (statement, table or index key).
*******************************************************************************/
static long hash(String text)
{
try
{
return (ByteBuffer.wrap(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))).getLong());
}
catch(NoSuchAlgorithmException e)
{
throw (new IllegalStateException("SHA-256 is required", e));
}
}
/*******************************************************************************
** DISK_SPACE_USED for one quoted table; null when H2 cannot size it.
*******************************************************************************/
private static Long diskSpace(PreparedStatement size, String table)
{
try
{
size.setString(1, table);
try(ResultSet rows = size.executeQuery())
{
return (rows.next() ? rows.getLong(1) : null);
}
}
catch(SQLException e)
{
return (null);
}
}
/*******************************************************************************
** Quoted identifier.
*******************************************************************************/
private static String quote(String identifier)
{
return ("\"" + identifier.replace("\"", "\"\"") + "\"");
}
/*******************************************************************************
** Milliseconds as a 3-decimal BigDecimal.
*******************************************************************************/
private static BigDecimal millis(double value)
{
return (BigDecimal.valueOf(value).setScale(3, RoundingMode.HALF_UP));
}
/*******************************************************************************
** Elapsed seconds, or null without a start.
*******************************************************************************/
private static BigDecimal seconds(Instant start, Instant now)
{
return (start == null ? null : BigDecimal.valueOf(Math.max(0, Duration.between(start, now).toMillis())).divide(BigDecimal.valueOf(1000), 3, RoundingMode.HALF_UP));
}
/*******************************************************************************
** Numeric setting value, or null.
*******************************************************************************/
private static Long parseLong(String value)
{
return (value != null && value.matches("\\d+") ? Long.parseLong(value) : null);
}
/*******************************************************************************
** Index rows folded per index.
*******************************************************************************/
private static class IndexBuilder
{
private String schema;
private String table;
private String name;
private String type;
private boolean generated;
private final List<String> columns = new ArrayList<>();
private final List<String> quoted = new ArrayList<>();
}
}
@@ -0,0 +1,101 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** Bounded-time JDBC reads shared by statistics providers.
*******************************************************************************/
final class JdbcReads
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private JdbcReads()
{
}
/*******************************************************************************
** Execute one bounded-time read and close JDBC resources.
*******************************************************************************/
static <T> List<T> read(Connection connection, String sql, RowReader<T> mapper) throws SQLException
{
List<T> result = new ArrayList<>();
try(Statement statement = connection.createStatement())
{
statement.setQueryTimeout(30);
try(ResultSet rows = statement.executeQuery(sql))
{
while(rows.next())
{
result.add(mapper.read(rows));
}
}
}
return (result);
}
/*******************************************************************************
** A single numeric read.
*******************************************************************************/
static long scalar(Connection connection, String sql) throws SQLException
{
return (read(connection, sql, row -> row.getLong(1)).getFirst());
}
/*******************************************************************************
** Timestamp conversion preserves null values.
*******************************************************************************/
static Instant instant(ResultSet row, String column) throws SQLException
{
Timestamp timestamp = row.getTimestamp(column);
return (timestamp == null ? null : timestamp.toInstant());
}
/*******************************************************************************
** Checked JDBC row mapping.
*******************************************************************************/
interface RowReader<T>
{
/*******************************************************************************
** Map one row.
*******************************************************************************/
T read(ResultSet row) throws SQLException;
}
}
@@ -24,13 +24,16 @@ package com.kingsrook.qbits.dbperformance.provider;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.instant;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.read;
import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.scalar;
/*******************************************************************************
@@ -85,7 +88,60 @@ public class PostgresStatsProvider implements DbStatsProvider
}
}
}
return (new ProviderCapabilities(version >= 130000, version, installed, privileged, access, message));
EnumSet<StatsFeature> available = EnumSet.copyOf(features());
if(!installed)
{
available.removeAll(EnumSet.of(StatsFeature.STATEMENTS, StatsFeature.STATEMENT_IO, StatsFeature.STATEMENT_RESET));
}
if(version < 140000)
{
available.remove(StatsFeature.STATEMENT_RESET);
}
if(version < 160000)
{
available.remove(StatsFeature.EXPLAIN_GENERIC);
}
return (new ProviderCapabilities(version >= 130000, version, installed, privileged, access, message, version >= 130000 ? available : EnumSet.noneOf(StatsFeature.class)));
}
/*******************************************************************************
** Every area: the cumulative statistics views cover them all.
*******************************************************************************/
@Override
public Set<StatsFeature> features()
{
return (StatsFeature.of(StatsFeature.values()));
}
/*******************************************************************************
** Role and extension prerequisites for each access code.
*******************************************************************************/
@Override
public String remedy(String statsAccess, String databaseUser)
{
return (switch(statsAccess == null ? "" : statsAccess)
{
case "FULL" -> "Full statistics access";
case "NO_READ_ALL_STATS" -> "grant pg_read_all_stats to " + databaseUser;
case "NO_PG_STAT_STATEMENTS" -> "Enable pg_stat_statements in shared_preload_libraries and create the extension";
default -> "Use a supported PostgreSQL version (13 or newer) and statistics provider";
});
}
/*******************************************************************************
** Guided setup rows: the extension and the statistics role.
*******************************************************************************/
@Override
public List<SetupCheck> setupChecks(String statsAccess, String message)
{
return (List.of(new SetupCheck("pg_stat_statements", statsAccess, "Preload and create pg_stat_statements; then run Collect now."),
new SetupCheck("pg_read_all_stats", statsAccess, message == null ? "Grant pg_read_all_stats to the application's database role." : message)));
}
@@ -237,25 +293,6 @@ public class PostgresStatsProvider implements DbStatsProvider
}
/*******************************************************************************
** A single numeric read.
*******************************************************************************/
private long scalar(Connection connection, String sql) throws SQLException
{
return (read(connection, sql, row -> row.getLong(1)).getFirst());
}
/*******************************************************************************
** Timestamp conversion preserves null reset/maintenance values.
*******************************************************************************/
private static Instant instant(ResultSet row, String column) throws SQLException
{
Timestamp timestamp = row.getTimestamp(column);
return (timestamp == null ? null : timestamp.toInstant());
}
/*******************************************************************************
** Copy cumulative counters to the entity's camelCase names.
*******************************************************************************/
@@ -282,37 +319,4 @@ public class PostgresStatsProvider implements DbStatsProvider
}
return (value.toString());
}
/*******************************************************************************
** Execute one bounded-time read and close JDBC resources.
*******************************************************************************/
private static <T> List<T> read(Connection connection, String sql, RowReader<T> mapper) throws SQLException
{
List<T> result = new ArrayList<>();
try(Statement statement = connection.createStatement())
{
statement.setQueryTimeout(30);
try(ResultSet rows = statement.executeQuery(sql))
{
while(rows.next())
{
result.add(mapper.read(rows));
}
}
}
return (result);
}
/*******************************************************************************
** Checked JDBC row mapping.
*******************************************************************************/
private interface RowReader<T>
{
/*******************************************************************************
** Map one row.
*******************************************************************************/
T read(ResultSet row) throws SQLException;
}
}
@@ -21,11 +21,44 @@
package com.kingsrook.qbits.dbperformance.provider;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
/*******************************************************************************
** ProviderCapabilities read from database statistics; no persistence or side effects.
** What one provider can read on one connection; no persistence or side effects.
** {@code statsAccess} is a stable code persisted on settings (FULL, or the most
** important missing prerequisite); {@code message} is the provider's own
** next step; {@code features} lists the statistics areas available right now.
*******************************************************************************/
public record ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message)
public record ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message, Set<StatsFeature> features)
{
/*******************************************************************************
** Defensive copy.
*******************************************************************************/
public ProviderCapabilities
{
features = features == null || features.isEmpty() ? Set.of() : Collections.unmodifiableSet(EnumSet.copyOf(features));
}
/*******************************************************************************
** Pre-feature form: a supported provider is assumed to offer every area.
*******************************************************************************/
public ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message)
{
this(supported, version, statements, readAllStats, statsAccess, message, supported ? EnumSet.allOf(StatsFeature.class) : Set.of());
}
/*******************************************************************************
** Whether an area is available on this connection.
*******************************************************************************/
public boolean has(StatsFeature feature)
{
return (features.contains(feature));
}
}
@@ -0,0 +1,31 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
/*******************************************************************************
** One guided-setup row: the prerequisite, its observed state and the next step.
*******************************************************************************/
public record SetupCheck(String check, String state, String nextStep)
{
}
@@ -0,0 +1,117 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.provider;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
/*******************************************************************************
** One area of database statistics. A provider declares which areas its vendor
** can supply ({@link DbStatsProvider#features()}) and which are available on a
** given connection ({@link ProviderCapabilities#features()}); presentation and
** analysis degrade per area instead of per vendor.
*******************************************************************************/
public enum StatsFeature
{
/** Transaction, block, temp-file and deadlock counters for the database. */
DATABASE_COUNTERS("Database counters"),
/** Buffer-cache hit and read counters (cache hit ratio). */
CACHE_COUNTERS("Cache hit ratio"),
/** Database size on disk. */
DATABASE_SIZE("Database size"),
/** Configured connection limit. */
MAX_CONNECTIONS("Connection limit"),
/** Cumulative per-statement calls, time and rows. */
STATEMENTS("Statement statistics"),
/** Per-statement shared-block and temp-block I/O. */
STATEMENT_IO("Statement I/O"),
/** A server-side reset time for statement statistics. */
STATEMENT_RESET("Statement reset time"),
/** Live row estimates per table. */
TABLE_ROWS("Table row estimates"),
/** Table, index and toast sizes. */
TABLE_SIZES("Table sizes"),
/** Sequential/index scan and tuple write counters per table. */
TABLE_ACCESS("Table access counters"),
/** Dead tuples, vacuum/analyze times and freeze age. */
TABLE_MAINTENANCE("Table maintenance"),
/** Index definitions and key columns. */
INDEX_DEFINITIONS("Index definitions"),
/** Per-index scan counters. */
INDEX_USAGE("Index usage"),
/** Per-index size. */
INDEX_SIZES("Index sizes"),
/** Live sessions with state and current statement. */
SESSIONS("Live sessions"),
/** Which session blocks which. */
BLOCKING("Blocking"),
/** Wait event type and name per session. */
WAIT_EVENTS("Wait events"),
/** Transaction start time per session. */
TRANSACTION_AGE("Transaction age"),
/** Connection counts by user, application and state. */
CONNECTIONS("Connection counts"),
/** Plans as structured JSON with costs (renderable tree). */
EXPLAIN_JSON("Structured query plans"),
/** Plans as vendor text (no cost tree). */
EXPLAIN_TEXT("Text query plans"),
/** Plans for parameterized SQL without values. */
EXPLAIN_GENERIC("Generic plans"),
/** Executing the query to measure the plan (ANALYZE). */
EXPLAIN_ANALYZE("Explain analyze");
private final String label;
/*******************************************************************************
** Constructor.
*******************************************************************************/
StatsFeature(String label)
{
this.label = label;
}
/*******************************************************************************
** Human-readable area name.
*******************************************************************************/
public String getLabel()
{
return (label);
}
/*******************************************************************************
** Immutable copy of a feature list.
*******************************************************************************/
public static Set<StatsFeature> of(StatsFeature... features)
{
EnumSet<StatsFeature> set = EnumSet.noneOf(StatsFeature.class);
Collections.addAll(set, features);
return (Collections.unmodifiableSet(set));
}
}
@@ -47,7 +47,7 @@ public class UnsupportedStatsProvider implements DbStatsProvider
@Override
public ProviderCapabilities capabilities(Connection connection)
{
return (new ProviderCapabilities(false, 0, false, false, "UNSUPPORTED", "This database vendor is not supported; PostgreSQL 13 or newer is required."));
return (new ProviderCapabilities(false, 0, false, false, "UNSUPPORTED", "This database vendor is not supported; PostgreSQL 13+ and H2 2.x are (see docs/PROVIDERS.md)."));
}
@@ -37,7 +37,10 @@ import com.kingsrook.qbits.dbperformance.api.DbPerformanceData;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.provider.ConnectionCount;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.SetupCheck;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@@ -131,9 +134,12 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
boolean attribution = settings == null ? Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()) : Boolean.TRUE.equals(settings.getValueBoolean("attributionEnabled"));
List<Map<String, Object>> rows = new ArrayList<>();
rows.add(check("Monitored backend", runtime.getConfig().getBackendName(), "Review Settings for this backend."));
rows.add(check("Provider", settings == null ? "Not detected yet" : settings.getValueString("provider"), "PostgreSQL 13+ is supported."));
rows.add(check("pg_stat_statements", access, "Preload and create pg_stat_statements; then run Collect now."));
rows.add(check("pg_read_all_stats", access, settings == null ? "Grant pg_read_all_stats to the application's database role." : settings.getValueString("statsAccessMessage")));
rows.add(check("Provider", settings == null ? "Not detected yet" : settings.getValueString("provider"), "PostgreSQL 13+ and H2 2.x are supported."));
DbStatsProvider provider = settings == null ? providerForBackend(runtime) : DbStatsProviders.forProviderName(settings.getValueString("provider"));
for(SetupCheck check : provider.setupChecks(access, settings == null ? null : settings.getValueString("statsAccessMessage")))
{
rows.add(check(check.check(), check.state(), check.nextStep()));
}
rows.add(check("Last snapshot", last == null ? "Nothing collected yet" : Duration.between(last, Instant.now()).toSeconds() + " seconds ago",
last == null ? "Run Collect now from Snapshots, or wait for the next scheduled collection." : "Collection normally runs every " + SnapshotCollector.positive(settings, "snapshotIntervalMinutes", 5) + " minutes."));
rows.add(check("Attribution", attribution ? "On" : "Off", attribution ? "Attribution starts after the next flush." : "Enable attribution in Settings and the host configuration."));
@@ -143,6 +149,24 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
/*******************************************************************************
** Before the first collection the vendor comes from the backend itself;
** an unreachable backend falls back to the unsupported checks.
*******************************************************************************/
private static DbStatsProvider providerForBackend(DbPerfRuntime runtime)
{
try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName())))
{
return (DbStatsProviders.forConnection(connection));
}
catch(Exception e)
{
return (DbStatsProviders.forProviderName(null));
}
}
/*******************************************************************************
** Plain text check row, never markup.
*******************************************************************************/
@@ -257,13 +281,29 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
List<QRecord> snapshots = data.snapshots();
List<MultiStatisticsData.StatisticsGroupData.Statistic> values = new ArrayList<>();
values.add(stat("Total connections", latest == null ? null : number(latest, "connectionsTotal")));
values.add(stat("Maximum connections", latest == null ? null : number(latest, "maxConnections")));
////////////////////////////////////////////////////////////////////////
// figures the provider does not keep (H2: limit, cache, temp, //
// deadlocks) are stored as null and left out rather than shown as 0 //
////////////////////////////////////////////////////////////////////////
boolean limit = latest == null || latest.getValue("maxConnections") != null;
boolean cache = latest == null || latest.getValue("cacheHitRatio") != null;
boolean counters = snapshots.isEmpty() || snapshots.stream().anyMatch(row -> row.getValue("deadlocks") != null || row.getValue("tempBytes") != null);
if(limit)
{
values.add(stat("Maximum connections", latest == null ? null : number(latest, "maxConnections")));
}
values.add(stat("Active", latest == null ? null : number(latest, "connectionsActive")));
values.add(stat("Idle", latest == null ? null : number(latest, "connectionsIdle")));
values.add(stat("Idle in transaction", latest == null ? null : number(latest, "connectionsIdleInTx")));
values.add(stat("Cache hit %", latest == null ? null : 100 * number(latest, "cacheHitRatio")));
values.add(stat("Temp MB (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "tempBytes")).sum() / 1048576));
values.add(stat("Deadlocks (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "deadlocks")).sum()));
if(cache)
{
values.add(stat("Cache hit %", latest == null ? null : 100 * number(latest, "cacheHitRatio")));
}
if(counters)
{
values.add(stat("Temp MB (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "tempBytes")).sum() / 1048576));
values.add(stat("Deadlocks (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "deadlocks")).sum()));
}
return (new MultiStatisticsData(null, List.of(new MultiStatisticsData.StatisticsGroupData().withHeader("Connection capacity")
.withSubheader(latest == null ? "Capacity appears after the next snapshot." : "Latest snapshot; percentages and MB rounded to whole numbers.").withStatisticList(values))));
}
@@ -355,16 +395,29 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer
*******************************************************************************/
public MultiTableData tables(DbPerformanceData data) throws QException
{
List<QRecord> scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100));
QRecord settings = data.settings();
DbStatsProvider provider = settings == null || settings.getValueString("provider") == null ? null : DbStatsProviders.forProviderName(settings.getValueString("provider"));
List<QRecord> growing = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("growthBytes7d", false)).withLimit(100));
TableData growth = table("Growth appears after table history has accumulated.", growing.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")),
"Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(),
"Table", "Size MB", "Growth MB (7 days)");
if(provider != null && !provider.features().contains(StatsFeature.TABLE_ACCESS))
{
//////////////////////////////////////////////////////////////////
// vendors without scan counters (H2) list rows and size instead //
//////////////////////////////////////////////////////////////////
List<QRecord> largest = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("nLiveTup", false)).withLimit(100));
return (new MultiTableData(List.of(table("Tables appear after the next table snapshot (normally hourly).", largest.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Rows (estimate)", row.getValue("nLiveTup"),
"Size MB", mb(row.getValue("totalBytes")))).toList(),
"Table", "Rows (estimate)", "Size MB"), growth)));
}
List<QRecord> scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100));
return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Sequential scans", row.getValue("seqScan"),
"Index scans", row.getValue("idxScan"), "Sequential share %", row.getValue("seqScanShare"))).toList(),
"Table", "Sequential scans", "Index scans", "Sequential share %"),
table("Growth appears after table history has accumulated.", growing.stream()
.map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")),
"Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(),
"Table", "Size MB", "Growth MB (7 days)"))));
"Table", "Sequential scans", "Index scans", "Sequential share %"), growth)));
}
@@ -519,7 +519,7 @@ entries:
- key: table:db_perf_settings;field:lastTableSnapshotAt
summary: "Time of the latest table and index collection pass."
- key: table:db_perf_settings;field:statsAccess
summary: "Statistics capability: FULL, reduced access, missing extension, or unsupported provider."
summary: "Statistics capability: FULL, reduced visibility, statement statistics off, or unsupported provider."
- key: table:db_perf_settings;field:statsAccessMessage
summary: "The collector explains missing capabilities and the next grant or setup step."
- key: table:db_perf_snapshot;slot:help
@@ -810,6 +810,10 @@ entries:
summary: "Collect an immediate snapshot and force table and index statistics."
content: |
Run from the Snapshots table. Requires manage settings. The Result screen reports success, partial access or collection failure. The first pass establishes counter baselines.
- key: process:dbPerfEnableStatementStatistics;slot:help
summary: "Turn on statement statistics where the database allows it at runtime."
content: |
Run from Settings. Requires manage settings. On H2 it runs SET QUERY_STATISTICS TRUE, which needs an H2 admin user and lasts until the database closes; add ;QUERY_STATISTICS=TRUE to the JDBC URL to keep it on. PostgreSQL cannot do this at runtime (pg_stat_statements needs shared_preload_libraries and CREATE EXTENSION), so the result explains the manual step. The Result screen shows the new statistics access.
- key: process:dbPerfRollupAndPurge;slot:help
summary: "Roll up completed days and remove expired details."
content: |
@@ -817,7 +821,7 @@ entries:
- key: process:dbPerfQueryPlan;slot:help
summary: "Inspect the plan of one statement, with optional read-only execution."
content: |
Choose PLAN (default) or ANALYZE. PLAN uses a generic plan for parameterized SQL on PostgreSQL 16+. ANALYZE requires explain analyze permission and exactly one SELECT with no modifying CTEs, INTO or row locks. Enter parameter values only for ANALYZE, one per line in placeholder order. Values are sent transiently; they and the ANALYZE plan are never stored. Every request is audited. Results include copyable plan text, cost, a SQL Workbench link and a literal-free assistant prompt. No model is called by this qbit.
Choose PLAN (default) or ANALYZE. PLAN uses a generic plan for parameterized SQL on PostgreSQL 16+ and H2. H2 plans are text (index choice and scan counts) without a cost tree; H2 refuses SQL naming side-effecting functions, DDL or aliases, and allows ANALYZE only for non-admin users. ANALYZE requires explain analyze permission and exactly one SELECT with no modifying CTEs, INTO or row locks. Enter parameter values only for ANALYZE, one per line in placeholder order. Values are sent transiently; they and the ANALYZE plan are never stored. Every request is audited. Results include copyable plan text, cost, a SQL Workbench link and a literal-free assistant prompt. No model is called by this qbit.
- key: widget:dbPerfSetup;slot:label
summary: "Check backend, statistics access, collection, attribution and scheduler setup."
- key: widget:dbPerfDatabaseTime;slot:label
@@ -1,5 +1,44 @@
components:
schemas:
BasicErrorResponseV1:
description: "The error body of every UI API operation (core and extensions)."
properties:
error:
description: "The error message (the same as message; kept for clients of\
\ UI API 1.0 to 1.5)."
type: "string"
code:
description: "A stable, machine-readable code. Well-known values: BAD_REQUEST\
\ (400), UNAUTHENTICATED (401),\nPERMISSION_DENIED (403), NOT_FOUND (404),\
\ CONFLICT (409), RATE_LIMITED (429), INTERNAL_ERROR (500),\nERROR (other).\
\ Extensions may add codes; treat an unknown code by the HTTP status.\
\ Since 1.6.0."
type: "string"
message:
description: "A message to show the user. Since 1.6.0."
type: "string"
fieldErrors:
description: "Errors about particular request fields, when known. Since\
\ 1.6.0."
items:
properties:
field:
description: "The field (or parameter) name."
type: "string"
message:
description: "The error about that field."
type: "string"
code:
description: "An optional machine-readable code, e.g. REQUIRED, TOO_LONG,\
\ PATTERN."
type: "string"
type: "object"
type: "array"
traceId:
description: "Identifies this request in the server logs (also the X-QQQ-Trace-Id\
\ response header); quote it when reporting a problem. Since 1.6.0."
type: "string"
type: "object"
DbPerformanceActivityV1:
properties:
blockingChains:
@@ -256,9 +295,42 @@ components:
type: "object"
securitySchemes:
sessionUuidCookie:
description: "The session cookie set by POST /qqq/v1/manageSession (web clients)."
in: "cookie"
name: "sessionUUID"
type: "apiKey"
bearerAuth:
bearerFormat: "JWT or session UUID"
description: "`Authorization: Bearer <token>`, where the token is either\n-\
\ an OAuth2 access token (a signed JWT) issued by the app's identity provider\
\ to the app's client,\n or to its native client (`values.nativeClientId`\
\ in GET /qqq/v1/metaData/authentication), for\n example from the device-authorization\
\ grant or auth-code + PKCE. It is verified against the\n provider's keys\
\ (issuer, audience, expiry, signature) and gives the same user and permissions\n\
\ as signing in on the web. When it expires (401), refresh it at the provider\
\ and retry; or\n- a session UUID returned by POST /qqq/v1/manageSession,\
\ which the server keeps refreshed."
scheme: "bearer"
type: "http"
oauth2:
description: "The app's OAuth2 / OpenID Connect provider. The real endpoints\
\ come from the provider's discovery\ndocument: `{values.baseUrl}.well-known/openid-configuration`,\
\ with values from\nGET /qqq/v1/metaData/authentication. Native clients use\
\ `values.nativeClientId` (a public client)\nwith auth-code + PKCE (loopback\
\ or custom-scheme redirect URI registered at the provider) or the\ndevice-authorization\
\ grant (`device_authorization_endpoint`, RFC 8628), then send the access\
\ token\nas `Authorization: Bearer`."
flows:
authorizationCode:
authorizationUrl: "/authorize"
scopes:
email: "Email address"
offline_access: "A refresh token, to renew the access token without signing\
\ in again"
openid: "OpenID Connect sign-in"
profile: "Name, username and groups (permissions come from the groups)"
tokenUrl: "/token"
type: "oauth2"
info:
contact:
email: "contact@kingsrook.com"
@@ -303,10 +375,37 @@ paths:
schema:
$ref: "#/components/schemas/DbPerformanceOverviewV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Database Performance overview"
tags:
- "db-performance"
/qqq/v1/qbit/db-performance/activity:
get:
description: "Requires db_performance.view and uses the caller's session."
@@ -343,10 +442,37 @@ paths:
schema:
$ref: "#/components/schemas/DbPerformanceActivityV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Database Performance activity"
tags:
- "db-performance"
/qqq/v1/qbit/db-performance/explain:
post:
description: "PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze.\
@@ -383,10 +509,37 @@ paths:
schema:
$ref: "#/components/schemas/DbPerformanceExplainResultV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
summary: "Explain a PostgreSQL query"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Explain a query (PostgreSQL JSON or H2 text plan)"
tags:
- "db-performance"
/qqq/v1/qbit/db-performance/statements/lookup:
get:
description: "Requires db_performance.view and uses the caller's session."
@@ -428,7 +581,40 @@ paths:
schema:
$ref: "#/components/schemas/DbPerformanceLookupV1"
description: "Successful Response"
404:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Not found, or not visible to this user (code NOT_FOUND)."
409:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Conflict with the current state, e.g. an edit conflict or\
\ an invalid transition (code CONFLICT)."
429:
content:
application/json:
schema:
$ref: "#/components/schemas/BasicErrorResponseV1"
description: "Too many requests (code RATE_LIMITED); retry later."
security:
- sessionUuidCookie:
- "N/A"
- bearerAuth:
- "N/A"
- oauth2:
- "openid"
- "profile"
- "email"
summary: "Database Performance lookup"
tags:
- "db-performance"
servers:
- description: "The QQQ server that serves this document (paths include /qqq/v1)."
url: "/"
tags:
- description: "Operations of the db-performance extension."
name: "db-performance"
@@ -56,7 +56,7 @@ class DbPerfApiContractTest
}
assertThat(UiApiContract.check(generated, Files.readString(path), null, 1, "mvnw -Dtest=DbPerfApiContractTest -Ddbperf.openapi.update=true test")).isEmpty();
var model = version.generateExtensionOpenAPIModel("qqq", extension);
assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> name.startsWith("DbPerformance") && name.endsWith("V1"));
assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> (name.startsWith("DbPerformance") || name.equals("BasicErrorResponseV1")) && name.endsWith("V1"));
assertThat(model.getPaths().keySet()).containsExactlyInAnyOrder("/qqq/v1/qbit/db-performance/overview", "/qqq/v1/qbit/db-performance/activity",
"/qqq/v1/qbit/db-performance/explain", "/qqq/v1/qbit/db-performance/statements/lookup");
assertThat(extension.getSemanticVersion()).isEqualTo("1.0.0");
@@ -76,7 +76,7 @@ class DbPerfHealthTest
DbPerfHealthChecks checks = checks(f, 0);
for(String access : List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "UNSUPPORTED", "UNKNOWN"))
{
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("statsAccess", access)));
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("provider", "POSTGRES").withValue("statsAccess", access)));
var outcome = checks.check("stats_access");
assertThat(outcome.getDetails().getValues()).containsEntry("statsAccess", access).hasSize(1);
assertThat(outcome.getStatus()).isEqualTo(switch(access)
@@ -91,6 +91,14 @@ class DbPerfHealthTest
assertThat(outcome.getMessage()).isEqualTo("grant pg_read_all_stats to sa");
}
}
for(String access : List.of("NO_QUERY_STATISTICS", "NOT_ADMIN"))
{
DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("provider", "H2").withValue("statsAccess", access)));
var outcome = checks.check("stats_access");
assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED);
assertThat(outcome.getDetails().getValues()).containsEntry("statsAccess", access);
assertThat(outcome.getMessage()).doesNotContain("pg_").contains(access.equals("NOT_ADMIN") ? "ADMIN TRUE" : "QUERY_STATISTICS=TRUE");
}
for(int total : List.of(79, 80, 94, 95))
{
DbPerfStore.update("db_perf_snapshot", List.of(new QRecord().withValue("id", f.snapshot.getValueInteger("id")).withValue("connectionsTotal", total)));
@@ -33,6 +33,7 @@ import com.kingsrook.qbits.dbperformance.api.OverviewSpec;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainResult;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess;
import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer;
@@ -103,7 +104,7 @@ class DbPerfPresentationTest
assertThat(new DbPerfWidgetRenderer().liveActivity(data).getNoRowsFoundHTML()).isEqualTo("No long-running or blocked sessions right now");
assertThat(new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getNoRowsFoundHTML()).isEqualTo("Attribution starts after the next flush");
assertThat(new DbPerfWidgetRenderer().connectionStatistics(data).getType()).isEqualTo("multiStatistics");
assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isEmpty();
assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isNotEmpty().allSatisfy(row -> assertThat(row).containsKeys("Application", "User", "State", "Connections"));
QContext.setQSession(new QSession());
for(String name : DbPerfWidgets.HOME)
{
@@ -235,7 +236,9 @@ class DbPerfPresentationTest
void explainAuditAndHandler() throws Exception
{
ExplainService service = new ExplainService(runtime);
assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, "select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("PostgreSQL");
ExplainResult h2 = service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, "select current_date", ExplainRequest.Mode.PLAN, List.of()));
assertThat(h2.planJson()).isNull();
assertThat(h2.planText()).containsIgnoringCase("CURRENT_DATE");
assertThatThrownBy(() -> service.explain(null)).hasMessageContaining("Choose PLAN");
assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, null, ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("either");
QContext.setQSession(new QSession());
@@ -0,0 +1,173 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import com.kingsrook.qbits.dbperformance.analyze.AnalyzeService;
import com.kingsrook.qbits.dbperformance.api.DbPerformanceData;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.explain.ExplainRequest;
import com.kingsrook.qbits.dbperformance.explain.ExplainResult;
import com.kingsrook.qbits.dbperformance.explain.ExplainService;
import com.kingsrook.qbits.dbperformance.processes.DbPerfEnableStatisticsStep;
import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** The qbit end to end on an H2 backend: collection, the enable action,
** feature-gated analysis, setup rows and text plans.
*******************************************************************************/
class H2CollectionTest
{
/*******************************************************************************
** Two collections store H2 statements, tables, indexes and connections.
*******************************************************************************/
@Test
void collectAnalyzeAndExplain() throws Exception
{
DbPerfRuntime runtime = runtime(";QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000");
String backend = runtime.getConfig().getBackendName();
workload(runtime);
QRecord first = new SnapshotCollector(runtime).collect(backend, true);
assertThat(first.getValueString("status")).as(first.getValueString("errorMessage")).isEqualTo("OK");
workload(runtime);
QRecord second = new SnapshotCollector(runtime).collect(backend, true);
assertThat(second.getValueString("status")).as(second.getValueString("errorMessage")).isEqualTo("OK");
assertThat(second.getValueInteger("intervalSeconds")).isNotNull();
assertThat(second.getValue("cacheHitRatio")).isNull();
assertThat(second.getValue("maxConnections")).isNull();
assertThat(second.getValueInteger("connectionsTotal")).isPositive();
QRecord settings = new DbPerfSettingsService(runtime).ensure(backend);
assertThat(settings.getValueString("provider")).isEqualTo("H2");
assertThat(settings.getValueString("statsAccess")).isEqualTo("FULL");
List<QRecord> statements = DbPerfStore.query("db_perf_statement", DbPerfStore.eq("backendName", backend));
assertThat(statements).anySatisfy(row ->
{
assertThat(row.getValueString("queryText")).containsIgnoringCase("h2_probe");
assertThat(row.getValueLong("lastCalls")).isGreaterThanOrEqualTo(6);
});
assertThat(DbPerfStore.query("db_perf_query_snapshot", new QQueryFilter())).anySatisfy(row -> assertThat(row.getValueLong("calls")).isEqualTo(3));
assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend)))
.anySatisfy(row -> assertThat(row.getValueString("tableName")).isEqualToIgnoringCase("h2_probe"));
assertThat(DbPerfStore.query("db_perf_index_stat", DbPerfStore.eq("backendName", backend)))
.anySatisfy(row -> assertThat(row.getValueString("indexName")).isEqualToIgnoringCase("h2_probe_name"));
Map<String, String> results = new AnalyzeService(runtime).run(backend).stream().collect(Collectors.toMap(AnalyzeService.Result::ruleCode, r -> String.valueOf(r.error())));
assertThat(results).containsKeys("SLOW_QUERY", "DUPLICATE_INDEX", "IDLE_IN_TRANSACTION", "N_PLUS_ONE")
.doesNotContainKeys("UNUSED_INDEX", "MISSING_INDEX", "CACHE_HIT_LOW", "BLOAT", "VACUUM_STALE", "DEADLOCKS", "TEMP_SPILL", "POOL_SIZING");
assertThat(results.values()).containsOnly("null");
assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isNotEmpty();
assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))).allSatisfy(row -> assertThat(row.getValue("seqScanShare")).isNull());
assertThat(new DbPerfWidgetRenderer().tables(new DbPerformanceData(runtime)).getTableDataList().getFirst().getColumns()).extracting(c -> c.getHeader())
.containsExactly("Table", "Rows (estimate)", "Size MB");
assertThat(new DbPerfWidgetRenderer().connectionStatistics(new DbPerformanceData(runtime)).getStatisticsGroupData().getFirst().getStatisticList())
.extracting(statistic -> statistic.getLabel()).contains("Total connections", "Idle in transaction").doesNotContain("Maximum connections", "Cache hit %", "Deadlocks (24h)");
assertThat(new DbPerfWidgetRenderer().setup(runtime, new DbPerformanceData(runtime)).getRows()).extracting(row -> row.get("Check"))
.contains("H2 QUERY_STATISTICS", "H2 admin user").doesNotContain("pg_read_all_stats");
ExplainService explain = new ExplainService(runtime);
ExplainResult plan = explain.explain(new ExplainRequest(backend, null, "select * from h2_probe where name = ?", ExplainRequest.Mode.PLAN, List.of()));
assertThat(plan.mode()).isEqualTo("GENERIC");
assertThat(plan.planJson()).isNull();
assertThat(plan.planText()).containsIgnoringCase("h2_probe_name");
assertThat(plan.assistantPrompt()).contains("database query plan");
assertThatThrownBy(() -> explain.explain(new ExplainRequest(backend, null, "select * from h2_probe where name = $1", ExplainRequest.Mode.ANALYZE, List.of("secret-value"))))
.hasMessageContaining("ANALYZE is not available");
assertThatThrownBy(() -> explain.explain(new ExplainRequest(backend, null, "select file_write('x', '/tmp/p') from h2_probe", ExplainRequest.Mode.PLAN, List.of())))
.hasMessageContaining("FILE_WRITE");
}
/*******************************************************************************
** Without QUERY_STATISTICS the snapshot is partial until the action runs.
*******************************************************************************/
@Test
void enableActionTurnsOnStatistics() throws Exception
{
DbPerfRuntime runtime = runtime("");
String backend = runtime.getConfig().getBackendName();
QRecord partial = new SnapshotCollector(runtime).collect(backend, true);
assertThat(partial.getValueString("status")).isEqualTo("PARTIAL");
assertThat(partial.getValueString("errorMessage")).contains("QUERY_STATISTICS");
assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))).isNotEmpty();
RunBackendStepOutput output = new RunBackendStepOutput();
new DbPerfEnableStatisticsStep().run(new RunBackendStepInput(), output);
assertThat(String.valueOf(output.getValue("processResults"))).contains("FULL");
assertThat(new DbPerfSettingsService(runtime).ensure(backend).getValueString("statsAccess")).isEqualTo("FULL");
workload(runtime);
assertThat(new SnapshotCollector(runtime).collect(backend, false).getValueString("status")).isEqualTo("OK");
}
/*******************************************************************************
** QQQ runtime on a fresh H2 backend with the given URL options.
*******************************************************************************/
private static DbPerfRuntime runtime(String options) throws Exception
{
String name = "dbperfh2" + UUID.randomUUID().toString().replace("-", "");
RDBMSBackendMetaData backend = new RDBMSBackendMetaData().withName(name).withVendor("h2").withHostName("mem").withDatabaseName(name).withUsername("sa");
backend.setJdbcUrl("jdbc:h2:mem:" + name + ";CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1;MODE=PostgreSQL" + options);
return (DbPerfRuntime.of(DbPerfTestSupport.instance(backend)));
}
/*******************************************************************************
** A table with a duplicate-prone index and three identical lookups.
*******************************************************************************/
private static void workload(DbPerfRuntime runtime) throws Exception
{
RDBMSBackendMetaData backend = (RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName());
try(Connection connection = DriverManager.getConnection(backend.getJdbcUrl(), "sa", ""); Statement statement = connection.createStatement())
{
statement.execute("CREATE TABLE IF NOT EXISTS h2_probe (id INT PRIMARY KEY, name VARCHAR(40))");
statement.execute("CREATE INDEX IF NOT EXISTS h2_probe_name ON h2_probe (name)");
statement.execute("MERGE INTO h2_probe KEY (id) SELECT x, 'n' || x FROM SYSTEM_RANGE(1, 20)");
for(int i = 0; i < 3; i++)
{
statement.executeQuery("SELECT * FROM h2_probe WHERE name = 'n7'").close();
}
}
}
}
@@ -0,0 +1,366 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import com.kingsrook.qbits.dbperformance.provider.ConnectionCount;
import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders;
import com.kingsrook.qbits.dbperformance.provider.H2StatsProvider;
import com.kingsrook.qbits.dbperformance.provider.IndexStats;
import com.kingsrook.qbits.dbperformance.provider.PostgresStatsProvider;
import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities;
import com.kingsrook.qbits.dbperformance.provider.SessionInfo;
import com.kingsrook.qbits.dbperformance.provider.StatementStats;
import com.kingsrook.qbits.dbperformance.provider.StatsFeature;
import com.kingsrook.qbits.dbperformance.provider.TableStats;
import com.kingsrook.qbits.dbperformance.provider.UnsupportedStatsProvider;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** H2StatsProvider against real in-memory H2 databases: every read returns
** rows produced by activity in the test, never fixtures.
*******************************************************************************/
class H2StatsProviderTest
{
private final H2StatsProvider provider = new H2StatsProvider();
/*******************************************************************************
** The registry maps JDBC product names and saved names to providers.
*******************************************************************************/
@Test
void registryChoosesByProductName() throws Exception
{
assertThat(DbStatsProviders.forProductName("H2")).isInstanceOf(H2StatsProvider.class);
assertThat(DbStatsProviders.forProductName("PostgreSQL")).isInstanceOf(PostgresStatsProvider.class);
assertThat(DbStatsProviders.forProductName("MySQL")).isInstanceOf(UnsupportedStatsProvider.class);
assertThat(DbStatsProviders.forProductName(null)).isInstanceOf(UnsupportedStatsProvider.class);
assertThat(DbStatsProviders.forProviderName("H2")).isInstanceOf(H2StatsProvider.class);
assertThat(DbStatsProviders.forProviderName("POSTGRES")).isInstanceOf(PostgresStatsProvider.class);
assertThat(DbStatsProviders.forProviderName(null)).isInstanceOf(UnsupportedStatsProvider.class);
try(Connection connection = open(false, ""))
{
assertThat(DbStatsProviders.forConnection(connection)).isInstanceOf(H2StatsProvider.class);
}
assertThat(DbStatsProviders.forProductName("ExampleDB")).isInstanceOf(UnsupportedStatsProvider.class);
DbStatsProviders.register("ExampleDB", H2StatsProvider::new);
assertThat(DbStatsProviders.forProductName("exampledb")).isInstanceOf(H2StatsProvider.class);
assertThat(new UnsupportedStatsProvider().capabilities(null).features()).isEmpty();
}
/*******************************************************************************
** Every read returns real rows after activity, in default and MySQL mode.
*******************************************************************************/
@ParameterizedTest
@ValueSource(strings = {"", ";MODE=MySQL", ";MODE=PostgreSQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE"})
void readsReturnRealRowsAfterActivity(String mode) throws Exception
{
String url = url(true, mode);
try(Connection connection = DriverManager.getConnection(url, "sa", ""))
{
activity(connection);
ProviderCapabilities capabilities = provider.capabilities(connection);
assertThat(capabilities.supported()).isTrue();
assertThat(capabilities.statsAccess()).isEqualTo("FULL");
assertThat(capabilities.statements()).isTrue();
assertThat(capabilities.readAllStats()).isTrue();
assertThat(capabilities.version()).isGreaterThanOrEqualTo(2000000);
assertThat(capabilities.features()).contains(StatsFeature.STATEMENTS, StatsFeature.SESSIONS, StatsFeature.BLOCKING, StatsFeature.TABLE_ROWS, StatsFeature.EXPLAIN_TEXT)
.doesNotContain(StatsFeature.CACHE_COUNTERS, StatsFeature.DATABASE_COUNTERS, StatsFeature.INDEX_USAGE, StatsFeature.TABLE_MAINTENANCE, StatsFeature.MAX_CONNECTIONS, StatsFeature.EXPLAIN_JSON);
assertThat(provider.features()).containsAll(capabilities.features());
assertThat(provider.readDatabase(connection).counters()).isEmpty();
assertThat(provider.readDatabase(connection).maxConnections()).isZero();
assertThat(provider.readStatementResetInfo(connection).statsReset()).isNull();
provider.readTables(connection);
List<StatementStats> statements = provider.readStatements(connection);
StatementStats lookup = statements.stream().filter(s -> s.queryText().contains("WHERE customer_id = 3")).findFirst().orElseThrow();
assertThat(lookup.calls()).isEqualTo(5);
assertThat(lookup.rows()).isEqualTo(5 * 10);
assertThat(lookup.totalExecMs()).isGreaterThanOrEqualTo(lookup.maxExecMs());
assertThat(lookup.dbUser()).isNull();
assertThat(statements).noneMatch(s -> s.queryText().toUpperCase().contains("INFORMATION_SCHEMA") || s.queryText().contains("DB_PERF_PROBE")
|| s.queryText().toUpperCase().startsWith("SET ") || s.queryText().contains("DISK_SPACE_USED") || s.queryText().equalsIgnoreCase("CALL DATABASE()"));
assertThat(provider.readStatements(connection)).anyMatch(s -> s.queryId() == lookup.queryId() && s.queryText().equals(lookup.queryText()));
TableStats orders = provider.readTables(connection).stream().filter(t -> t.tableName().equalsIgnoreCase("orders")).findFirst().orElseThrow();
assertThat(orders.counters()).containsEntry("nLiveTup", 50L).containsKeys("tableBytes", "totalBytes").doesNotContainKeys("seqScan", "nDeadTup");
assertThat(orders.relid()).isPositive();
List<IndexStats> indexes = provider.readIndexes(connection).stream().filter(i -> i.tableName().equalsIgnoreCase("orders")).toList();
assertThat(indexes).anySatisfy(i -> assertThat(i.primary() && i.unique()).isTrue());
assertThat(indexes).anySatisfy(i ->
{
assertThat(i.indexName()).isEqualToIgnoringCase("orders_status");
assertThat(i.unique()).isFalse();
assertThat(i.columns()).isEqualToIgnoringCase("status, created DESC");
assertThat(i.definition()).contains("CREATE INDEX").doesNotContain("USING CONSTRAINT");
});
assertThat(indexes).anySatisfy(i -> assertThat(i.definition()).contains("USING CONSTRAINT"));
try(Connection holder = DriverManager.getConnection(url, "sa", ""); Connection waiter = DriverManager.getConnection(url + ";LOCK_TIMEOUT=10000", "sa", ""))
{
holder.setAutoCommit(false);
try(Statement statement = holder.createStatement())
{
statement.executeUpdate("UPDATE orders SET status = 'HELD' WHERE id = 1");
}
CompletableFuture<Void> blocked = CompletableFuture.runAsync(() -> update(waiter));
List<SessionInfo> sessions = waitForBlocked(connection);
SessionInfo idle = sessions.stream().filter(s -> s.state().equals("idle in transaction")).findFirst().orElseThrow();
assertThat(idle.stateChange()).isNotNull();
assertThat(idle.xactSeconds()).isNotNull();
SessionInfo waiting = sessions.stream().filter(s -> !s.blockedBy().isEmpty()).findFirst().orElseThrow();
assertThat(waiting.state()).isEqualTo("active");
assertThat(waiting.waitEventType()).isEqualTo("Lock");
assertThat(waiting.blockedBy()).containsExactly(idle.pid());
assertThat(waiting.queryText()).contains("UPDATE orders");
assertThat(waiting.queryId()).isNotNull();
assertThat(waiting.querySeconds()).isNotNull();
assertThat(provider.readConnections(connection).stream().mapToInt(ConnectionCount::connectionCount).sum()).isGreaterThanOrEqualTo(3);
assertThat(provider.readConnections(connection)).anyMatch(c -> "idle in transaction".equals(c.state()));
holder.rollback();
blocked.get();
}
Map<String, String> settings = provider.readSettings(connection);
assertThat(settings).containsEntry("QUERY_STATISTICS", "TRUE").containsKey("info.VERSION");
}
}
/*******************************************************************************
** Without QUERY_STATISTICS statements are unavailable until enabled.
*******************************************************************************/
@Test
void capabilitiesWithoutQueryStatisticsAndEnable() throws Exception
{
try(Connection connection = open(false, ""))
{
activity(connection);
ProviderCapabilities off = provider.capabilities(connection);
assertThat(off.supported()).isTrue();
assertThat(off.statements()).isFalse();
assertThat(off.statsAccess()).isEqualTo("NO_QUERY_STATISTICS");
assertThat(off.message()).contains("QUERY_STATISTICS=TRUE");
assertThat(off.has(StatsFeature.STATEMENTS)).isFalse();
assertThat(off.has(StatsFeature.SESSIONS)).isTrue();
assertThat(provider.readStatements(connection)).isEmpty();
assertThat(provider.readTables(connection)).isNotEmpty();
assertThat(provider.readSettings(connection)).containsEntry("QUERY_STATISTICS", "FALSE");
assertThat(provider.enableStatementStatistics(connection)).contains("JDBC URL");
activity(connection);
ProviderCapabilities on = provider.capabilities(connection);
assertThat(on.statsAccess()).isEqualTo("FULL");
assertThat(on.has(StatsFeature.STATEMENTS)).isTrue();
assertThat(provider.readStatements(connection)).anyMatch(s -> s.queryText().contains("WHERE customer_id = 3") && s.calls() == 5);
assertThat(provider.enableStatementStatistics(connection)).contains("already on");
}
}
/*******************************************************************************
** A non-admin user sees only its own session and cannot enable statistics.
*******************************************************************************/
@Test
void nonAdminUser() throws Exception
{
String url = url(true, "");
try(Connection admin = DriverManager.getConnection(url, "sa", ""))
{
try(Statement statement = admin.createStatement())
{
statement.execute("CREATE USER app PASSWORD 'secret'");
statement.execute("CREATE TABLE probe (id INT PRIMARY KEY)");
statement.execute("GRANT SELECT ON probe TO app");
}
try(Connection app = DriverManager.getConnection(url.replace(";DB_CLOSE_DELAY=-1", "").replace(";QUERY_STATISTICS=TRUE", ""), "app", "secret"))
{
ProviderCapabilities capabilities = provider.capabilities(app);
assertThat(capabilities.statsAccess()).isEqualTo("NOT_ADMIN");
assertThat(capabilities.readAllStats()).isFalse();
assertThat(capabilities.statements()).isTrue();
assertThat(capabilities.has(StatsFeature.BLOCKING)).isFalse();
assertThat(provider.readSessions(app)).isEmpty();
assertThat(provider.remedy("NOT_ADMIN", "APP")).contains("ALTER USER APP ADMIN TRUE");
try(Statement statement = admin.createStatement())
{
statement.execute("SET QUERY_STATISTICS FALSE");
}
assertThatThrownBy(() -> provider.enableStatementStatistics(app)).isInstanceOf(SQLException.class);
}
}
}
/*******************************************************************************
** EXPLAIN plans without executing; ANALYZE only for a non-admin SELECT;
** side-effecting functions, aliases, DDL and multiple statements refused.
*******************************************************************************/
@Test
void explainText() throws Exception
{
String url = url(true, ";MODE=MySQL");
try(Connection connection = DriverManager.getConnection(url, "sa", ""))
{
activity(connection);
assertThat(provider.capabilities(connection).has(StatsFeature.EXPLAIN_ANALYZE)).isFalse();
assertThat(provider.capabilities(connection).has(StatsFeature.EXPLAIN_TEXT)).isTrue();
assertThat(provider.explainText(connection, "SELECT * FROM orders WHERE status = 'NEW'", List.of(), false, 5000)).containsIgnoringCase("orders_status").doesNotContain("scanCount");
assertThat(provider.explainText(connection, "SELECT * FROM orders WHERE customer_id = ?1", List.of(), false, 5000)).contains("?1");
assertThat(connection.getAutoCommit()).isTrue();
assertThat(connection.isReadOnly()).isFalse();
assertThatThrownBy(() -> provider.explainText(connection, "SELECT * FROM orders WHERE customer_id = ?1", List.of("3"), true, 5000)).hasMessageContaining("admin");
assertThatThrownBy(() -> provider.explainText(connection, "SELECT FILE_WRITE('x', '/tmp/dbperf-h2-explain-probe') FROM orders", List.of(), false, 5000)).hasMessageContaining("FILE_WRITE");
assertThatThrownBy(() -> provider.explainText(connection, "SELECT \"file_write\"('x', '/tmp/p')", List.of(), false, 5000)).hasMessageContaining("FILE_WRITE");
assertThatThrownBy(() -> provider.explainText(connection, "DROP TABLE orders", List.of(), false, 5000)).hasMessageContaining("DROP");
assertThatThrownBy(() -> provider.explainText(connection, "SELECT 1; SELECT 2", List.of(), false, 5000)).hasMessageContaining("one statement");
assertThat(provider.explainText(connection, "SELECT 'drop; file_write' AS label FROM orders", List.of(), false, 5000)).containsIgnoringCase("orders");
try(Statement statement = connection.createStatement())
{
statement.execute("CREATE ALIAS my_side_effect FOR \"java.lang.System.nanoTime\"");
statement.execute("CREATE USER reader PASSWORD 'secret'");
statement.execute("GRANT SELECT ON orders TO reader");
}
assertThatThrownBy(() -> provider.explainText(connection, "SELECT my_side_effect() FROM orders", List.of(), false, 5000)).hasMessageContaining("MY_SIDE_EFFECT");
try(Connection reader = DriverManager.getConnection(url.replace(";DB_CLOSE_DELAY=-1", "").replace(";QUERY_STATISTICS=TRUE", ""), "reader", "secret"))
{
assertThat(provider.capabilities(reader).has(StatsFeature.EXPLAIN_ANALYZE)).isTrue();
assertThat(provider.explainText(reader, "SELECT * FROM orders WHERE customer_id = ?1", List.of("3"), true, 5000)).contains("scanCount");
assertThatThrownBy(() -> provider.explainText(reader, "VALUES (1)", List.of(), true, 5000)).hasMessageContaining("single SELECT");
}
}
assertThat(java.nio.file.Files.exists(java.nio.file.Path.of("/tmp/dbperf-h2-explain-probe"))).isFalse();
}
/*******************************************************************************
** H2 session states use PostgreSQL's names.
*******************************************************************************/
@Test
void stateMapping() throws Exception
{
assertThat(provider.setupChecks("NO_QUERY_STATISTICS", "m")).extracting(c -> c.check()).containsExactly("H2 QUERY_STATISTICS", "H2 admin user");
assertThat(provider.remedy("FULL", "sa")).isEqualTo("Full statistics access");
assertThat(provider.remedy("UNSUPPORTED", "sa")).contains("H2 2.0");
assertThat(new PostgresStatsProvider().remedy("NO_READ_ALL_STATS", "sa")).isEqualTo("grant pg_read_all_stats to sa");
assertThat(new PostgresStatsProvider().setupChecks("FULL", null)).extracting(c -> c.check()).containsExactly("pg_stat_statements", "pg_read_all_stats");
}
/*******************************************************************************
** Tables, a secondary index, a foreign key and repeated queries.
*******************************************************************************/
private static void activity(Connection connection) throws SQLException
{
try(Statement statement = connection.createStatement())
{
statement.execute("CREATE TABLE IF NOT EXISTS customer (id INT PRIMARY KEY, name VARCHAR(50))");
statement.execute("CREATE TABLE IF NOT EXISTS orders (id INT PRIMARY KEY, customer_id INT REFERENCES customer(id), status VARCHAR(20), created TIMESTAMP)");
statement.execute("CREATE INDEX IF NOT EXISTS orders_status ON orders (status, created DESC)");
statement.execute("MERGE INTO customer KEY (id) SELECT x, 'c' || x FROM SYSTEM_RANGE(1, 5)");
statement.execute("MERGE INTO orders KEY (id) SELECT x, MOD(x, 5) + 1, 'NEW', CURRENT_TIMESTAMP FROM SYSTEM_RANGE(1, 50)");
for(int i = 0; i < 5; i++)
{
statement.executeQuery("SELECT * FROM orders WHERE customer_id = 3").close();
}
}
}
/*******************************************************************************
** Poll until the waiter shows as blocked.
*******************************************************************************/
private List<SessionInfo> waitForBlocked(Connection connection) throws Exception
{
for(int i = 0; i < 100; i++)
{
List<SessionInfo> sessions = provider.readSessions(connection);
if(sessions.stream().anyMatch(s -> !s.blockedBy().isEmpty()))
{
return (sessions);
}
Thread.sleep(50);
}
throw (new AssertionError("no blocked session observed"));
}
/*******************************************************************************
** Blocking update for the waiter thread.
*******************************************************************************/
private static void update(Connection waiter)
{
try(Statement statement = waiter.createStatement())
{
statement.executeUpdate("UPDATE orders SET status = 'WAIT' WHERE id = 1");
}
catch(SQLException e)
{
throw (new IllegalStateException(e));
}
}
/*******************************************************************************
** A fresh in-memory database.
*******************************************************************************/
private static Connection open(boolean statistics, String mode) throws SQLException
{
return (DriverManager.getConnection(url(statistics, mode), "sa", ""));
}
/*******************************************************************************
** URL of a fresh in-memory database.
*******************************************************************************/
private static String url(boolean statistics, String mode)
{
return ("jdbc:h2:mem:h2stats" + UUID.randomUUID().toString().replace("-", "") + ";DB_CLOSE_DELAY=-1" + (statistics ? ";QUERY_STATISTICS=TRUE" : "") + mode);
}
}
@@ -28,6 +28,7 @@ import java.util.List;
import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector;
import com.kingsrook.qbits.dbperformance.provider.UnsupportedStatsProvider;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -161,9 +162,17 @@ class SnapshotCollectorTest
void unsupportedVendor() throws Exception
{
DbPerfRuntime runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
QRecord snapshot = new SnapshotCollector(runtime).collect(runtime.getConfig().getBackendName(), false);
QRecord snapshot = new SnapshotCollector(runtime, Clock.systemUTC(), new UnsupportedStatsProvider()).collect(runtime.getConfig().getBackendName(), false);
assertThat(snapshot.getValueString("status")).isEqualTo("PARTIAL");
assertThat(snapshot.getValueString("errorMessage")).contains("not supported");
//////////////////////////////////////////////////////////////////////
// the H2 test backend itself is supported, partially without //
// QUERY_STATISTICS //
//////////////////////////////////////////////////////////////////////
QRecord h2 = new SnapshotCollector(runtime).collect(runtime.getConfig().getBackendName(), false);
assertThat(h2.getValueString("status")).isEqualTo("PARTIAL");
assertThat(h2.getValueString("errorMessage")).contains("QUERY_STATISTICS");
}
@@ -0,0 +1,135 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dbperformance.alerts;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.kingsrook.qbits.dbperformance.DbPerfRuntime;
import com.kingsrook.qbits.dbperformance.DbPerfTestSupport;
import com.kingsrook.qbits.dbperformance.analyze.RuleContext;
import com.kingsrook.qbits.dbperformance.collect.DbPerfStore;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Spike detection reads seven days of query snapshots (~230,000 rows in kof22)
** on every evaluation. It must page them and keep only per-statement numbers.
*******************************************************************************/
class SpikeHistoryBoundedTest
{
private static final AtomicInteger MAX_PAGE = new AtomicInteger();
private static final AtomicInteger LOADED = new AtomicInteger();
/*******************************************************************************
** Records the largest page QQQ hands back.
*******************************************************************************/
public static class CountingCustomizer implements TableCustomizerInterface
{
/***************************************************************************
** Count, change nothing.
***************************************************************************/
@Override
public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records)
{
LOADED.addAndGet(records.size());
MAX_PAGE.accumulateAndGet(records.size(), Math::max);
return (records);
}
}
/*******************************************************************************
** 2,500 samples across page boundaries: pages stay at BATCH_SIZE, invalid
** snapshots and the window are honored, the current interval is found.
*******************************************************************************/
@Test
void historyIsPagedAndExact() throws Exception
{
DbPerfRuntime runtime = DbPerfRuntime.of(DbPerfTestSupport.instance());
String backend = runtime.getConfig().getBackendName();
runtime.getInstance().getTable("db_perf_query_snapshot").withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CountingCustomizer.class));
Instant now = Instant.parse("2026-10-16T20:00:00Z");
QRecord ok = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now.minusSeconds(3600)).withValue("status", "OK"))).getFirst();
QRecord failed = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now.minusSeconds(1800)).withValue("status", "FAILED"))).getFirst();
QRecord current = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", backend).withValue("capturedAt", now).withValue("status", "OK"))).getFirst();
int one = statement(backend, 1L);
int two = statement(backend, 2L);
List<QRecord> rows = new ArrayList<>();
for(int i = 0; i < 2500; i++)
{
rows.add(sample(ok, i % 2 == 0 ? one : two, now.minusSeconds(3600 + i), i));
}
rows.add(sample(failed, one, now.minusSeconds(1800), 999999));
rows.add(sample(ok, one, now.minusSeconds(8 * 86400L), 999999));
rows.add(sample(current, one, now, 42));
DbPerfStore.insert("db_perf_query_snapshot", rows);
MAX_PAGE.set(0);
LOADED.set(0);
RuleContext ctx = new RuleContext(runtime, backend, new QRecord(), DbPerfStore.first("db_perf_settings", new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()), now);
Map<Integer, SnapshotEvaluator.StatementHistory> history = SnapshotEvaluator.history(ctx, current.getValueInteger("id"));
assertThat(MAX_PAGE.get()).as("largest page of query snapshots held at once").isLessThanOrEqualTo(DbPerfStore.BATCH_SIZE);
assertThat(history.get(one).sortedPrevious()).hasSize(1250).doesNotContain(999999.0);
assertThat(history.get(two).sortedPrevious()).hasSize(1250);
assertThat(history.get(one).sortedPrevious()[0]).isEqualTo(0.0);
assertThat(history.get(one)).extracting("current").isEqualTo(42.0);
assertThat(history.get(two)).extracting("current").isNull();
}
/*******************************************************************************
** One statement identity; returns its id.
*******************************************************************************/
private static int statement(String backend, long queryId) throws Exception
{
return (DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", backend).withValue("queryId", queryId)
.withValue("queryText", "select " + queryId).withValue("fingerprint", String.valueOf(queryId).repeat(64).substring(0, 64)))).getFirst().getValueInteger("id"));
}
/*******************************************************************************
** One query snapshot row.
*******************************************************************************/
private static QRecord sample(QRecord snapshot, int statementId, Instant at, int totalMs)
{
return (new QRecord().withValue("snapshotId", snapshot.getValueInteger("id")).withValue("statementId", statementId)
.withValue("capturedAt", at).withValue("totalMs", BigDecimal.valueOf(totalMs)));
}
}