From 65922dd413774d7847462c69339310a86c7f52e1 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 20 Oct 2026 18:52:28 -0500 Subject: [PATCH 01/18] ci: adopt verified central QQQ orb 0.7.3 Co-Authored-By: Codex --- .circleci/config.yml | 68 +++++++------------------------------------- .qqq-ci.yml | 16 +++++++++++ 2 files changed, 27 insertions(+), 57 deletions(-) create mode 100644 .qqq-ci.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index aec59b2..113ce37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,60 +1,14 @@ version: 2.1 - +setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.6.9 - -commands: - configure-package-install: - steps: - - run: - name: Keep package installation unattended - command: | - # These build machines are discarded after the job. - sudo install -d /etc/needrestart/conf.d - printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null - + qqq-orb: kingsrook/qqq-orb@0.7.3 workflows: - test_only: - jobs: - - qqq-orb/mvn_test_only: - pre-steps: - - configure-package-install - context: [qqq-maven-registry-credentials] - filters: - branches: - ignore: /(develop|release\/.*)/ - - publish_snapshot: - jobs: - - qqq-orb/mvn_publish: - pre-steps: - - configure-package-install - context: [qqq-maven-registry-credentials] - branch_type: snapshot - filters: - branches: - only: [develop] - - publish_release_candidate: - jobs: - - qqq-orb/mvn_publish: - pre-steps: - - configure-package-install - context: [qqq-maven-registry-credentials] - branch_type: release_candidate - filters: - branches: - only: [/release\/.*/] - - publish_release: - jobs: - - qqq-orb/mvn_publish: - pre-steps: - - configure-package-install - context: [qqq-maven-registry-credentials] - branch_type: release - filters: - branches: - ignore: /.*/ - tags: - only: [/^v\d+\.\d+\.\d+$/] + setup: + jobs: + - qqq-orb/generate_pipeline: + orb_version: 0.7.3 + filters: + branches: + ignore: automation/qbit-rc-pending + tags: + only: /^v\d+\.\d+\.\d+$/ diff --git a/.qqq-ci.yml b/.qqq-ci.yml new file mode 100644 index 0000000..3f2b2d1 --- /dev/null +++ b/.qqq-ci.yml @@ -0,0 +1,16 @@ +profile: qbit +orb_version: 0.7.3 +default_branch: develop +java_version: '21' +browsers: true +api_checks: false +docs_only: false +release_on_main: false +release_tag_policy: stable +test_branch_policy: non-release +publish: + snapshot: true + release_candidate: true + release: true + asciidoc: false +release_completion: false -- 2.54.0 From 2a2ba58b066dd4a779b2b5728bf6a6ea375c12ea Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 00:57:50 -0500 Subject: [PATCH 02/18] ci: adopt shared orb 0.7.11 (#19) Co-authored-by: Codex --- .circleci/config.yml | 4 ++-- .qqq-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 113ce37..def7f95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,12 @@ version: 2.1 setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.7.3 + qqq-orb: kingsrook/qqq-orb@0.7.11 workflows: setup: jobs: - qqq-orb/generate_pipeline: - orb_version: 0.7.3 + orb_version: 0.7.11 filters: branches: ignore: automation/qbit-rc-pending diff --git a/.qqq-ci.yml b/.qqq-ci.yml index 3f2b2d1..7b032d6 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -1,5 +1,5 @@ profile: qbit -orb_version: 0.7.3 +orb_version: 0.7.11 default_branch: develop java_version: '21' browsers: true -- 2.54.0 From ea9036509cca258abb04d4e78f25dd6295ee1f87 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 02:25:42 -0500 Subject: [PATCH 03/18] ci: centralize rc release completion in qqq orb (#20) Co-authored-by: Codex --- .github/workflows/release-rc.yml | 76 ++++++++++++++++++++++++++------ .qqq-ci.yml | 2 +- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release-rc.yml b/.github/workflows/release-rc.yml index a342ba8..0158aa2 100644 --- a/.github/workflows/release-rc.yml +++ b/.github/workflows/release-rc.yml @@ -1,26 +1,74 @@ -# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6): -# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into -# the release line and the release/* push runs CircleCI publish_release_candidate. name: Release candidate - -on: +'on': workflow_dispatch: inputs: dry_run: - description: Print the plan without merging or creating branches + description: Plan a cut without changing release state type: boolean default: false + finalize: + description: Retry finalization of an existing cut; do not cut another RC + type: boolean + default: false + recover: + description: Recover the same published RC after a deployment timeout; never + reupload + type: boolean + default: false + circle_build: + description: Original Circle publisher build number (recovery only) + type: string + default: '' + merge_sha: + description: Existing cut merge_sha (finalize retry only) + type: string + default: '' + bump_sha: + description: Existing cut bump_sha (finalize retry only) + type: string + default: '' + branch: + description: Existing cut branch (finalize retry only) + type: string + default: '' + version: + description: Existing cut version (finalize retry only) + type: string + default: '' pull_request: - types: [closed] - branches: [develop] - + types: + - closed + branches: + - develop + repository_dispatch: + types: + - qbit-rc-published permissions: contents: write - + actions: read + checks: read jobs: release-rc: - if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc')) - uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main - secrets: inherit + if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' + || (github.event_name == 'pull_request' && github.event.pull_request.merged + == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name, + 'release:rc')) + uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@7d73ac90b6b5a6acb61b4d3fa56625c5b6dcc652 with: - dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} + dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false + }} + finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name + == 'workflow_dispatch' && inputs.finalize) }} + recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false + }} + circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build + || '' }} + merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha + || inputs.merge_sha || '' }} + bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha + || inputs.bump_sha || '' }} + branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch + || inputs.branch || '' }} + version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version + || inputs.version || '' }} + secrets: inherit diff --git a/.qqq-ci.yml b/.qqq-ci.yml index 7b032d6..a8863c3 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -13,4 +13,4 @@ publish: release_candidate: true release: true asciidoc: false -release_completion: false +release_completion: true -- 2.54.0 From 681598be57926ed6e4b66f2693382de7f0eb4275 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 08:56:36 -0500 Subject: [PATCH 04/18] ci: adopt central orb 0.7.12 and stable gates (#21) Co-authored-by: Codex --- .circleci/config.yml | 4 ++-- .github/workflows/release-rc.yml | 2 +- .qqq-ci.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index def7f95..c9cb461 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,12 @@ version: 2.1 setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.7.11 + qqq-orb: kingsrook/qqq-orb@0.7.12 workflows: setup: jobs: - qqq-orb/generate_pipeline: - orb_version: 0.7.11 + orb_version: 0.7.12 filters: branches: ignore: automation/qbit-rc-pending diff --git a/.github/workflows/release-rc.yml b/.github/workflows/release-rc.yml index 0158aa2..b48a32a 100644 --- a/.github/workflows/release-rc.yml +++ b/.github/workflows/release-rc.yml @@ -53,7 +53,7 @@ jobs: || (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name, 'release:rc')) - uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@7d73ac90b6b5a6acb61b4d3fa56625c5b6dcc652 + uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44 with: dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }} diff --git a/.qqq-ci.yml b/.qqq-ci.yml index a8863c3..be6666b 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -1,5 +1,5 @@ profile: qbit -orb_version: 0.7.11 +orb_version: 0.7.12 default_branch: develop java_version: '21' browsers: true -- 2.54.0 From 917864642381c38caf09a1bbd98947683b4cbb8e Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 12:17:56 -0500 Subject: [PATCH 05/18] ci: centralize branch verification and source scanning (#23) Co-authored-by: Codex --- .circleci/config.yml | 4 ++-- .github/workflows/codeql.yml | 23 +++++++++++++++++++++++ .qqq-ci.yml | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/codeql.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index c9cb461..23c1a84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,12 @@ version: 2.1 setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.7.12 + qqq-orb: kingsrook/qqq-orb@0.7.15 workflows: setup: jobs: - qqq-orb/generate_pipeline: - orb_version: 0.7.12 + orb_version: 0.7.15 filters: branches: ignore: automation/qbit-rc-pending diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..bd3ceaf --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,23 @@ +name: CodeQL +'on': + push: + branches: + - main + - develop + - release/** + - hotfix/** + pull_request: {} + schedule: + - cron: 41 16 * * 0 + workflow_dispatch: {} +permissions: + contents: read + actions: read + security-events: write + packages: read +concurrency: + group: codeql-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + analyze: + uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22 diff --git a/.qqq-ci.yml b/.qqq-ci.yml index be6666b..adef7c7 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -1,5 +1,5 @@ profile: qbit -orb_version: 0.7.12 +orb_version: 0.7.15 default_branch: develop java_version: '21' browsers: true -- 2.54.0 From ccc0cc2842e9ac457606f94d86defe32b168b8ef Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 14:15:18 -0500 Subject: [PATCH 06/18] ci: adopt shared orb 0.7.17 (#26) Co-authored-by: Codex --- .circleci/config.yml | 4 ++-- .qqq-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23c1a84..1d4fa0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,12 @@ version: 2.1 setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.7.15 + qqq-orb: kingsrook/qqq-orb@0.7.17 workflows: setup: jobs: - qqq-orb/generate_pipeline: - orb_version: 0.7.15 + orb_version: 0.7.17 filters: branches: ignore: automation/qbit-rc-pending diff --git a/.qqq-ci.yml b/.qqq-ci.yml index adef7c7..a9e9f43 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -1,5 +1,5 @@ profile: qbit -orb_version: 0.7.15 +orb_version: 0.7.17 default_branch: develop java_version: '21' browsers: true -- 2.54.0 From 65e63551f9a9e3775e9ebdb693de6861ea5160da Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 14:43:20 -0500 Subject: [PATCH 07/18] ci: adopt shared orb 0.7.18 (#27) Co-authored-by: Codex --- .circleci/config.yml | 4 ++-- .qqq-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d4fa0d..42b73df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,12 @@ version: 2.1 setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.7.17 + qqq-orb: kingsrook/qqq-orb@0.7.18 workflows: setup: jobs: - qqq-orb/generate_pipeline: - orb_version: 0.7.17 + orb_version: 0.7.18 filters: branches: ignore: automation/qbit-rc-pending diff --git a/.qqq-ci.yml b/.qqq-ci.yml index a9e9f43..967a18e 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -1,5 +1,5 @@ profile: qbit -orb_version: 0.7.17 +orb_version: 0.7.18 default_branch: develop java_version: '21' browsers: true -- 2.54.0 From 4fd2c30b07016da9a9db7d761d1ccb68ccce0632 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 21 Oct 2026 15:26:36 -0500 Subject: [PATCH 08/18] ci: adopt managed orb 0.8 patch track (#28) Co-authored-by: Codex --- .circleci/config.yml | 4 ++-- .qqq-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 42b73df..f890c28 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,12 @@ version: 2.1 setup: true orbs: - qqq-orb: kingsrook/qqq-orb@0.7.18 + qqq-orb: kingsrook/qqq-orb@0.8 workflows: setup: jobs: - qqq-orb/generate_pipeline: - orb_version: 0.7.18 + orb_version: '0.8' filters: branches: ignore: automation/qbit-rc-pending diff --git a/.qqq-ci.yml b/.qqq-ci.yml index 967a18e..6777ede 100644 --- a/.qqq-ci.yml +++ b/.qqq-ci.yml @@ -1,5 +1,5 @@ profile: qbit -orb_version: 0.7.18 +orb_version: '0.8' default_branch: develop java_version: '21' browsers: true -- 2.54.0 From d46b6ca222319659c25725d5469591854f238bd8 Mon Sep 17 00:00:00 2001 From: James Maes Date: Sun, 1 Nov 2026 06:14:44 -0600 Subject: [PATCH 09/18] feat(provider): H2 statistics provider and per-area capabilities (#34) H2StatsProvider over INFORMATION_SCHEMA (statements via QUERY_STATISTICS, sessions, blocking, connections, indexes, size, settings, approximated table rows and sizes, text plans); per-area provider capabilities with rules skipped where a provider lacks the data; a vendor registry with a seam for MySQL/MariaDB; provider-specific setup and health checks; an Enable statement statistics action; and the H2 plan guard (single SELECT/WITH only, side-effecting functions refused, EXPLAIN ANALYZE only for non-admin users, read-only connection, always rolled back). Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 10 + README.md | 17 +- docs/PROVIDERS.md | 73 ++ docs/SPEC.md | 4 +- .../alerts/SnapshotEvaluator.java | 2 +- .../dbperformance/analyze/AnalyzeService.java | 12 +- .../dbperformance/analyze/DbPerfRule.java | 13 + .../dbperformance/analyze/StoredRule.java | 25 + .../qbits/dbperformance/api/ExplainSpec.java | 2 +- .../collect/SnapshotCollector.java | 12 +- .../dbperformance/explain/ExplainService.java | 179 +++-- .../dbperformance/explain/ExplainSql.java | 31 +- .../explain/PlanTextRenderer.java | 35 + .../health/DbPerfHealthChecks.java | 9 +- .../metadata/DbPerfPossibleValues.java | 4 +- .../processes/DbPerfEnableStatisticsStep.java | 99 +++ .../processes/DbPerfProcesses.java | 5 + .../provider/DbStatsProvider.java | 60 +- .../provider/DbStatsProviders.java | 66 +- .../provider/H2StatsProvider.java | 691 ++++++++++++++++++ .../dbperformance/provider/JdbcReads.java | 101 +++ .../provider/PostgresStatsProvider.java | 114 +-- .../provider/ProviderCapabilities.java | 37 +- .../dbperformance/provider/SetupCheck.java | 31 + .../dbperformance/provider/StatsFeature.java | 117 +++ .../provider/UnsupportedStatsProvider.java | 2 +- .../widgets/DbPerfWidgetRenderer.java | 79 +- .../resources/help/qbit-db-performance.yaml | 8 +- .../openapi/qqq-v1/db-performance.yaml | 2 +- .../qbits/dbperformance/DbPerfHealthTest.java | 10 +- .../dbperformance/DbPerfPresentationTest.java | 7 +- .../qbits/dbperformance/H2CollectionTest.java | 173 +++++ .../dbperformance/H2StatsProviderTest.java | 366 ++++++++++ .../dbperformance/SnapshotCollectorTest.java | 11 +- 34 files changed, 2243 insertions(+), 164 deletions(-) create mode 100644 docs/PROVIDERS.md create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfEnableStatisticsStep.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/provider/H2StatsProvider.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/provider/JdbcReads.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/provider/SetupCheck.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/provider/StatsFeature.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/H2CollectionTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/H2StatsProviderTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 046e267..38d36ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes are documented here. This project follows Keep a Changelog a ## [Unreleased] +### Added +- H2 2.x support (`H2StatsProvider`) from `INFORMATION_SCHEMA`: statement statistics from `QUERY_STATISTICS` (when the database runs with `QUERY_STATISTICS=TRUE`), sessions with blocking and idle-in-transaction detection, connection counts, table row estimates and sizes, index definitions, settings, and text query plans (`EXPLAIN`; `EXPLAIN ANALYZE` only for non-admin H2 users, since H2 has no read-only transaction: plans refuse side-effecting functions such as `FILE_WRITE`, DDL/DML keywords, user-defined aliases and multiple statements). The enterprise test server showed UNSUPPORTED on H2; it now collects. +- Per-area capabilities (`StatsFeature`): providers declare which statistics areas they can supply and `ProviderCapabilities.features()` reports what is available on a connection. Snapshot cache-hit ratio and connection limit stay empty when the vendor has none, and analysis rules declare the areas they need (`DbPerfRule.requires()`) and are skipped on providers that cannot supply them, so H2 never gets "unused index" or "low cache hit" findings from missing counters. +- Provider registry keyed by JDBC product name (`DbStatsProviders.register`), with `docs/PROVIDERS.md` describing what MySQL/MariaDB and SQLite would map to. +- **Enable statement statistics** action on Settings (`dbPerfEnableStatementStatistics`, manage settings): on H2 runs `SET QUERY_STATISTICS_MAX_ENTRIES 1000` and `SET QUERY_STATISTICS TRUE` as an admin; on PostgreSQL explains the manual extension setup. It is the qbit's only change to the monitored database and is explicit. + +### Changed +- Health and the Setup panel take their prerequisite checks and remedies from the backend's provider: PostgreSQL still names `pg_stat_statements` and `pg_read_all_stats`; H2 names `QUERY_STATISTICS` and the admin role (new access codes `NO_QUERY_STATISTICS` and `NOT_ADMIN`). +- The previous-snapshot baseline no longer depends on a PostgreSQL-only counter (it uses the database size every supported provider records). + ### Fixed - Query-spike alerts no longer hold seven days of query snapshots as records on every snapshot evaluation (~230,000 rows at kof22's rate). `SnapshotEvaluator` reads the window in 1,000-row keyset pages and keeps only each statement's current interval time, latest capture time and earlier interval times as primitive doubles; the spike rule (twelve prior samples, median, absence after an hour) is unchanged. - The Attribution home panel no longer loads a day of attribution detail into the heap. It sums statements and time per unit and table in the database (at most 5,000 groups), so a render holds about a thousand aggregate rows instead of the roughly 570,000 detail rows (540 MB in PostgreSQL) that ran the kof22 web pod out of memory on 2026-10-16. The daily attribution rollup reads the day in 1,000-row keyset pages (`DbPerfStore.forEachPage`), so worker memory is bounded by distinct dimension keys, not by the day's row count. The hourly analysis (`dbPerfAnalyze`) reads attribution the same way: `RuleContext.window("attribution", ...)` returns database sums per unit, table and operation instead of every detail row (it filled the kof22 worker heap to ~1.8 GB every hour). diff --git a/README.md b/README.md index fdf2bc5..d147fc4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # QBit DB Performance -Database performance for QQQ apps: PostgreSQL statement deltas, table and index statistics, live sessions and blocking, connection capacity, audited query plans, and typed home dashboards in standard secured QQQ tables. Approved design: [docs/SPEC.md](docs/SPEC.md). This branch implements collection and the presentation/plan-viewer phase; attribution listeners, findings evaluation, alerts and health are developed in a separate phase. No release has been published yet. +Database performance for QQQ apps: PostgreSQL and H2 statement deltas, table and index statistics, live sessions and blocking, connection capacity, audited query plans, and typed home dashboards in standard secured QQQ tables. Approved design: [docs/SPEC.md](docs/SPEC.md). This branch implements collection and the presentation/plan-viewer phase; attribution listeners, findings evaluation, alerts and health are developed in a separate phase. No release has been published yet. ## Requirements @@ -8,6 +8,7 @@ Database performance for QQQ apps: PostgreSQL statement deltas, table and index - PostgreSQL **13+**; **16+** for parameterized generic plans, with the pgJDBC driver (built against 42.7.14). - `pg_stat_statements` preloaded, created in the monitored database, and readable by the application role. - `pg_read_all_stats` granted to that role for complete statistics and other sessions' visibility. +- Or H2 **2.x** (embedded or server), with `QUERY_STATISTICS=TRUE` for statement statistics (see [H2](#h2)). - A host scheduler for automatic collection and rollups. Docker is required for the PostgreSQL acceptance tests. ## Install @@ -47,6 +48,16 @@ GRANT pg_read_all_stats TO your_application_role; Configure `shared_preload_libraries` with `pg_stat_statements` and create the extension in the monitored database through your normal deployment process. Missing access produces reduced-capability status and setup guidance. +### H2 + +The same qbit runs on an H2 backend; the provider is chosen from the JDBC product name, so no configuration is needed. Add query statistics to the H2 JDBC URL (or run **Enable statement statistics** on Settings as an H2 admin user; H2 forgets it when the database closes): + +``` +jdbc:h2:file:/data/app;MODE=MySQL;DB_CLOSE_DELAY=-1;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 +``` + +The cap matters: H2 keeps 100 statements by default and evicts a third of them, oldest first, whenever it is passed. Use an admin user (the default `sa` is one) to see other sessions and blocking. H2 keeps no transaction, cache-hit, scan, vacuum or index-usage counters, so those dashboard figures stay empty and the rules that need them (missing/unused index, bloat, vacuum, cache hit, temp spill, deadlocks, pool sizing) are skipped rather than reporting zeros. Plans are H2 text (index choice and scan counts), not a cost tree. [docs/PROVIDERS.md](docs/PROVIDERS.md) has the full capability table and what other vendors would need. + With `schedulerName`, `dbPerfCollectSnapshot` runs every five minutes and `dbPerfRollupAndPurge` daily at 03:15 (UTC by default). Omitting it registers no schedules. Use **Collect now** on Snapshots for the first baseline and a second collection for deltas. Table and index statistics normally update hourly. The scheduled processes are durable-eligible when the host has durable jobs enabled. UI hosts must include `qqq-middleware-javalin` (an optional dependency of this qbit). The producer registers UI API extension `db-performance` version `1.0.0` under `/qqq/v1/qbit/db-performance`. Core-only hosts can use tables and collection without middleware. @@ -80,7 +91,7 @@ Snapshot totals include all observed statement deltas. Only the top configured d | `db_performance.view_live_text` | Raw SQL in the transient activity API only; also requires view | | `db_performance.explain` | PLAN/GENERIC requests and Query plan record action | | `db_performance.explain_analyze` | Execute the guarded ANALYZE service; the record action also needs explain | -| `db_performance.manage_settings` | Settings/rule edits and Collect now | +| `db_performance.manage_settings` | Settings/rule edits, Collect now and Enable statement statistics | | `db_performance.manage_findings` | Finding status actions supplied by the findings phase | Hosts can change `permissionBaseName`; grant the corresponding complete names. Persistent collector and explain-audit writes run as the system user, retaining the requesting session's user ID for plans. Dashboard and API reads retain the viewer's session. @@ -116,6 +127,8 @@ Paths above are relative to `/qqq/v1/qbit/db-performance`. The [committed OpenAP Stored SQL is normalized and fingerprinted. Raw `pg_stat_activity` text is never persisted and appears only in the activity API with `view_live_text`; home widgets always use normalized text. SQL literals and parameter values must not be used as identifiers: normalization preserves quoted identifiers. Settings and rules are excluded from AI exposure. +On H2, PLAN runs `EXPLAIN` (not executed; `?n` slots stay unbound for generic plans) and ANALYZE runs `EXPLAIN ANALYZE` with the values bound. H2 has no READ ONLY transaction and a rollback does not undo file, session, sequence or DDL side effects, so on H2 both PLAN and ANALYZE refuse more than one statement and any SQL that names a side-effecting function (`FILE_WRITE`, `FILE_READ`, `CSVWRITE`, `CSVREAD`, `LINK_SCHEMA`, `SCRIPT`, `RUNSCRIPT`, `DISK_SPACE_USED`, `CANCEL_SESSION`, `ABORT_SESSION`, `SLEEP`, sequences), a DDL/DML or session keyword, or any user-defined alias; quoted names count. ANALYZE executes the query, so it is offered only to non-admin H2 users (an admin can call file functions) and only for a single SELECT/WITH; capabilities report it as unavailable otherwise. The connection is marked read-only (a hint in H2) and the transaction always rolls back. + PLAN executes EXPLAIN in a read-only transaction with a local timeout and rollback. Parameterized PLAN uses PostgreSQL 16+ GENERIC_PLAN. Only eligible literal-free PLAN/GENERIC plans are stored. A user's SQL must equal its own normalized form to qualify; SQL loaded from the normalized statement table also qualifies. ANALYZE accepts one SELECT or a non-modifying WITH, rejects INTO and row locks, binds supplied strings as unknown JDBC parameters, and executes inside a read-only transaction. Functions are still constrained by PostgreSQL READ ONLY. The EXPLAIN command uses pgJDBC simple-query execution with JDBC-managed parameter quoting, so PostgreSQL can coerce unknown-typed EXECUTE arguments. The transaction always rolls back and the prepared statement is explicitly deallocated before the connection returns to its pool. SQLSTATE 57014 records TIMEOUT. The service stores only a sanitized audit, never ANALYZE plans or parameter values. Returned ANALYZE plans are transient and can contain values; the assistant prompt redacts literals in plan expressions. Review transient plans before copying them elsewhere. diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md new file mode 100644 index 0000000..22a81de --- /dev/null +++ b/docs/PROVIDERS.md @@ -0,0 +1,73 @@ +# Statistics providers + +The qbit reads database statistics through `DbStatsProvider`. `DbStatsProviders` picks the provider from the JDBC product name (`DatabaseMetaData.getDatabaseProductName()`); anything unregistered gets `UnsupportedStatsProvider`, which shows a setup message instead of failing. Each provider declares the statistics areas (`StatsFeature`) its vendor can supply; `ProviderCapabilities.features()` says which are available on the current connection. Dashboards leave unavailable figures empty and analysis rules that need a missing area (`DbPerfRule.requires()`) are skipped, so a vendor with fewer counters produces fewer findings, never false ones. + +| JDBC product name | Provider | Saved name | Status | +|---|---|---|---| +| `PostgreSQL` | `PostgresStatsProvider` | `POSTGRES` | Supported (13+) | +| `H2` | `H2StatsProvider` | `H2` | Supported (2.x) | +| `MySQL`, `MariaDB` | none yet | `MYSQL` (reserved) | Unsupported | +| `SQLite` | none yet | | Unsupported | + +## Areas by provider + +S = supported, A = approximated, - = not available from the vendor. + +| Area (`StatsFeature`) | PostgreSQL | H2 | H2 source and notes | +|---|---|---|---| +| Database counters (commits, rollbacks, temp, deadlocks) | S | - | H2 keeps no transaction or temp-file counters | +| Cache hit ratio | S | - | `info.CACHE_HIT_RATIO` exists but is not a cumulative hit/read pair; shown in settings only | +| Database size | S | S | `SETTINGS` `info.FILE_SIZE` (0 for in-memory databases) | +| Connection limit | S | - | H2 has no connection limit | +| Statement statistics | S | S | `QUERY_STATISTICS` with `QUERY_STATISTICS=TRUE`: calls, total/mean/max ms, rows. No query id or user: the id is a stable SHA-256 hash of the text and the user is empty. The collector's own `INFORMATION_SCHEMA` and `DISK_SPACE_USED` reads and connection setup (`SET ...`, which H2 replays for every JDBC URL setting on each new connection, and `CALL DATABASE()`) are excluded. H2 keeps `QUERY_STATISTICS_MAX_ENTRIES` statements (default 100, set 1000) and evicts the oldest third past the cap | +| Statement I/O (blocks, temp) | S | - | | +| Statement reset time | S (14+) | - | A restart or eviction shows as falling call counts, which the delta logic treats as a reset | +| Table row estimates | S | A | `TABLES.ROW_COUNT_ESTIMATE` | +| Table sizes | S | A | `DISK_SPACE_USED(table)`: table data only; H2 cannot size indexes, so total = table bytes | +| Table access counters (scans, tuple writes) | S | - | | +| Table maintenance (dead rows, vacuum, freeze) | S | - | MVStore has no vacuum | +| Index definitions | S | S | `INDEXES` + `INDEX_COLUMNS`; constraint-generated indexes are marked `USING CONSTRAINT` and never proposed for removal | +| Index usage and size | S | - | | +| Live sessions | S | S | `SESSIONS`: RUNNING = active, SLEEP with uncommitted changes = idle in transaction. Non-admin users see only their own session | +| Blocking | S | S | `SESSIONS.BLOCKER_ID` (admin users) | +| Wait events | S | A | Blocked sessions report wait type `Lock` | +| Transaction age | S | A | No transaction start in H2: an idle transaction's age is the time since it went idle (a lower bound) | +| Connection counts | S | S | `SESSIONS` grouped by user, client info and state | +| Plans | JSON tree with costs | Text | `EXPLAIN` / `EXPLAIN ANALYZE` (index choice, scan counts); generic plans keep `?n` unbound. ANALYZE only for non-admin users (see below) | + +Access codes saved on settings: PostgreSQL `FULL`, `NO_READ_ALL_STATS`, `NO_PG_STAT_STATEMENTS`, `UNSUPPORTED`; H2 `FULL`, `NOT_ADMIN`, `NO_QUERY_STATISTICS`, `UNSUPPORTED`. Health and the Setup panel ask the provider for the remedy text (`remedy`, `setupChecks`). **Enable statement statistics** on Settings calls `enableStatementStatistics`: H2 runs `SET QUERY_STATISTICS_MAX_ENTRIES 1000` and `SET QUERY_STATISTICS TRUE` (admin, until the database closes); PostgreSQL cannot do it at runtime. + +## H2 query plan rule + +H2 has no READ ONLY transaction and a rollback does not undo file, session, sequence or DDL side effects, so on H2 both PLAN and ANALYZE refuse more than one statement and any SQL that names a side-effecting function (`FILE_WRITE`, `FILE_READ`, `CSVWRITE`, `CSVREAD`, `LINK_SCHEMA`, `SCRIPT`, `RUNSCRIPT`, `DISK_SPACE_USED`, `CANCEL_SESSION`, `ABORT_SESSION`, `SLEEP`, sequences), a DDL/DML or session keyword, or any user-defined alias; quoted names count. ANALYZE executes the query, so it is offered only to non-admin H2 users (an admin can call file functions) and only for a single SELECT/WITH; capabilities report it as unavailable otherwise. The connection is marked read-only (a hint in H2) and the transaction always rolls back. A host whose H2 user is `sa` (an admin, as in the enterprise test server) therefore gets PLAN only. + +## Adding a vendor + +1. Implement `DbStatsProvider`: the reads, `capabilities`, `features`, `remedy`, `setupChecks`, and optionally `enableStatementStatistics` and `explainText`. Map session states to `active` / `idle` / `idle in transaction` and leave unavailable counters out of the maps rather than writing zeros. +2. Register it under the JDBC product name: add it to the table in `DbStatsProviders`, or call `DbStatsProviders.register("ProductName", MyProvider::new)` from the host before collection. +3. Add its saved name to `DbPerfPossibleValues.Provider` and any new access codes to `StatsAccess` and the health check. +4. Make sure the qbit's Liquibase changelog runs on the vendor (the qbit stores its tables in the monitored backend). +5. Test every read against a real database (Testcontainers or embedded), with and without the vendor's statistics prerequisites. + +## MySQL and MariaDB + +What a provider would map (MySQL 8.0+, MariaDB 10.6+ with `performance_schema=ON`, which needs a server restart): + +| Area | Source | Notes | +|---|---|---| +| Statements | `performance_schema.events_statements_summary_by_digest` | Digest id and normalized text, calls, timer (picoseconds), rows examined/sent, temp tables, `FIRST_SEEN`; `TRUNCATE` resets. Needs `SELECT` on `performance_schema` | +| Database counters | `SHOW GLOBAL STATUS` (`Com_commit`, `Com_rollback`, `Created_tmp_disk_tables`, `Innodb_deadlocks`) | Server-wide, not per schema | +| Cache hit ratio | `Innodb_buffer_pool_read_requests` / `Innodb_buffer_pool_reads` | | +| Connection limit and counts | `max_connections`, `information_schema.PROCESSLIST` / `performance_schema.threads` | Needs `PROCESS` to see other users | +| Sessions and blocking | `PROCESSLIST`, `sys.innodb_lock_waits` (MySQL) / `information_schema.INNODB_LOCK_WAITS` (MariaDB) | | +| Tables | `information_schema.TABLES` (`TABLE_ROWS` estimate, `DATA_LENGTH`, `INDEX_LENGTH`) | Sizes are estimates | +| Table access | `performance_schema.table_io_waits_summary_by_table` | Read/write counts, no seq-vs-index split | +| Index usage | `performance_schema.table_io_waits_summary_by_index_usage`, `sys.schema_unused_indexes` | | +| Maintenance | - | InnoDB purge is automatic; no vacuum or dead-row counters | +| Plans | `EXPLAIN FORMAT=JSON`, `EXPLAIN ANALYZE` (MySQL 8.0.18+, tree text) | MariaDB uses `ANALYZE FORMAT=JSON`; a JSON renderer would be needed | + +Beyond the provider class, MySQL support also needs: the MySQL/MariaDB Connector/J on the host classpath; a provider-neutral `ExplainService` path for JSON plans (today JSON plans are PostgreSQL's format and the renderer is PostgreSQL-specific); MySQL-compatible SQL in the changelog (verify it runs, since the qbit stores its own tables there) and in the migration snippets the findings suggest (`CREATE INDEX CONCURRENTLY` is PostgreSQL syntax; MySQL needs `ALGORITHM=INPLACE, LOCK=NONE`); help text that currently names PostgreSQL views; and Testcontainers tests for both MySQL and MariaDB, since their lock views differ. + +## SQLite + +SQLite has no server, sessions, statement statistics or users. A provider could supply only: database size (`PRAGMA page_count` x `page_size`), tables and row counts (`sqlite_schema` plus `COUNT(*)` or `sqlite_stat1` after `ANALYZE`), table and index sizes (the `dbstat` virtual table, when compiled in), index definitions (`sqlite_schema.sql`, `PRAGMA index_list` / `index_info`), and text plans (`EXPLAIN QUERY PLAN`). Statement statistics would have to come from QQQ attribution (the qbit's own statement listener), not the database. It is useful only as an inventory and plan viewer. diff --git a/docs/SPEC.md b/docs/SPEC.md index f78b2ab..15f8b38 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -47,7 +47,9 @@ actions └► Attribution aggregator (memory) │ │ Analyze → f `explain(...)`. `PostgresStatsProvider` is the v1 implementation (PostgreSQL 13+, features that need 16+/17 such as `EXPLAIN (GENERIC_PLAN)` and per-entry `stats_since` are detected). The provider is chosen per monitored backend from the JDBC vendor; an unsupported vendor is - reported in the Setup card, never a crash. + reported in the Setup card, never a crash. `H2StatsProvider` (H2 2.x, `INFORMATION_SCHEMA`) + followed; providers declare per-area `StatsFeature`s and rules are skipped where an area is + missing. See [PROVIDERS.md](PROVIDERS.md). - **Connections** are the backend's own (QQQ `ConnectionManager`), so they run as the app's database user. That user needs the built-in `pg_read_all_stats` role to see other sessions' query text and all statements; without it the qbit runs in a reduced mode and the Setup card and diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/alerts/SnapshotEvaluator.java b/src/main/java/com/kingsrook/qbits/dbperformance/alerts/SnapshotEvaluator.java index 1cd6880..0188c3c 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/alerts/SnapshotEvaluator.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/alerts/SnapshotEvaluator.java @@ -151,7 +151,7 @@ public class SnapshotEvaluator long blocked = activity.stream().filter(r -> "BLOCKED".equals(r.getValueString("reason"))) .filter(r -> Math.max(RuleContext.number(r, "querySeconds"), RuleContext.number(r, "xactSeconds")) >= RuleContext.number(settings, "blockedAlertSeconds")).count(); signal(BLOCKED_CHAIN, backend, null, blocked > 0, "Database blocking chain", blocked + " blocked sessions exceed the duration threshold", snapshot, "db_perf_snapshot"); - if("OK".equals(status) || "NO_READ_ALL_STATS".equals(settings.getValueString("statsAccess"))) + if("OK".equals(status) || List.of("NO_READ_ALL_STATS", "NOT_ADMIN").contains(settings.getValueString("statsAccess"))) { spikes(ctx, snapshot); } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/AnalyzeService.java b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/AnalyzeService.java index 40bbb89..e8aca16 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/AnalyzeService.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/AnalyzeService.java @@ -25,9 +25,12 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; import com.kingsrook.qbits.dbperformance.DbPerfRuntime; import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -70,7 +73,8 @@ public class AnalyzeService /******************************************************************************* - ** Disabled rules are neither evaluated nor reconciled. + ** Disabled rules, and rules whose evidence the backend's provider cannot + ** supply (unknown before the first collection), are neither evaluated nor reconciled. *******************************************************************************/ public List run(String backend) throws Exception { @@ -79,9 +83,15 @@ public class AnalyzeService throw (new IllegalArgumentException("Backend is not configured for this runtime")); } QRecord settings = new DbPerfSettingsService(runtime).ensure(backend); + String provider = settings.getValueString("provider"); + Set available = provider == null ? null : DbStatsProviders.forProviderName(provider).features(); List results = new ArrayList<>(); for(DbPerfRule rule : rules) { + if(available != null && !available.containsAll(rule.requires())) + { + continue; + } try { QRecord row = runtime.asSystem(() -> DbPerfStore.first("db_perf_rule", DbPerfStore.eq("code", rule.code()))); diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/DbPerfRule.java b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/DbPerfRule.java index 3847329..9a4b67d 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/DbPerfRule.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/DbPerfRule.java @@ -22,6 +22,8 @@ package com.kingsrook.qbits.dbperformance.analyze; import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; /******************************************************************************* @@ -39,4 +41,15 @@ public interface DbPerfRule ** Evaluate the configured thresholds against stored observations. *******************************************************************************/ List evaluate(RuleContext ctx) throws Exception; + + + + /******************************************************************************* + ** Statistics areas the evidence comes from. A rule is skipped (neither + ** evaluated nor reconciled) when the backend's provider cannot supply them. + *******************************************************************************/ + default Set requires() + { + return (Set.of()); + } } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/StoredRule.java b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/StoredRule.java index f358d67..5ace304 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/analyze/StoredRule.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/analyze/StoredRule.java @@ -36,6 +36,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -60,6 +61,30 @@ public enum StoredRule implements DbPerfRule } + /******************************************************************************* + ** Provider areas behind each rule's evidence; attribution rules need none. + *******************************************************************************/ + @Override + public Set requires() + { + return (switch(this) + { + case SLOW_QUERY -> Set.of(StatsFeature.STATEMENTS); + case MISSING_INDEX, OVER_AUDITED -> Set.of(StatsFeature.TABLE_ACCESS); + case UNUSED_INDEX -> Set.of(StatsFeature.INDEX_USAGE); + case DUPLICATE_INDEX -> Set.of(StatsFeature.INDEX_DEFINITIONS); + case BLOAT, VACUUM_STALE -> Set.of(StatsFeature.TABLE_MAINTENANCE); + case POOL_SIZING -> Set.of(StatsFeature.MAX_CONNECTIONS, StatsFeature.CONNECTIONS); + case IDLE_IN_TRANSACTION -> Set.of(StatsFeature.SESSIONS); + case CACHE_HIT_LOW -> Set.of(StatsFeature.CACHE_COUNTERS); + case TEMP_SPILL -> Set.of(StatsFeature.STATEMENT_IO); + case DEADLOCKS -> Set.of(StatsFeature.DATABASE_COUNTERS); + case N_PLUS_ONE, HOT_JOB -> Set.of(); + }); + } + + + /******************************************************************************* ** Disabled evaluation is inert even when called outside AnalyzeService. *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java index d615036..f2393cc 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java @@ -50,7 +50,7 @@ public class ExplainSpec extends DbPerformanceJsonSpec public BasicOperation defineBasicOperation() { return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/db-performance/explain") - .withShortSummary("Explain a PostgreSQL query").withLongDescription("PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze. Values and ANALYZE plans are never persisted.")); + .withShortSummary("Explain a query (PostgreSQL JSON or H2 text plan)").withLongDescription("PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze. Values and ANALYZE plans are never persisted.")); } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/collect/SnapshotCollector.java b/src/main/java/com/kingsrook/qbits/dbperformance/collect/SnapshotCollector.java index ef6b5a4..fee3c4e 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/collect/SnapshotCollector.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/collect/SnapshotCollector.java @@ -47,6 +47,7 @@ import com.kingsrook.qbits.dbperformance.provider.IndexStats; import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities; import com.kingsrook.qbits.dbperformance.provider.SessionInfo; import com.kingsrook.qbits.dbperformance.provider.StatementStats; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; import com.kingsrook.qbits.dbperformance.provider.TableStats; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -217,7 +218,7 @@ public class SnapshotCollector Instant now = snapshot.getValueInstant("capturedAt"); QQueryFilter filter = DbPerfStore.eq("backendName", backendName) .withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("OK", "PARTIAL"))) - .withCriteria(new QFilterCriteria("rawXactCommit", QCriteriaOperator.IS_NOT_BLANK)) + .withCriteria(new QFilterCriteria("databaseSizeBytes", QCriteriaOperator.IS_NOT_BLANK)) .withOrderBy(new QFilterOrderBy("capturedAt", false)); QRecord previous = DbPerfStore.first("db_perf_snapshot", filter); Instant previousCapture = previous == null ? null : previous.getValueInstant("capturedAt"); @@ -256,8 +257,9 @@ public class SnapshotCollector } } snapshot.withValue("rawStatsReset", database.statsReset()).withValue("statsResetDetected", databaseReset) - .withValue("databaseSizeBytes", database.sizeBytes()).withValue("maxConnections", database.maxConnections()) - .withValue("cacheHitRatio", ratio(number(snapshot, "blksHit"), number(snapshot, "blksHit") + number(snapshot, "blksRead"))); + .withValue("databaseSizeBytes", database.sizeBytes()) + .withValue("cacheHitRatio", capability.has(StatsFeature.CACHE_COUNTERS) ? ratio(number(snapshot, "blksHit"), number(snapshot, "blksHit") + number(snapshot, "blksRead")) : null) + .withValue("maxConnections", capability.has(StatsFeature.MAX_CONNECTIONS) ? database.maxConnections() : null); if(capability.statements()) { Instant reset = provider.readStatementResetInfo(connection).statsReset(); @@ -441,8 +443,8 @@ public class SnapshotCollector QRecord row = new QRecord().withValue("backendName", backend).withValue("schemaName", entry.schemaName()).withValue("tableName", entry.tableName()).withValue("relid", entry.relid()).withValue("lastSeenAt", now); entry.counters().forEach(row::setValue); entry.maintenance().forEach(row::setValue); - row.withValue("deadRatio", ratio(number(row, "nDeadTup"), number(row, "nLiveTup") + number(row, "nDeadTup"))) - .withValue("seqScanShare", ratio(number(row, "seqScan"), number(row, "seqScan") + number(row, "idxScan"))); + row.withValue("deadRatio", entry.counters().containsKey("nDeadTup") ? ratio(number(row, "nDeadTup"), number(row, "nLiveTup") + number(row, "nDeadTup")) : null) + .withValue("seqScanShare", entry.counters().containsKey("seqScan") ? ratio(number(row, "seqScan"), number(row, "seqScan") + number(row, "idxScan")) : null); if(old == null) { inserts.add(row); diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java index 820e017..8cb1a03 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java @@ -38,6 +38,10 @@ import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; +import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -135,8 +139,9 @@ public class ExplainService boolean generic = !analyze && parsed.parameterCount() > 0; audit.setValue("mode", analyze ? "ANALYZE" : generic ? "GENERIC" : "PLAN"); QRecord settings = new DbPerfSettingsService(runtime).ensure(request.backendName()); - json = execute(parsed.sql(), parameters, analyze, generic, SnapshotCollector.positive(settings, "explainTimeoutMs", 10000)); - plan = PlanTextRenderer.render(json); + Explained explained = execute(parsed, parameters, analyze, generic, SnapshotCollector.positive(settings, "explainTimeoutMs", 10000)); + json = explained.json(); + plan = explained.plan(); if(!analyze && (storedStatement || sql.equals(SqlNormalizer.normalize(sql)))) { audit.setValue("planJson", json); @@ -181,7 +186,7 @@ public class ExplainService } return (new ExplainResult(audit.getValueString("mode"), json, plan.text(), plan.totalCost(), plan.actualTotalMs(), warnings, audit.getValueInteger("id"), runtime.getConfig().getSqlWorkbenchRunnerPath() + "?sql=" + encode("EXPLAIN " + sql) + "&backend=" + encode(request.backendName()), - "Explain this PostgreSQL query plan and suggest improvements.\n\nQuery:\n" + SqlNormalizer.normalize(sql) + "\n\nPlan:\n" + plan.safeText())); + "Explain this " + (json == null ? "database" : "PostgreSQL") + " query plan and suggest improvements.\n\nQuery:\n" + SqlNormalizer.normalize(sql) + "\n\nPlan:\n" + plan.safeText())); } @@ -190,89 +195,141 @@ public class ExplainService ** Roll back even on parse, execution or timeout failure, deallocate after rollback, ** and restore pool state before releasing this dedicated connection. *******************************************************************************/ - private String execute(String sql, List parameters, boolean analyze, boolean generic, int timeout) throws Exception + private Explained execute(ExplainSql.Parsed parsed, List parameters, boolean analyze, boolean generic, int timeout) throws Exception { try(Connection connection = openConnection()) { if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName())) { - throw (new QBadRequestException("Query plans currently require PostgreSQL.")); - } - if(generic && connection.getMetaData().getDatabaseMajorVersion() < 16) - { - throw (new QBadRequestException("Generic plans require PostgreSQL 16 or later. Use ANALYZE with parameter values if authorized.")); - } - boolean autoCommit = connection.getAutoCommit(); - boolean readOnly = connection.isReadOnly(); - String preparedName = "dbperf_" + UUID.randomUUID().toString().replace("-", ""); - boolean prepared = false; - try - { - connection.setAutoCommit(false); - try(Statement statement = connection.createStatement()) + ////////////////////////////////////////////////////////////////// + // other vendors plan through their provider as text (no JSON) // + ////////////////////////////////////////////////////////////////// + DbStatsProvider provider = DbStatsProviders.forConnection(connection); + ProviderCapabilities capability = provider.capabilities(connection); + if(!capability.has(StatsFeature.EXPLAIN_TEXT) || generic && !capability.has(StatsFeature.EXPLAIN_GENERIC)) { - statement.execute("SET TRANSACTION READ ONLY"); - statement.execute("SET LOCAL statement_timeout = " + timeout); - statement.execute("SET LOCAL standard_conforming_strings = on"); - if(!analyze) - { - // GENERIC_PLAN must not be parsed as an external JDBC bind request. - statement.unwrap(PgStatement.class).executeWithFlags("EXPLAIN (" + (generic ? "GENERIC_PLAN, " : "") + "FORMAT JSON) " + sql, QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); - try(ResultSet rows = statement.getResultSet()) - { - rows.next(); - return (rows.getString(1)); - } - } - statement.execute("PREPARE " + preparedName + " AS " + sql); - prepared = true; + throw (new QBadRequestException("Query plans are not available for this database (" + provider.getName() + ").")); } - String arguments = parameters.isEmpty() ? "" : "(" + String.join(", ", Collections.nCopies(parameters.size(), "?")) + ")"; - try(PreparedStatement statement = connection.prepareStatement("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE " + preparedName + arguments)) + if(analyze && !capability.has(StatsFeature.EXPLAIN_ANALYZE)) { - for(int i = 0; i < parameters.size(); i++) + throw (new QBadRequestException("ANALYZE is not available on this " + provider.getName() + " connection (it would execute the query without a read-only transaction). Use PLAN.")); + } + String text; + try + { + text = provider.explainText(connection, parsed.indexedSql(), parameters, analyze, timeout); + } + catch(java.sql.SQLException e) + { + if("42000".equals(e.getSQLState()) && e.getMessage() != null && e.getMessage().contains("H2")) { - statement.setObject(i + 1, parameters.get(i), Types.OTHER); + throw (new QBadRequestException(e.getMessage())); } - // EXECUTE's arguments are utility expressions: unknown bind types cannot - // be inferred in extended Parse. Keep JDBC binding/escaping, but send this - // one command in simple mode so PostgreSQL coerces the quoted strings. - statement.unwrap(PgStatement.class).executeWithFlags(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); + throw (e); + } + return (new Explained(null, new PlanTextRenderer.Plan(text, PlanTextRenderer.redactText(text), null, null))); + } + return (new Explained(postgres(connection, parsed.sql(), parameters, analyze, generic, timeout), null).rendered()); + } + } + + + + /******************************************************************************* + ** PostgreSQL JSON plan; see {@link #execute}. + *******************************************************************************/ + private String postgres(Connection connection, String sql, List parameters, boolean analyze, boolean generic, int timeout) throws Exception + { + if(generic && connection.getMetaData().getDatabaseMajorVersion() < 16) + { + throw (new QBadRequestException("Generic plans require PostgreSQL 16 or later. Use ANALYZE with parameter values if authorized.")); + } + boolean autoCommit = connection.getAutoCommit(); + boolean readOnly = connection.isReadOnly(); + String preparedName = "dbperf_" + UUID.randomUUID().toString().replace("-", ""); + boolean prepared = false; + try + { + connection.setAutoCommit(false); + try(Statement statement = connection.createStatement()) + { + statement.execute("SET TRANSACTION READ ONLY"); + statement.execute("SET LOCAL statement_timeout = " + timeout); + statement.execute("SET LOCAL standard_conforming_strings = on"); + if(!analyze) + { + // GENERIC_PLAN must not be parsed as an external JDBC bind request. + statement.unwrap(PgStatement.class).executeWithFlags("EXPLAIN (" + (generic ? "GENERIC_PLAN, " : "") + "FORMAT JSON) " + sql, QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); try(ResultSet rows = statement.getResultSet()) { rows.next(); return (rows.getString(1)); } } + statement.execute("PREPARE " + preparedName + " AS " + sql); + prepared = true; + } + String arguments = parameters.isEmpty() ? "" : "(" + String.join(", ", Collections.nCopies(parameters.size(), "?")) + ")"; + try(PreparedStatement statement = connection.prepareStatement("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE " + preparedName + arguments)) + { + for(int i = 0; i < parameters.size(); i++) + { + statement.setObject(i + 1, parameters.get(i), Types.OTHER); + } + // EXECUTE's arguments are utility expressions: unknown bind types cannot + // be inferred in extended Parse. Keep JDBC binding/escaping, but send this + // one command in simple mode so PostgreSQL coerces the quoted strings. + statement.unwrap(PgStatement.class).executeWithFlags(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); + try(ResultSet rows = statement.getResultSet()) + { + rows.next(); + return (rows.getString(1)); + } + } + } + finally + { + try + { + connection.rollback(); + if(prepared) + { + try(Statement cleanup = connection.createStatement()) + { + cleanup.execute("DEALLOCATE " + preparedName); + } + finally + { + connection.rollback(); + } + } } finally { - try - { - connection.rollback(); - if(prepared) - { - try(Statement cleanup = connection.createStatement()) - { - cleanup.execute("DEALLOCATE " + preparedName); - } - finally - { - connection.rollback(); - } - } - } - finally - { - connection.setReadOnly(readOnly); - connection.setAutoCommit(autoCommit); - } + connection.setReadOnly(readOnly); + connection.setAutoCommit(autoCommit); } } } + /******************************************************************************* + ** Raw plan (JSON for PostgreSQL, null for text-plan vendors) and its rendering. + *******************************************************************************/ + private record Explained(String json, PlanTextRenderer.Plan plan) + { + /******************************************************************************* + ** Render a PostgreSQL JSON plan. + *******************************************************************************/ + Explained rendered() throws Exception + { + return (new Explained(json, PlanTextRenderer.render(json))); + } + } + + + /******************************************************************************* ** Always obtain a dedicated connection, independent of any caller QQQ transaction. ** Protected for contract tests that verify pool state restoration on driver errors. diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java index dbe12f6..39e849d 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java @@ -51,10 +51,21 @@ public final class ExplainSql /******************************************************************************* - ** Executable SQL with contiguous PostgreSQL bind slots and unquoted tokens. + ** Executable SQL with contiguous PostgreSQL bind slots and unquoted tokens, + ** plus the same SQL with JDBC-indexed ?n slots for vendors such as H2. *******************************************************************************/ - public record Parsed(String sql, List tokens, int parameterCount) + public record Parsed(String sql, List tokens, int parameterCount, String indexedSql) { + /******************************************************************************* + ** PostgreSQL-only form (no JDBC-indexed variant). + *******************************************************************************/ + public Parsed(String sql, List tokens, int parameterCount) + { + this(sql, tokens, parameterCount, sql); + } + + + /******************************************************************************* ** Reject writes, SELECT INTO and every locking SELECT, including nested CTEs. *******************************************************************************/ @@ -83,6 +94,7 @@ public final class ExplainSql StringBuilder result = new StringBuilder(); List tokens = new ArrayList<>(); Set positions = new HashSet<>(); + List slots = new ArrayList<>(); int jdbc = 0; int i = 0; boolean ended = false; @@ -186,6 +198,10 @@ public final class ExplainSql throw (new QBadRequestException("SQL parameter positions must be between 1 and 65535.")); } } + if(i > start + 1) + { + slots.add(new int[] {result.length(), result.length() + (i - start), Integer.parseInt(sql.substring(start + 1, i))}); + } result.append(sql, start, i); continue; } @@ -199,7 +215,9 @@ public final class ExplainSql } else { - result.append('$').append(++jdbc); + String slot = "$" + (++jdbc); + slots.add(new int[] {result.length(), result.length() + slot.length(), jdbc}); + result.append(slot); i++; } continue; @@ -223,7 +241,12 @@ public final class ExplainSql { throw (new QBadRequestException("Use either ? parameters or contiguous $1 through $n parameters.")); } - return (new Parsed(result.toString().trim(), List.copyOf(tokens), Math.max(jdbc, maximum))); + StringBuilder indexed = new StringBuilder(result); + for(int slot = slots.size() - 1; slot >= 0; slot--) + { + indexed.replace(slots.get(slot)[0], slots.get(slot)[1], "?" + slots.get(slot)[2]); + } + return (new Parsed(result.toString().trim(), List.copyOf(tokens), Math.max(jdbc, maximum), indexed.toString().trim())); } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java index bd94a3f..e2037eb 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java @@ -55,6 +55,41 @@ public final class PlanTextRenderer + /******************************************************************************* + ** Literal-free form of a vendor text plan for prompts: quoted strings are + ** redacted everywhere, numbers outside comments (H2 keeps index names and + ** scan counts inside comments). + *******************************************************************************/ + public static String redactText(String text) + { + if(text == null) + { + return (null); + } + StringBuilder result = new StringBuilder(); + java.util.regex.Matcher comment = java.util.regex.Pattern.compile("/\\*.*?\\*/", java.util.regex.Pattern.DOTALL).matcher(text); + int last = 0; + while(comment.find()) + { + result.append(redactLiterals(text.substring(last, comment.start()), true)).append(redactLiterals(comment.group(), false)); + last = comment.end(); + } + return (result.append(redactLiterals(text.substring(last), true)).toString()); + } + + + + /******************************************************************************* + ** Quoted strings, and optionally standalone numbers, become ?. + *******************************************************************************/ + private static String redactLiterals(String text, boolean numbers) + { + String result = text.replaceAll("'(?:[^']|'')*'", "?"); + return (numbers ? result.replaceAll("(? HealthCheckOutcome.up("Full statistics access"); - case "NO_READ_ALL_STATS" -> HealthCheckOutcome.degraded("grant pg_read_all_stats to " + user); - case "NO_PG_STAT_STATEMENTS" -> HealthCheckOutcome.down("Enable pg_stat_statements in shared_preload_libraries and create the extension"); - case "UNSUPPORTED" -> HealthCheckOutcome.down("Use a supported PostgreSQL version (13 or newer) and statistics provider"); + case "NO_READ_ALL_STATS", "NOT_ADMIN", "NO_QUERY_STATISTICS" -> HealthCheckOutcome.degraded(remedy); + case "NO_PG_STAT_STATEMENTS", "UNSUPPORTED" -> HealthCheckOutcome.down(remedy); default -> HealthCheckOutcome.unknown("Statistics access has not been classified"); }; - String safe = List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "UNSUPPORTED").contains(access) ? access : "UNKNOWN"; + String safe = List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "NO_QUERY_STATISTICS", "NOT_ADMIN", "UNSUPPORTED").contains(access) ? access : "UNKNOWN"; return (outcome.withDetails(new HealthDetails().withValue("statsAccess", safe))); } var filter = DbPerfStore.eq("backendName", backend).withOrderBy(new QFilterOrderBy("capturedAt", false)); diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfPossibleValues.java b/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfPossibleValues.java index 15c53c4..db2c25b 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfPossibleValues.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfPossibleValues.java @@ -43,7 +43,7 @@ public final class DbPerfPossibleValues *******************************************************************************/ public enum Provider implements PossibleValueEnum { - POSTGRES, MYSQL, UNSUPPORTED; + POSTGRES, H2, MYSQL, UNSUPPORTED; /******************************************************************************* @@ -73,7 +73,7 @@ public final class DbPerfPossibleValues *******************************************************************************/ public enum StatsAccess implements PossibleValueEnum { - FULL, NO_READ_ALL_STATS, NO_PG_STAT_STATEMENTS, UNSUPPORTED; + FULL, NO_READ_ALL_STATS, NO_PG_STAT_STATEMENTS, NO_QUERY_STATISTICS, NOT_ADMIN, UNSUPPORTED; /******************************************************************************* diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfEnableStatisticsStep.java b/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfEnableStatisticsStep.java new file mode 100644 index 0000000..b2b11de --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfEnableStatisticsStep.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.processes; + + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dbperformance.DbPerfRuntime; +import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; +import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; + + +/******************************************************************************* + ** Settings action: ask the backend's provider to turn on statement statistics + ** (H2: SET QUERY_STATISTICS TRUE as an admin), then record the new access. + ** This is the qbit's only change to the monitored database and it is explicit. + *******************************************************************************/ +public class DbPerfEnableStatisticsStep implements BackendStep +{ + /******************************************************************************* + ** Requires manage_settings; vendor errors become result lines. + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance()); + if(QContext.getQSession() == null || !QContext.getQSession().hasPermission(runtime.getConfig().getPermissionBaseName() + ".manage_settings")) + { + throw (new QException("Managing settings permission is required")); + } + List lines = new ArrayList<>(); + String backend = runtime.getConfig().getBackendName(); + try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(backend))) + { + DbStatsProvider provider = DbStatsProviders.forConnection(connection); + try + { + lines.add(new ProcessSummaryLine(Status.OK, 1, provider.enableStatementStatistics(connection))); + } + catch(SQLFeatureNotSupportedException e) + { + lines.add(new ProcessSummaryLine(Status.ERROR, 0, e.getMessage())); + } + catch(SQLException e) + { + lines.add(new ProcessSummaryLine(Status.ERROR, 0, "The database refused the change (SQLSTATE " + e.getSQLState() + "); an admin database user is required.")); + } + ProviderCapabilities capability = provider.capabilities(connection); + QRecord settings = new DbPerfSettingsService(runtime).ensure(backend); + DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", settings.getValueInteger("id")).withValue("provider", provider.getName()) + .withValue("statsAccess", capability.statsAccess()).withValue("statsAccessMessage", capability.message()))); + lines.add(new ProcessSummaryLine(Status.INFO, 0, "statistics access: " + capability.statsAccess())); + } + catch(QException e) + { + throw (e); + } + catch(Exception e) + { + throw (new QException("Unable to enable statement statistics", e)); + } + output.addValue("processResults", new ArrayList<>(lines)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfProcesses.java b/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfProcesses.java index 75ea6f6..8a732c2 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfProcesses.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/processes/DbPerfProcesses.java @@ -54,6 +54,7 @@ public final class DbPerfProcesses public static final String COLLECT = "dbPerfCollectSnapshot"; public static final String ROLLUP = "dbPerfRollupAndPurge"; public static final String COLLECT_NOW = "dbPerfCollectNow"; + public static final String ENABLE_STATISTICS = "dbPerfEnableStatementStatistics"; /******************************************************************************* @@ -100,6 +101,10 @@ public final class DbPerfProcesses .withTableName("db_perf_snapshot") .withStep(new QFrontendStepMetaData().withName("result").withLabel("Result") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS)))); + instance.addProcess(base(config, ENABLE_STATISTICS, "Enable statement statistics", DbPerfEnableStatisticsStep.class, false) + .withTableName("db_perf_settings") + .withStep(new QFrontendStepMetaData().withName("result").withLabel("Result") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS)))); } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProvider.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProvider.java index 6975d9d..431642f 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProvider.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProvider.java @@ -23,12 +23,17 @@ package com.kingsrook.qbits.dbperformance.provider; import java.sql.Connection; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.util.List; import java.util.Map; +import java.util.Set; /******************************************************************************* - ** Read-only database statistics SPI. Implementations never change the database. + ** Database statistics SPI. Every read is read-only and never changes the + ** database; {@link #enableStatementStatistics} is the one explicit, opt-in + ** change, run only from the settings action by a user who manages settings. + ** A provider is chosen per JDBC product name by {@link DbStatsProviders}. *******************************************************************************/ public interface DbStatsProvider { @@ -90,4 +95,57 @@ public interface DbStatsProvider ** readSettings using the supplied backend connection. *******************************************************************************/ Map readSettings(Connection connection) throws SQLException; + + + + /******************************************************************************* + ** Statistics areas this vendor can supply at best, independent of a + ** connection. Analysis rules that need an absent area are skipped. + *******************************************************************************/ + default Set features() + { + return (Set.of()); + } + + + + /******************************************************************************* + ** Operator next step for a persisted statsAccess code (health and setup). + *******************************************************************************/ + default String remedy(String statsAccess, String databaseUser) + { + return ("FULL".equals(statsAccess) ? "Full statistics access" : "Use a supported database vendor and statistics provider"); + } + + + + /******************************************************************************* + ** Vendor prerequisite rows for the guided setup panel, from saved settings. + *******************************************************************************/ + default List setupChecks(String statsAccess, String message) + { + return (List.of(new SetupCheck("Statistics access", statsAccess, message == null ? remedy(statsAccess, null) : message))); + } + + + + /******************************************************************************* + ** Turn on statement statistics where the vendor allows it at runtime, and + ** return what was done. The default explains why it is not possible. + *******************************************************************************/ + default String enableStatementStatistics(Connection connection) throws SQLException + { + throw (new SQLFeatureNotSupportedException("Statement statistics cannot be enabled at runtime for this database; see the setup panel.")); + } + + + + /******************************************************************************* + ** Vendor text plan for providers without structured JSON plans. The SQL uses + ** JDBC ?n slots; parameters are bound only for ANALYZE. + *******************************************************************************/ + default String explainText(Connection connection, String sql, List parameters, boolean analyze, int timeoutMs) throws SQLException + { + throw (new SQLFeatureNotSupportedException("Query plans are not available for this database.")); + } } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProviders.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProviders.java index 9052046..09a48ef 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProviders.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/DbStatsProviders.java @@ -23,13 +23,31 @@ package com.kingsrook.qbits.dbperformance.provider; import java.sql.Connection; import java.sql.SQLException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; /******************************************************************************* - ** Select a provider from JDBC product identity. + ** Provider registry keyed by JDBC {@code DatabaseMetaData.getDatabaseProductName()}. + ** Adding a vendor is one implementation of {@link DbStatsProvider} plus one + ** {@link #register} call (see docs/PROVIDERS.md); anything unregistered gets + ** {@link UnsupportedStatsProvider}. *******************************************************************************/ public final class DbStatsProviders { + ////////////////////////////////////////////////////////////////////////// + // JDBC product name (lower case) -> provider. MySQL reports "MySQL", // + // MariaDB "MariaDB" (Connector/J 3) and SQLite "SQLite"; none are // + // registered yet: docs/PROVIDERS.md says what each could supply. // + ////////////////////////////////////////////////////////////////////////// + private static final Map> BY_PRODUCT = new ConcurrentHashMap<>(Map.of( + "postgresql", PostgresStatsProvider::new, + "h2", H2StatsProvider::new)); + + + /******************************************************************************* ** Utility class. *******************************************************************************/ @@ -38,11 +56,53 @@ public final class DbStatsProviders } + /******************************************************************************* - ** PostgreSQL first; other vendors are explicitly unsupported. + ** Provider for the connection's JDBC product. *******************************************************************************/ public static DbStatsProvider forConnection(Connection connection) throws SQLException { - return ("PostgreSQL".equalsIgnoreCase(connection.getMetaData().getDatabaseProductName()) ? new PostgresStatsProvider() : new UnsupportedStatsProvider()); + return (forProductName(connection.getMetaData().getDatabaseProductName())); + } + + + + /******************************************************************************* + ** Provider for a JDBC product name; unsupported when not registered. + *******************************************************************************/ + public static DbStatsProvider forProductName(String productName) + { + Supplier supplier = productName == null ? null : BY_PRODUCT.get(productName.toLowerCase(Locale.ROOT)); + return (supplier == null ? new UnsupportedStatsProvider() : supplier.get()); + } + + + + /******************************************************************************* + ** Provider by its saved name (db_perf_settings.provider), for code that has + ** settings but no connection; unsupported for null or unknown names. + *******************************************************************************/ + public static DbStatsProvider forProviderName(String providerName) + { + for(Supplier supplier : BY_PRODUCT.values()) + { + DbStatsProvider provider = supplier.get(); + if(provider.getName().equalsIgnoreCase(providerName)) + { + return (provider); + } + } + return (new UnsupportedStatsProvider()); + } + + + + /******************************************************************************* + ** Register (or replace) the provider for a JDBC product name. Hosts may add + ** vendors before the qbit collects; the name must be a Provider possible value. + *******************************************************************************/ + public static void register(String productName, Supplier provider) + { + BY_PRODUCT.put(productName.toLowerCase(Locale.ROOT), provider); } } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/H2StatsProvider.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/H2StatsProvider.java new file mode 100644 index 0000000..a79a0a3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/H2StatsProvider.java @@ -0,0 +1,691 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.provider; + + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.instant; +import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.read; +import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.scalar; + + +/******************************************************************************* + ** H2 2.x statistics from INFORMATION_SCHEMA. Statement statistics need the + ** database setting QUERY_STATISTICS (URL {@code ;QUERY_STATISTICS=TRUE} or + ** {@code SET QUERY_STATISTICS TRUE} as an admin); every other area works + ** without it. H2 keeps no transaction, cache, scan or vacuum counters, so + ** those areas are reported as unavailable rather than as zeros. See + ** docs/PROVIDERS.md for what is exact, approximated and unsupported. + *******************************************************************************/ +public class H2StatsProvider implements DbStatsProvider +{ + public static final String NAME = "H2"; + + private static final String NO_QUERY_STATISTICS = "NO_QUERY_STATISTICS"; + private static final String NOT_ADMIN = "NOT_ADMIN"; + private static final int MAX_ENTRIES = 1000; + private static final Pattern WORD = Pattern.compile("[A-Za-z_][A-Za-z0-9_$]*"); + + ////////////////////////////////////////////////////////////////////////// + // H2 has no READ ONLY transaction and a rollback does not undo file, // + // session, sequence or DDL side effects, so plans refuse SQL naming // + // these (and any user-defined alias) outright // + ////////////////////////////////////////////////////////////////////////// + private static final Set DENIED = Set.of("FILE_WRITE", "FILE_READ", "CSVWRITE", "CSVREAD", "LINK_SCHEMA", "SCRIPT", "RUNSCRIPT", + "DISK_SPACE_USED", "CANCEL_SESSION", "ABORT_SESSION", "BACKUP", "SHUTDOWN", "CALL", "SET", "DROP", "CREATE", "ALTER", "TRUNCATE", "GRANT", + "REVOKE", "COMMIT", "ROLLBACK", "SAVEPOINT", "CHECKPOINT", "NEXTVAL", "SETVAL", "CURRVAL", "LOCK", "EXECUTE", "INSERT", "UPDATE", + "DELETE", "MERGE", "UPSERT", "REPLACE", "INTO", "FOR", "JAVA_OBJECT", "SLEEP"); + + /////////////////////////////////////////////////////////////////////////// + // statement that guarantees QUERY_STATISTICS has a row when it is on; // + // it and INFORMATION_SCHEMA reads (the collector's own) are not stored // + /////////////////////////////////////////////////////////////////////////// + private static final String PROBE = "SELECT 1 AS DB_PERF_PROBE"; + private static final String DISK_SPACE = "SELECT DISK_SPACE_USED(?)"; + + private static final Set FEATURES = StatsFeature.of(StatsFeature.DATABASE_SIZE, StatsFeature.STATEMENTS, + StatsFeature.TABLE_ROWS, StatsFeature.TABLE_SIZES, StatsFeature.INDEX_DEFINITIONS, StatsFeature.SESSIONS, StatsFeature.BLOCKING, + StatsFeature.CONNECTIONS, StatsFeature.EXPLAIN_TEXT, StatsFeature.EXPLAIN_GENERIC, StatsFeature.EXPLAIN_ANALYZE); + + private static final List SETTINGS = List.of("info.VERSION", "MODE", "MV_STORE", "info.FILE_SIZE", "info.PAGE_SIZE", + "info.CACHE_MAX_SIZE", "info.CACHE_SIZE", "info.CACHE_HIT_RATIO", "info.FILE_READ", "info.FILE_WRITE", "MAX_QUERY_TIMEOUT", "QUERY_TIMEOUT"); + + + + /******************************************************************************* + ** Provider name. + *******************************************************************************/ + @Override + public String getName() + { + return (NAME); + } + + + + /******************************************************************************* + ** Version, admin rights and whether QUERY_STATISTICS is collecting. + *******************************************************************************/ + @Override + public ProviderCapabilities capabilities(Connection connection) throws SQLException + { + int major = connection.getMetaData().getDatabaseMajorVersion(); + int version = version(connection); + boolean supported = major >= 2; + if(!supported) + { + return (new ProviderCapabilities(false, version, false, false, "UNSUPPORTED", "H2 2.0 or newer is required.", Set.of())); + } + boolean statistics = queryStatistics(connection); + boolean admin = admin(connection); + String access = !statistics ? NO_QUERY_STATISTICS : !admin ? NOT_ADMIN : "FULL"; + String message = switch(access) + { + case NO_QUERY_STATISTICS -> "H2 query statistics are off. Run Enable statement statistics on Settings as an H2 admin user, or add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 to the JDBC URL."; + case NOT_ADMIN -> "The H2 user is not an admin, so only its own session is visible. Use an admin user (ALTER USER ... ADMIN TRUE) to see every session and blocking."; + default -> "Full statistics access (H2 does not keep transaction, cache, scan or vacuum counters)."; + }; + EnumSet available = EnumSet.copyOf(FEATURES); + if(!statistics) + { + available.remove(StatsFeature.STATEMENTS); + } + if(!admin) + { + available.remove(StatsFeature.BLOCKING); + } + else + { + //////////////////////////////////////////////////////////////////// + // an admin may call file and session functions; ANALYZE executes // + // the query, so it is offered only to non-admin users // + //////////////////////////////////////////////////////////////////// + available.remove(StatsFeature.EXPLAIN_ANALYZE); + } + return (new ProviderCapabilities(true, version, statistics, admin, access, message, available)); + } + + + + /******************************************************************************* + ** Areas H2 can supply at best. + *******************************************************************************/ + @Override + public Set features() + { + return (FEATURES); + } + + + + /******************************************************************************* + ** Size from the store; H2 has no cumulative counters, reset time or limit. + *******************************************************************************/ + @Override + public DatabaseStats readDatabase(Connection connection) throws SQLException + { + List size = read(connection, "SELECT SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS WHERE SETTING_NAME = 'info.FILE_SIZE'", row -> parseLong(row.getString(1))); + return (new DatabaseStats(Map.of(), null, 0, size.isEmpty() || size.getFirst() == null ? 0 : size.getFirst())); + } + + + + /******************************************************************************* + ** QUERY_STATISTICS rows (times in milliseconds), without connection setup + ** (see {@link #excluded}). H2 has no query id or user, so the id is a stable + ** hash of the statement text and the user is null. + *******************************************************************************/ + @Override + public List readStatements(Connection connection) throws SQLException + { + List result = new ArrayList<>(); + read(connection, "SELECT SQL_STATEMENT, EXECUTION_COUNT, CUMULATIVE_EXECUTION_TIME, AVERAGE_EXECUTION_TIME, MAX_EXECUTION_TIME, CUMULATIVE_ROW_COUNT FROM INFORMATION_SCHEMA.QUERY_STATISTICS", row -> + { + String sql = row.getString("SQL_STATEMENT"); + if(sql != null && !excluded(sql)) + { + result.add(new StatementStats(hash(sql), null, true, sql, row.getLong("EXECUTION_COUNT"), millis(row.getDouble("CUMULATIVE_EXECUTION_TIME")), + millis(row.getDouble("AVERAGE_EXECUTION_TIME")), millis(row.getDouble("MAX_EXECUTION_TIME")), row.getLong("CUMULATIVE_ROW_COUNT"), 0, 0, 0, null, null)); + } + return (null); + }); + return (result); + } + + + + /******************************************************************************* + ** Not application statements: the collector's own probe, table sizing and + ** INFORMATION_SCHEMA reads, and connection setup. H2 replays every JDBC URL setting as a SET on + ** each new connection and drivers call DATABASE() for the catalog, so with + ** unpooled connections these dominate the call counts without being work. + *******************************************************************************/ + static boolean excluded(String sql) + { + String upper = sql.strip().toUpperCase(Locale.ROOT); + return (PROBE.equals(sql) || DISK_SPACE.equals(sql) || upper.contains("INFORMATION_SCHEMA.") || upper.startsWith("SET ") || upper.equals("CALL DATABASE()")); + } + + + + /******************************************************************************* + ** H2 keeps no reset time; a reset shows as falling call counts instead. + *******************************************************************************/ + @Override + public StatementResetInfo readStatementResetInfo(Connection connection) + { + return (new StatementResetInfo(null, 0)); + } + + + + /******************************************************************************* + ** Base tables with H2's row-count estimate and DISK_SPACE_USED (table data + ** only: H2 cannot size indexes, so totalBytes equals tableBytes). + *******************************************************************************/ + @Override + public List readTables(Connection connection) throws SQLException + { + List result = new ArrayList<>(); + List tables = read(connection, "SELECT TABLE_SCHEMA, TABLE_NAME, ROW_COUNT_ESTIMATE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA <> 'INFORMATION_SCHEMA' ORDER BY TABLE_SCHEMA, TABLE_NAME", + row -> new String[] {row.getString(1), row.getString(2), row.getString(3)}); + try(PreparedStatement size = connection.prepareStatement(DISK_SPACE)) + { + size.setQueryTimeout(30); + for(String[] table : tables) + { + Map counters = new LinkedHashMap<>(); + counters.put("nLiveTup", table[2] == null ? 0L : Long.parseLong(table[2])); + Long bytes = diskSpace(size, quote(table[0]) + "." + quote(table[1])); + if(bytes != null) + { + counters.put("tableBytes", bytes); + counters.put("totalBytes", bytes); + } + result.add(new TableStats(table[0], table[1], hash("TABLE:" + table[0] + "." + table[1]) & Long.MAX_VALUE, counters, Map.of())); + } + } + return (result); + } + + + + /******************************************************************************* + ** Index definitions and ordered key columns. H2 has no index usage or size + ** statistics; constraint-generated indexes are marked USING CONSTRAINT so + ** they are never proposed for removal. + *******************************************************************************/ + @Override + public List readIndexes(Connection connection) throws SQLException + { + Map indexes = new LinkedHashMap<>(); + String sql = "SELECT i.INDEX_SCHEMA, i.TABLE_NAME, i.INDEX_NAME, i.INDEX_TYPE_NAME, i.IS_GENERATED, c.COLUMN_NAME, c.ORDERING_SPECIFICATION FROM INFORMATION_SCHEMA.INDEXES i " + + "JOIN INFORMATION_SCHEMA.INDEX_COLUMNS c ON c.INDEX_SCHEMA = i.INDEX_SCHEMA AND c.TABLE_NAME = i.TABLE_NAME AND c.INDEX_NAME = i.INDEX_NAME " + + "WHERE i.INDEX_SCHEMA <> 'INFORMATION_SCHEMA' ORDER BY i.INDEX_SCHEMA, i.TABLE_NAME, i.INDEX_NAME, c.ORDINAL_POSITION"; + read(connection, sql, row -> + { + IndexBuilder index = indexes.computeIfAbsent(row.getString(1) + "." + row.getString(3), key -> new IndexBuilder()); + index.schema = row.getString(1); + index.table = row.getString(2); + index.name = row.getString(3); + index.type = row.getString(4) == null ? "" : row.getString(4).toUpperCase(Locale.ROOT); + index.generated = row.getBoolean(5); + String order = "DESC".equalsIgnoreCase(row.getString(7)) ? " DESC" : ""; + index.columns.add(row.getString(6) + order); + index.quoted.add(quote(row.getString(6)) + order); + return (null); + }); + List result = new ArrayList<>(); + for(IndexBuilder index : indexes.values()) + { + boolean primary = index.type.contains("PRIMARY"); + boolean unique = primary || index.type.contains("UNIQUE"); + String columns = String.join(", ", index.columns); + String keys = String.join(", ", index.quoted); + String definition = primary ? "ALTER TABLE " + quote(index.schema) + "." + quote(index.table) + " ADD PRIMARY KEY (" + keys + ")" + : "CREATE " + (unique ? "UNIQUE " : "") + "INDEX " + quote(index.name) + " ON " + quote(index.schema) + "." + quote(index.table) + (index.generated ? " USING CONSTRAINT" : "") + " (" + keys + ")"; + result.add(new IndexStats(index.schema, index.table, index.name, hash("INDEX:" + index.schema + "." + index.name) & Long.MAX_VALUE, definition, columns, unique, primary, true, 0, 0, 0, 0, 0)); + } + return (result); + } + + + + /******************************************************************************* + ** Other sessions in PostgreSQL's state vocabulary: RUNNING is active, BLOCKED + ** is active waiting on a Lock, SLEEP with uncommitted changes is idle in + ** transaction. H2 has no transaction start, so an idle transaction's age is + ** the time since it went idle (a lower bound). + *******************************************************************************/ + @Override + public List readSessions(Connection connection) throws SQLException + { + Instant now = Instant.now(); + String sql = "SELECT SESSION_ID, USER_NAME, CLIENT_INFO, CLIENT_ADDR, SERVER, SESSION_STATE, CONTAINS_UNCOMMITTED, BLOCKER_ID, EXECUTING_STATEMENT, EXECUTING_STATEMENT_START, SLEEP_SINCE " + + "FROM INFORMATION_SCHEMA.SESSIONS WHERE SESSION_ID <> SESSION_ID() ORDER BY SESSION_ID"; + return (read(connection, sql, row -> + { + String state = state(row.getString("SESSION_STATE"), row.getBoolean("CONTAINS_UNCOMMITTED")); + boolean blocked = "BLOCKED".equalsIgnoreCase(row.getString("SESSION_STATE")); + Instant queryStart = instant(row, "EXECUTING_STATEMENT_START"); + Instant sleepSince = instant(row, "SLEEP_SINCE"); + boolean active = "active".equals(state); + String query = row.getString("EXECUTING_STATEMENT"); + Integer blocker = row.getObject("BLOCKER_ID") == null ? null : row.getInt("BLOCKER_ID"); + return (new SessionInfo(row.getInt("SESSION_ID"), row.getString("USER_NAME"), row.getString("CLIENT_INFO"), row.getString("CLIENT_ADDR"), state, + blocked ? "Lock" : null, blocked ? "row lock" : null, row.getString("SERVER") == null ? "embedded" : row.getString("SERVER"), + null, queryStart, active ? queryStart : sleepSince, active ? seconds(queryStart, now) : null, + state.startsWith("idle in transaction") ? seconds(sleepSince, now) : null, blocker == null ? List.of() : List.of(blocker), query == null ? null : hash(query), query)); + })); + } + + + + /******************************************************************************* + ** Every visible session, including this one, by user, client and state. + *******************************************************************************/ + @Override + public List readConnections(Connection connection) throws SQLException + { + Map, Integer> counts = new LinkedHashMap<>(); + read(connection, "SELECT USER_NAME, CLIENT_INFO, SESSION_STATE, CONTAINS_UNCOMMITTED FROM INFORMATION_SCHEMA.SESSIONS ORDER BY SESSION_ID", row -> + counts.merge(Arrays.asList(row.getString(1), row.getString(2), state(row.getString(3), row.getBoolean(4))), 1, Integer::sum)); + List result = new ArrayList<>(); + counts.forEach((key, count) -> result.add(new ConnectionCount(key.get(0), key.get(1), key.get(2), count))); + return (result); + } + + + + /******************************************************************************* + ** Operational, non-secret settings plus the QUERY_STATISTICS state. + *******************************************************************************/ + @Override + public Map readSettings(Connection connection) throws SQLException + { + Map result = new LinkedHashMap<>(); + read(connection, "SELECT SETTING_NAME, SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS", row -> SETTINGS.contains(row.getString(1)) ? result.put(row.getString(1), row.getString(2)) : null); + result.put("QUERY_STATISTICS", queryStatistics(connection) ? "TRUE" : "FALSE"); + return (result); + } + + + + /******************************************************************************* + ** Health and setup next steps for H2's access codes. + *******************************************************************************/ + @Override + public String remedy(String statsAccess, String databaseUser) + { + return (switch(statsAccess == null ? "" : statsAccess) + { + case "FULL" -> "Full statistics access"; + case NO_QUERY_STATISTICS -> "Turn on H2 query statistics: run Enable statement statistics on Settings, or add ;QUERY_STATISTICS=TRUE to the JDBC URL"; + case NOT_ADMIN -> "Make " + databaseUser + " an H2 admin (ALTER USER " + databaseUser + " ADMIN TRUE) to see every session"; + default -> "Use H2 2.0 or newer"; + }); + } + + + + /******************************************************************************* + ** Guided setup rows: query statistics and the admin role. + *******************************************************************************/ + @Override + public List setupChecks(String statsAccess, String message) + { + return (List.of(new SetupCheck("H2 QUERY_STATISTICS", statsAccess, "Run Enable statement statistics on Settings (admin user), or add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000 to the JDBC URL; then run Collect now."), + new SetupCheck("H2 admin user", statsAccess, message == null ? "Use an H2 admin user to see every session and blocking." : message))); + } + + + + /******************************************************************************* + ** SET QUERY_STATISTICS TRUE (admin only), raising H2's default of 100 kept + ** statements: H2 evicts a third of them, oldest first, whenever the cap is + ** passed, and an application easily runs more distinct statements. H2 does + ** not persist either setting: they last until the database closes, so also + ** put them in the JDBC URL. An existing setup is left untouched. + *******************************************************************************/ + @Override + public String enableStatementStatistics(Connection connection) throws SQLException + { + if(queryStatistics(connection)) + { + return ("H2 query statistics are already on."); + } + try(Statement statement = connection.createStatement()) + { + statement.execute("SET QUERY_STATISTICS_MAX_ENTRIES " + MAX_ENTRIES); + statement.execute("SET QUERY_STATISTICS TRUE"); + } + return ("H2 query statistics are on (up to " + MAX_ENTRIES + " statements) until the database closes. Add ;QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=" + MAX_ENTRIES + " to the JDBC URL to keep them on after a restart."); + } + + + + /******************************************************************************* + ** EXPLAIN (not executed) or EXPLAIN ANALYZE (executed with bound values). + ** H2 has no READ ONLY transaction, so both refuse more than one statement + ** and any SQL naming a side-effecting function, DDL/DML keyword or + ** user-defined alias; ANALYZE additionally requires a single SELECT/WITH + ** and a non-admin user. The connection is marked read-only (a hint in H2) + ** and the transaction always rolls back. + *******************************************************************************/ + @Override + public String explainText(Connection connection, String sql, List parameters, boolean analyze, int timeoutMs) throws SQLException + { + List words = words(sql); + if(statements(sql) > 1) + { + throw (new SQLException("Query plans on H2 accept exactly one statement.", "42000")); + } + List aliases = read(connection, "SELECT UPPER(ROUTINE_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA <> 'INFORMATION_SCHEMA'", row -> row.getString(1)); + for(String word : words) + { + if(DENIED.contains(word) || aliases.contains(word)) + { + throw (new SQLException("Query plans on H2 refuse " + word + ": H2 cannot run it read-only.", "42000")); + } + } + if(analyze) + { + if(words.isEmpty() || !Set.of("SELECT", "WITH").contains(words.getFirst())) + { + throw (new SQLException("EXPLAIN ANALYZE on H2 accepts a single SELECT only.", "42000")); + } + if(admin(connection)) + { + throw (new SQLException("EXPLAIN ANALYZE on H2 is disabled for admin users (it executes the query and H2 has no read-only transactions). Use PLAN, or connect as a non-admin user.", "42000")); + } + } + boolean autoCommit = connection.getAutoCommit(); + boolean readOnly = connection.isReadOnly(); + try + { + connection.setReadOnly(true); + connection.setAutoCommit(false); + if(!analyze) + { + //////////////////////////////////////////////////////////////// + // a plain statement plans ?n slots without values (generic) // + //////////////////////////////////////////////////////////////// + try(Statement statement = connection.createStatement()) + { + statement.setQueryTimeout(Math.max(1, (timeoutMs + 999) / 1000)); + return (plan(statement.executeQuery("EXPLAIN " + sql))); + } + } + try(PreparedStatement statement = connection.prepareStatement("EXPLAIN ANALYZE " + sql)) + { + statement.setQueryTimeout(Math.max(1, (timeoutMs + 999) / 1000)); + for(int i = 0; i < parameters.size(); i++) + { + statement.setString(i + 1, parameters.get(i)); + } + return (plan(statement.executeQuery())); + } + } + finally + { + try + { + connection.rollback(); + } + finally + { + connection.setAutoCommit(autoCommit); + connection.setReadOnly(readOnly); + } + } + } + + + + /******************************************************************************* + ** Upper-case words outside string literals and comments; double-quoted + ** identifiers count as words, since H2 resolves quoted function names. + *******************************************************************************/ + static List words(String sql) + { + String code = code(sql); + List words = new ArrayList<>(); + Matcher matcher = WORD.matcher(code); + while(matcher.find()) + { + words.add(matcher.group().toUpperCase(Locale.ROOT)); + } + return (words); + } + + + + /******************************************************************************* + ** Statements separated by semicolons outside literals and comments. + *******************************************************************************/ + static int statements(String sql) + { + int count = 0; + for(String part : code(sql).split(";")) + { + count += part.isBlank() ? 0 : 1; + } + return (count); + } + + + + /******************************************************************************* + ** SQL with string literals and comments blanked, quoted identifiers unquoted. + *******************************************************************************/ + private static String code(String sql) + { + return (sql.replaceAll("(?s)/\\*.*?\\*/", " ").replaceAll("--[^\n]*", " ").replaceAll("'(?:[^']|'')*'", " ").replace('"', ' ')); + } + + + + /******************************************************************************* + ** Plan rows joined as text; closes the result. + *******************************************************************************/ + private static String plan(ResultSet rows) throws SQLException + { + try(rows) + { + StringBuilder plan = new StringBuilder(); + while(rows.next()) + { + plan.append(plan.isEmpty() ? "" : "\n").append(rows.getString(1)); + } + return (plan.toString()); + } + } + + + + /******************************************************************************* + ** QUERY_STATISTICS is not exposed in SETTINGS; when it is on, the probe just + ** run is always recorded, and when it is off the view is empty. + *******************************************************************************/ + boolean queryStatistics(Connection connection) throws SQLException + { + read(connection, PROBE, row -> null); + return (scalar(connection, "SELECT COUNT(*) FROM INFORMATION_SCHEMA.QUERY_STATISTICS") > 0); + } + + + + /******************************************************************************* + ** Non-admin users see only their own USERS row and session. + *******************************************************************************/ + private boolean admin(Connection connection) throws SQLException + { + return (!read(connection, "SELECT IS_ADMIN FROM INFORMATION_SCHEMA.USERS WHERE USER_NAME = CURRENT_USER", row -> row.getBoolean(1)).stream().filter(Boolean::booleanValue).toList().isEmpty()); + } + + + + /******************************************************************************* + ** major*1000000 + minor*1000 + build, e.g. 2004240 for 2.4.240. + *******************************************************************************/ + private int version(Connection connection) throws SQLException + { + String[] parts = read(connection, "SELECT H2VERSION()", row -> row.getString(1)).getFirst().split("[.\\s]"); + int version = 0; + for(int i = 0; i < 3; i++) + { + version = version * 1000 + (i < parts.length && parts[i].matches("\\d+") ? Integer.parseInt(parts[i]) : 0); + } + return (version); + } + + + + /******************************************************************************* + ** PostgreSQL-compatible state names used by collection and presentation. + *******************************************************************************/ + static String state(String h2State, boolean uncommitted) + { + String value = h2State == null ? "" : h2State.toUpperCase(Locale.ROOT); + return (switch(value) + { + case "RUNNING", "BLOCKED" -> "active"; + case "SLEEP" -> uncommitted ? "idle in transaction" : "idle"; + default -> value.toLowerCase(Locale.ROOT); + }); + } + + + + /******************************************************************************* + ** Stable 64-bit identity for text (statement, table or index key). + *******************************************************************************/ + static long hash(String text) + { + try + { + return (ByteBuffer.wrap(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))).getLong()); + } + catch(NoSuchAlgorithmException e) + { + throw (new IllegalStateException("SHA-256 is required", e)); + } + } + + + + /******************************************************************************* + ** DISK_SPACE_USED for one quoted table; null when H2 cannot size it. + *******************************************************************************/ + private static Long diskSpace(PreparedStatement size, String table) + { + try + { + size.setString(1, table); + try(ResultSet rows = size.executeQuery()) + { + return (rows.next() ? rows.getLong(1) : null); + } + } + catch(SQLException e) + { + return (null); + } + } + + + + /******************************************************************************* + ** Quoted identifier. + *******************************************************************************/ + private static String quote(String identifier) + { + return ("\"" + identifier.replace("\"", "\"\"") + "\""); + } + + + + /******************************************************************************* + ** Milliseconds as a 3-decimal BigDecimal. + *******************************************************************************/ + private static BigDecimal millis(double value) + { + return (BigDecimal.valueOf(value).setScale(3, RoundingMode.HALF_UP)); + } + + + + /******************************************************************************* + ** Elapsed seconds, or null without a start. + *******************************************************************************/ + private static BigDecimal seconds(Instant start, Instant now) + { + return (start == null ? null : BigDecimal.valueOf(Math.max(0, Duration.between(start, now).toMillis())).divide(BigDecimal.valueOf(1000), 3, RoundingMode.HALF_UP)); + } + + + + /******************************************************************************* + ** Numeric setting value, or null. + *******************************************************************************/ + private static Long parseLong(String value) + { + return (value != null && value.matches("\\d+") ? Long.parseLong(value) : null); + } + + + + /******************************************************************************* + ** Index rows folded per index. + *******************************************************************************/ + private static class IndexBuilder + { + private String schema; + private String table; + private String name; + private String type; + private boolean generated; + private final List columns = new ArrayList<>(); + private final List quoted = new ArrayList<>(); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/JdbcReads.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/JdbcReads.java new file mode 100644 index 0000000..7b453f9 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/JdbcReads.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.provider; + + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** Bounded-time JDBC reads shared by statistics providers. + *******************************************************************************/ +final class JdbcReads +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private JdbcReads() + { + } + + + + /******************************************************************************* + ** Execute one bounded-time read and close JDBC resources. + *******************************************************************************/ + static List read(Connection connection, String sql, RowReader mapper) throws SQLException + { + List result = new ArrayList<>(); + try(Statement statement = connection.createStatement()) + { + statement.setQueryTimeout(30); + try(ResultSet rows = statement.executeQuery(sql)) + { + while(rows.next()) + { + result.add(mapper.read(rows)); + } + } + } + return (result); + } + + + + /******************************************************************************* + ** A single numeric read. + *******************************************************************************/ + static long scalar(Connection connection, String sql) throws SQLException + { + return (read(connection, sql, row -> row.getLong(1)).getFirst()); + } + + + + /******************************************************************************* + ** Timestamp conversion preserves null values. + *******************************************************************************/ + static Instant instant(ResultSet row, String column) throws SQLException + { + Timestamp timestamp = row.getTimestamp(column); + return (timestamp == null ? null : timestamp.toInstant()); + } + + + + /******************************************************************************* + ** Checked JDBC row mapping. + *******************************************************************************/ + interface RowReader + { + /******************************************************************************* + ** Map one row. + *******************************************************************************/ + T read(ResultSet row) throws SQLException; + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/PostgresStatsProvider.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/PostgresStatsProvider.java index 71ae670..4f2745f 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/provider/PostgresStatsProvider.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/PostgresStatsProvider.java @@ -24,13 +24,16 @@ package com.kingsrook.qbits.dbperformance.provider; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; import java.time.Instant; import java.util.ArrayList; +import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.instant; +import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.read; +import static com.kingsrook.qbits.dbperformance.provider.JdbcReads.scalar; /******************************************************************************* @@ -85,7 +88,60 @@ public class PostgresStatsProvider implements DbStatsProvider } } } - return (new ProviderCapabilities(version >= 130000, version, installed, privileged, access, message)); + EnumSet available = EnumSet.copyOf(features()); + if(!installed) + { + available.removeAll(EnumSet.of(StatsFeature.STATEMENTS, StatsFeature.STATEMENT_IO, StatsFeature.STATEMENT_RESET)); + } + if(version < 140000) + { + available.remove(StatsFeature.STATEMENT_RESET); + } + if(version < 160000) + { + available.remove(StatsFeature.EXPLAIN_GENERIC); + } + return (new ProviderCapabilities(version >= 130000, version, installed, privileged, access, message, version >= 130000 ? available : EnumSet.noneOf(StatsFeature.class))); + } + + + + /******************************************************************************* + ** Every area: the cumulative statistics views cover them all. + *******************************************************************************/ + @Override + public Set features() + { + return (StatsFeature.of(StatsFeature.values())); + } + + + + /******************************************************************************* + ** Role and extension prerequisites for each access code. + *******************************************************************************/ + @Override + public String remedy(String statsAccess, String databaseUser) + { + return (switch(statsAccess == null ? "" : statsAccess) + { + case "FULL" -> "Full statistics access"; + case "NO_READ_ALL_STATS" -> "grant pg_read_all_stats to " + databaseUser; + case "NO_PG_STAT_STATEMENTS" -> "Enable pg_stat_statements in shared_preload_libraries and create the extension"; + default -> "Use a supported PostgreSQL version (13 or newer) and statistics provider"; + }); + } + + + + /******************************************************************************* + ** Guided setup rows: the extension and the statistics role. + *******************************************************************************/ + @Override + public List setupChecks(String statsAccess, String message) + { + return (List.of(new SetupCheck("pg_stat_statements", statsAccess, "Preload and create pg_stat_statements; then run Collect now."), + new SetupCheck("pg_read_all_stats", statsAccess, message == null ? "Grant pg_read_all_stats to the application's database role." : message))); } @@ -237,25 +293,6 @@ public class PostgresStatsProvider implements DbStatsProvider } - /******************************************************************************* - ** A single numeric read. - *******************************************************************************/ - private long scalar(Connection connection, String sql) throws SQLException - { - return (read(connection, sql, row -> row.getLong(1)).getFirst()); - } - - - /******************************************************************************* - ** Timestamp conversion preserves null reset/maintenance values. - *******************************************************************************/ - private static Instant instant(ResultSet row, String column) throws SQLException - { - Timestamp timestamp = row.getTimestamp(column); - return (timestamp == null ? null : timestamp.toInstant()); - } - - /******************************************************************************* ** Copy cumulative counters to the entity's camelCase names. *******************************************************************************/ @@ -282,37 +319,4 @@ public class PostgresStatsProvider implements DbStatsProvider } return (value.toString()); } - - - /******************************************************************************* - ** Execute one bounded-time read and close JDBC resources. - *******************************************************************************/ - private static List read(Connection connection, String sql, RowReader mapper) throws SQLException - { - List result = new ArrayList<>(); - try(Statement statement = connection.createStatement()) - { - statement.setQueryTimeout(30); - try(ResultSet rows = statement.executeQuery(sql)) - { - while(rows.next()) - { - result.add(mapper.read(rows)); - } - } - } - return (result); - } - - - /******************************************************************************* - ** Checked JDBC row mapping. - *******************************************************************************/ - private interface RowReader - { - /******************************************************************************* - ** Map one row. - *******************************************************************************/ - T read(ResultSet row) throws SQLException; - } } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/ProviderCapabilities.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/ProviderCapabilities.java index 879d0d4..bc04d5f 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/provider/ProviderCapabilities.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/ProviderCapabilities.java @@ -21,11 +21,44 @@ package com.kingsrook.qbits.dbperformance.provider; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; /******************************************************************************* - ** ProviderCapabilities read from database statistics; no persistence or side effects. + ** What one provider can read on one connection; no persistence or side effects. + ** {@code statsAccess} is a stable code persisted on settings (FULL, or the most + ** important missing prerequisite); {@code message} is the provider's own + ** next step; {@code features} lists the statistics areas available right now. *******************************************************************************/ -public record ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message) +public record ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message, Set features) { + /******************************************************************************* + ** Defensive copy. + *******************************************************************************/ + public ProviderCapabilities + { + features = features == null || features.isEmpty() ? Set.of() : Collections.unmodifiableSet(EnumSet.copyOf(features)); + } + + + + /******************************************************************************* + ** Pre-feature form: a supported provider is assumed to offer every area. + *******************************************************************************/ + public ProviderCapabilities(boolean supported, int version, boolean statements, boolean readAllStats, String statsAccess, String message) + { + this(supported, version, statements, readAllStats, statsAccess, message, supported ? EnumSet.allOf(StatsFeature.class) : Set.of()); + } + + + + /******************************************************************************* + ** Whether an area is available on this connection. + *******************************************************************************/ + public boolean has(StatsFeature feature) + { + return (features.contains(feature)); + } } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/SetupCheck.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/SetupCheck.java new file mode 100644 index 0000000..909070e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/SetupCheck.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.provider; + + + + +/******************************************************************************* + ** One guided-setup row: the prerequisite, its observed state and the next step. + *******************************************************************************/ +public record SetupCheck(String check, String state, String nextStep) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/StatsFeature.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/StatsFeature.java new file mode 100644 index 0000000..5190174 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/StatsFeature.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.provider; + + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + + +/******************************************************************************* + ** One area of database statistics. A provider declares which areas its vendor + ** can supply ({@link DbStatsProvider#features()}) and which are available on a + ** given connection ({@link ProviderCapabilities#features()}); presentation and + ** analysis degrade per area instead of per vendor. + *******************************************************************************/ +public enum StatsFeature +{ + /** Transaction, block, temp-file and deadlock counters for the database. */ + DATABASE_COUNTERS("Database counters"), + /** Buffer-cache hit and read counters (cache hit ratio). */ + CACHE_COUNTERS("Cache hit ratio"), + /** Database size on disk. */ + DATABASE_SIZE("Database size"), + /** Configured connection limit. */ + MAX_CONNECTIONS("Connection limit"), + /** Cumulative per-statement calls, time and rows. */ + STATEMENTS("Statement statistics"), + /** Per-statement shared-block and temp-block I/O. */ + STATEMENT_IO("Statement I/O"), + /** A server-side reset time for statement statistics. */ + STATEMENT_RESET("Statement reset time"), + /** Live row estimates per table. */ + TABLE_ROWS("Table row estimates"), + /** Table, index and toast sizes. */ + TABLE_SIZES("Table sizes"), + /** Sequential/index scan and tuple write counters per table. */ + TABLE_ACCESS("Table access counters"), + /** Dead tuples, vacuum/analyze times and freeze age. */ + TABLE_MAINTENANCE("Table maintenance"), + /** Index definitions and key columns. */ + INDEX_DEFINITIONS("Index definitions"), + /** Per-index scan counters. */ + INDEX_USAGE("Index usage"), + /** Per-index size. */ + INDEX_SIZES("Index sizes"), + /** Live sessions with state and current statement. */ + SESSIONS("Live sessions"), + /** Which session blocks which. */ + BLOCKING("Blocking"), + /** Wait event type and name per session. */ + WAIT_EVENTS("Wait events"), + /** Transaction start time per session. */ + TRANSACTION_AGE("Transaction age"), + /** Connection counts by user, application and state. */ + CONNECTIONS("Connection counts"), + /** Plans as structured JSON with costs (renderable tree). */ + EXPLAIN_JSON("Structured query plans"), + /** Plans as vendor text (no cost tree). */ + EXPLAIN_TEXT("Text query plans"), + /** Plans for parameterized SQL without values. */ + EXPLAIN_GENERIC("Generic plans"), + /** Executing the query to measure the plan (ANALYZE). */ + EXPLAIN_ANALYZE("Explain analyze"); + + private final String label; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + StatsFeature(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Human-readable area name. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Immutable copy of a feature list. + *******************************************************************************/ + public static Set of(StatsFeature... features) + { + EnumSet set = EnumSet.noneOf(StatsFeature.class); + Collections.addAll(set, features); + return (Collections.unmodifiableSet(set)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/provider/UnsupportedStatsProvider.java b/src/main/java/com/kingsrook/qbits/dbperformance/provider/UnsupportedStatsProvider.java index 0ad4679..0d075e5 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/provider/UnsupportedStatsProvider.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/provider/UnsupportedStatsProvider.java @@ -47,7 +47,7 @@ public class UnsupportedStatsProvider implements DbStatsProvider @Override public ProviderCapabilities capabilities(Connection connection) { - return (new ProviderCapabilities(false, 0, false, false, "UNSUPPORTED", "This database vendor is not supported; PostgreSQL 13 or newer is required.")); + return (new ProviderCapabilities(false, 0, false, false, "UNSUPPORTED", "This database vendor is not supported; PostgreSQL 13+ and H2 2.x are (see docs/PROVIDERS.md).")); } diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java index 9c61e4a..f9c5c64 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java @@ -37,7 +37,10 @@ import com.kingsrook.qbits.dbperformance.api.DbPerformanceData; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; import com.kingsrook.qbits.dbperformance.provider.ConnectionCount; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProvider; import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; +import com.kingsrook.qbits.dbperformance.provider.SetupCheck; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -131,9 +134,12 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer boolean attribution = settings == null ? Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()) : Boolean.TRUE.equals(settings.getValueBoolean("attributionEnabled")); List> rows = new ArrayList<>(); rows.add(check("Monitored backend", runtime.getConfig().getBackendName(), "Review Settings for this backend.")); - rows.add(check("Provider", settings == null ? "Not detected yet" : settings.getValueString("provider"), "PostgreSQL 13+ is supported.")); - rows.add(check("pg_stat_statements", access, "Preload and create pg_stat_statements; then run Collect now.")); - rows.add(check("pg_read_all_stats", access, settings == null ? "Grant pg_read_all_stats to the application's database role." : settings.getValueString("statsAccessMessage"))); + rows.add(check("Provider", settings == null ? "Not detected yet" : settings.getValueString("provider"), "PostgreSQL 13+ and H2 2.x are supported.")); + DbStatsProvider provider = settings == null ? providerForBackend(runtime) : DbStatsProviders.forProviderName(settings.getValueString("provider")); + for(SetupCheck check : provider.setupChecks(access, settings == null ? null : settings.getValueString("statsAccessMessage"))) + { + rows.add(check(check.check(), check.state(), check.nextStep())); + } rows.add(check("Last snapshot", last == null ? "Nothing collected yet" : Duration.between(last, Instant.now()).toSeconds() + " seconds ago", last == null ? "Run Collect now from Snapshots, or wait for the next scheduled collection." : "Collection normally runs every " + SnapshotCollector.positive(settings, "snapshotIntervalMinutes", 5) + " minutes.")); rows.add(check("Attribution", attribution ? "On" : "Off", attribution ? "Attribution starts after the next flush." : "Enable attribution in Settings and the host configuration.")); @@ -143,6 +149,24 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer + /******************************************************************************* + ** Before the first collection the vendor comes from the backend itself; + ** an unreachable backend falls back to the unsupported checks. + *******************************************************************************/ + private static DbStatsProvider providerForBackend(DbPerfRuntime runtime) + { + try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName()))) + { + return (DbStatsProviders.forConnection(connection)); + } + catch(Exception e) + { + return (DbStatsProviders.forProviderName(null)); + } + } + + + /******************************************************************************* ** Plain text check row, never markup. *******************************************************************************/ @@ -257,13 +281,29 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer List snapshots = data.snapshots(); List values = new ArrayList<>(); values.add(stat("Total connections", latest == null ? null : number(latest, "connectionsTotal"))); - values.add(stat("Maximum connections", latest == null ? null : number(latest, "maxConnections"))); + //////////////////////////////////////////////////////////////////////// + // figures the provider does not keep (H2: limit, cache, temp, // + // deadlocks) are stored as null and left out rather than shown as 0 // + //////////////////////////////////////////////////////////////////////// + boolean limit = latest == null || latest.getValue("maxConnections") != null; + boolean cache = latest == null || latest.getValue("cacheHitRatio") != null; + boolean counters = snapshots.isEmpty() || snapshots.stream().anyMatch(row -> row.getValue("deadlocks") != null || row.getValue("tempBytes") != null); + if(limit) + { + values.add(stat("Maximum connections", latest == null ? null : number(latest, "maxConnections"))); + } values.add(stat("Active", latest == null ? null : number(latest, "connectionsActive"))); values.add(stat("Idle", latest == null ? null : number(latest, "connectionsIdle"))); values.add(stat("Idle in transaction", latest == null ? null : number(latest, "connectionsIdleInTx"))); - values.add(stat("Cache hit %", latest == null ? null : 100 * number(latest, "cacheHitRatio"))); - values.add(stat("Temp MB (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "tempBytes")).sum() / 1048576)); - values.add(stat("Deadlocks (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "deadlocks")).sum())); + if(cache) + { + values.add(stat("Cache hit %", latest == null ? null : 100 * number(latest, "cacheHitRatio"))); + } + if(counters) + { + values.add(stat("Temp MB (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "tempBytes")).sum() / 1048576)); + values.add(stat("Deadlocks (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "deadlocks")).sum())); + } return (new MultiStatisticsData(null, List.of(new MultiStatisticsData.StatisticsGroupData().withHeader("Connection capacity") .withSubheader(latest == null ? "Capacity appears after the next snapshot." : "Latest snapshot; percentages and MB rounded to whole numbers.").withStatisticList(values)))); } @@ -355,16 +395,29 @@ public class DbPerfWidgetRenderer extends AbstractWidgetRenderer *******************************************************************************/ public MultiTableData tables(DbPerformanceData data) throws QException { - List scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100)); + QRecord settings = data.settings(); + DbStatsProvider provider = settings == null || settings.getValueString("provider") == null ? null : DbStatsProviders.forProviderName(settings.getValueString("provider")); List growing = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("growthBytes7d", false)).withLimit(100)); + TableData growth = table("Growth appears after table history has accumulated.", growing.stream() + .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")), + "Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(), + "Table", "Size MB", "Growth MB (7 days)"); + if(provider != null && !provider.features().contains(StatsFeature.TABLE_ACCESS)) + { + ////////////////////////////////////////////////////////////////// + // vendors without scan counters (H2) list rows and size instead // + ////////////////////////////////////////////////////////////////// + List largest = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("nLiveTup", false)).withLimit(100)); + return (new MultiTableData(List.of(table("Tables appear after the next table snapshot (normally hourly).", largest.stream() + .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Rows (estimate)", row.getValue("nLiveTup"), + "Size MB", mb(row.getValue("totalBytes")))).toList(), + "Table", "Rows (estimate)", "Size MB"), growth))); + } + List scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100)); return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream() .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Sequential scans", row.getValue("seqScan"), "Index scans", row.getValue("idxScan"), "Sequential share %", row.getValue("seqScanShare"))).toList(), - "Table", "Sequential scans", "Index scans", "Sequential share %"), - table("Growth appears after table history has accumulated.", growing.stream() - .map(row -> map("Table", row.getValueString("schemaName") + "." + row.getValueString("tableName"), "Size MB", mb(row.getValue("totalBytes")), - "Growth MB (7 days)", mb(row.getValue("growthBytes7d")))).toList(), - "Table", "Size MB", "Growth MB (7 days)")))); + "Table", "Sequential scans", "Index scans", "Sequential share %"), growth))); } diff --git a/src/main/resources/help/qbit-db-performance.yaml b/src/main/resources/help/qbit-db-performance.yaml index 8d6d7ef..0e5e5c9 100644 --- a/src/main/resources/help/qbit-db-performance.yaml +++ b/src/main/resources/help/qbit-db-performance.yaml @@ -519,7 +519,7 @@ entries: - key: table:db_perf_settings;field:lastTableSnapshotAt summary: "Time of the latest table and index collection pass." - key: table:db_perf_settings;field:statsAccess - summary: "Statistics capability: FULL, reduced access, missing extension, or unsupported provider." + summary: "Statistics capability: FULL, reduced visibility, statement statistics off, or unsupported provider." - key: table:db_perf_settings;field:statsAccessMessage summary: "The collector explains missing capabilities and the next grant or setup step." - key: table:db_perf_snapshot;slot:help @@ -810,6 +810,10 @@ entries: summary: "Collect an immediate snapshot and force table and index statistics." content: | Run from the Snapshots table. Requires manage settings. The Result screen reports success, partial access or collection failure. The first pass establishes counter baselines. +- key: process:dbPerfEnableStatementStatistics;slot:help + summary: "Turn on statement statistics where the database allows it at runtime." + content: | + Run from Settings. Requires manage settings. On H2 it runs SET QUERY_STATISTICS TRUE, which needs an H2 admin user and lasts until the database closes; add ;QUERY_STATISTICS=TRUE to the JDBC URL to keep it on. PostgreSQL cannot do this at runtime (pg_stat_statements needs shared_preload_libraries and CREATE EXTENSION), so the result explains the manual step. The Result screen shows the new statistics access. - key: process:dbPerfRollupAndPurge;slot:help summary: "Roll up completed days and remove expired details." content: | @@ -817,7 +821,7 @@ entries: - key: process:dbPerfQueryPlan;slot:help summary: "Inspect the plan of one statement, with optional read-only execution." content: | - Choose PLAN (default) or ANALYZE. PLAN uses a generic plan for parameterized SQL on PostgreSQL 16+. ANALYZE requires explain analyze permission and exactly one SELECT with no modifying CTEs, INTO or row locks. Enter parameter values only for ANALYZE, one per line in placeholder order. Values are sent transiently; they and the ANALYZE plan are never stored. Every request is audited. Results include copyable plan text, cost, a SQL Workbench link and a literal-free assistant prompt. No model is called by this qbit. + Choose PLAN (default) or ANALYZE. PLAN uses a generic plan for parameterized SQL on PostgreSQL 16+ and H2. H2 plans are text (index choice and scan counts) without a cost tree; H2 refuses SQL naming side-effecting functions, DDL or aliases, and allows ANALYZE only for non-admin users. ANALYZE requires explain analyze permission and exactly one SELECT with no modifying CTEs, INTO or row locks. Enter parameter values only for ANALYZE, one per line in placeholder order. Values are sent transiently; they and the ANALYZE plan are never stored. Every request is audited. Results include copyable plan text, cost, a SQL Workbench link and a literal-free assistant prompt. No model is called by this qbit. - key: widget:dbPerfSetup;slot:label summary: "Check backend, statistics access, collection, attribution and scheduler setup." - key: widget:dbPerfDatabaseTime;slot:label diff --git a/src/main/resources/openapi/qqq-v1/db-performance.yaml b/src/main/resources/openapi/qqq-v1/db-performance.yaml index 8472a4e..6c47106 100644 --- a/src/main/resources/openapi/qqq-v1/db-performance.yaml +++ b/src/main/resources/openapi/qqq-v1/db-performance.yaml @@ -386,7 +386,7 @@ paths: security: - sessionUuidCookie: - "N/A" - summary: "Explain a PostgreSQL query" + summary: "Explain a query (PostgreSQL JSON or H2 text plan)" /qqq/v1/qbit/db-performance/statements/lookup: get: description: "Requires db_performance.view and uses the caller's session." diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHealthTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHealthTest.java index d3e9d03..dfe36a5 100644 --- a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHealthTest.java +++ b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHealthTest.java @@ -76,7 +76,7 @@ class DbPerfHealthTest DbPerfHealthChecks checks = checks(f, 0); for(String access : List.of("FULL", "NO_READ_ALL_STATS", "NO_PG_STAT_STATEMENTS", "UNSUPPORTED", "UNKNOWN")) { - DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("statsAccess", access))); + DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("provider", "POSTGRES").withValue("statsAccess", access))); var outcome = checks.check("stats_access"); assertThat(outcome.getDetails().getValues()).containsEntry("statsAccess", access).hasSize(1); assertThat(outcome.getStatus()).isEqualTo(switch(access) @@ -91,6 +91,14 @@ class DbPerfHealthTest assertThat(outcome.getMessage()).isEqualTo("grant pg_read_all_stats to sa"); } } + for(String access : List.of("NO_QUERY_STATISTICS", "NOT_ADMIN")) + { + DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", f.settings.getValueInteger("id")).withValue("provider", "H2").withValue("statsAccess", access))); + var outcome = checks.check("stats_access"); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED); + assertThat(outcome.getDetails().getValues()).containsEntry("statsAccess", access); + assertThat(outcome.getMessage()).doesNotContain("pg_").contains(access.equals("NOT_ADMIN") ? "ADMIN TRUE" : "QUERY_STATISTICS=TRUE"); + } for(int total : List.of(79, 80, 94, 95)) { DbPerfStore.update("db_perf_snapshot", List.of(new QRecord().withValue("id", f.snapshot.getValueInteger("id")).withValue("connectionsTotal", total))); diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java index 4a0106a..80ef0dc 100644 --- a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java +++ b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java @@ -33,6 +33,7 @@ import com.kingsrook.qbits.dbperformance.api.OverviewSpec; import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; import com.kingsrook.qbits.dbperformance.explain.ExplainRequest; +import com.kingsrook.qbits.dbperformance.explain.ExplainResult; import com.kingsrook.qbits.dbperformance.explain.ExplainService; import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess; import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer; @@ -103,7 +104,7 @@ class DbPerfPresentationTest assertThat(new DbPerfWidgetRenderer().liveActivity(data).getNoRowsFoundHTML()).isEqualTo("No long-running or blocked sessions right now"); assertThat(new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getNoRowsFoundHTML()).isEqualTo("Attribution starts after the next flush"); assertThat(new DbPerfWidgetRenderer().connectionStatistics(data).getType()).isEqualTo("multiStatistics"); - assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isEmpty(); + assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isNotEmpty().allSatisfy(row -> assertThat(row).containsKeys("Application", "User", "State", "Connections")); QContext.setQSession(new QSession()); for(String name : DbPerfWidgets.HOME) { @@ -235,7 +236,9 @@ class DbPerfPresentationTest void explainAuditAndHandler() throws Exception { ExplainService service = new ExplainService(runtime); - assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, "select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("PostgreSQL"); + ExplainResult h2 = service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, "select current_date", ExplainRequest.Mode.PLAN, List.of())); + assertThat(h2.planJson()).isNull(); + assertThat(h2.planText()).containsIgnoringCase("CURRENT_DATE"); assertThatThrownBy(() -> service.explain(null)).hasMessageContaining("Choose PLAN"); assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, null, ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("either"); QContext.setQSession(new QSession()); diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/H2CollectionTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/H2CollectionTest.java new file mode 100644 index 0000000..1d278dd --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/H2CollectionTest.java @@ -0,0 +1,173 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import com.kingsrook.qbits.dbperformance.analyze.AnalyzeService; +import com.kingsrook.qbits.dbperformance.api.DbPerformanceData; +import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; +import com.kingsrook.qbits.dbperformance.explain.ExplainRequest; +import com.kingsrook.qbits.dbperformance.explain.ExplainResult; +import com.kingsrook.qbits.dbperformance.explain.ExplainService; +import com.kingsrook.qbits.dbperformance.processes.DbPerfEnableStatisticsStep; +import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** The qbit end to end on an H2 backend: collection, the enable action, + ** feature-gated analysis, setup rows and text plans. + *******************************************************************************/ +class H2CollectionTest +{ + /******************************************************************************* + ** Two collections store H2 statements, tables, indexes and connections. + *******************************************************************************/ + @Test + void collectAnalyzeAndExplain() throws Exception + { + DbPerfRuntime runtime = runtime(";QUERY_STATISTICS=TRUE;QUERY_STATISTICS_MAX_ENTRIES=1000"); + String backend = runtime.getConfig().getBackendName(); + workload(runtime); + QRecord first = new SnapshotCollector(runtime).collect(backend, true); + assertThat(first.getValueString("status")).as(first.getValueString("errorMessage")).isEqualTo("OK"); + workload(runtime); + QRecord second = new SnapshotCollector(runtime).collect(backend, true); + assertThat(second.getValueString("status")).as(second.getValueString("errorMessage")).isEqualTo("OK"); + assertThat(second.getValueInteger("intervalSeconds")).isNotNull(); + assertThat(second.getValue("cacheHitRatio")).isNull(); + assertThat(second.getValue("maxConnections")).isNull(); + assertThat(second.getValueInteger("connectionsTotal")).isPositive(); + + QRecord settings = new DbPerfSettingsService(runtime).ensure(backend); + assertThat(settings.getValueString("provider")).isEqualTo("H2"); + assertThat(settings.getValueString("statsAccess")).isEqualTo("FULL"); + + List statements = DbPerfStore.query("db_perf_statement", DbPerfStore.eq("backendName", backend)); + assertThat(statements).anySatisfy(row -> + { + assertThat(row.getValueString("queryText")).containsIgnoringCase("h2_probe"); + assertThat(row.getValueLong("lastCalls")).isGreaterThanOrEqualTo(6); + }); + assertThat(DbPerfStore.query("db_perf_query_snapshot", new QQueryFilter())).anySatisfy(row -> assertThat(row.getValueLong("calls")).isEqualTo(3)); + assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))) + .anySatisfy(row -> assertThat(row.getValueString("tableName")).isEqualToIgnoringCase("h2_probe")); + assertThat(DbPerfStore.query("db_perf_index_stat", DbPerfStore.eq("backendName", backend))) + .anySatisfy(row -> assertThat(row.getValueString("indexName")).isEqualToIgnoringCase("h2_probe_name")); + + Map results = new AnalyzeService(runtime).run(backend).stream().collect(Collectors.toMap(AnalyzeService.Result::ruleCode, r -> String.valueOf(r.error()))); + assertThat(results).containsKeys("SLOW_QUERY", "DUPLICATE_INDEX", "IDLE_IN_TRANSACTION", "N_PLUS_ONE") + .doesNotContainKeys("UNUSED_INDEX", "MISSING_INDEX", "CACHE_HIT_LOW", "BLOAT", "VACUUM_STALE", "DEADLOCKS", "TEMP_SPILL", "POOL_SIZING"); + assertThat(results.values()).containsOnly("null"); + + assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isNotEmpty(); + assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))).allSatisfy(row -> assertThat(row.getValue("seqScanShare")).isNull()); + assertThat(new DbPerfWidgetRenderer().tables(new DbPerformanceData(runtime)).getTableDataList().getFirst().getColumns()).extracting(c -> c.getHeader()) + .containsExactly("Table", "Rows (estimate)", "Size MB"); + assertThat(new DbPerfWidgetRenderer().connectionStatistics(new DbPerformanceData(runtime)).getStatisticsGroupData().getFirst().getStatisticList()) + .extracting(statistic -> statistic.getLabel()).contains("Total connections", "Idle in transaction").doesNotContain("Maximum connections", "Cache hit %", "Deadlocks (24h)"); + assertThat(new DbPerfWidgetRenderer().setup(runtime, new DbPerformanceData(runtime)).getRows()).extracting(row -> row.get("Check")) + .contains("H2 QUERY_STATISTICS", "H2 admin user").doesNotContain("pg_read_all_stats"); + + ExplainService explain = new ExplainService(runtime); + ExplainResult plan = explain.explain(new ExplainRequest(backend, null, "select * from h2_probe where name = ?", ExplainRequest.Mode.PLAN, List.of())); + assertThat(plan.mode()).isEqualTo("GENERIC"); + assertThat(plan.planJson()).isNull(); + assertThat(plan.planText()).containsIgnoringCase("h2_probe_name"); + assertThat(plan.assistantPrompt()).contains("database query plan"); + assertThatThrownBy(() -> explain.explain(new ExplainRequest(backend, null, "select * from h2_probe where name = $1", ExplainRequest.Mode.ANALYZE, List.of("secret-value")))) + .hasMessageContaining("ANALYZE is not available"); + assertThatThrownBy(() -> explain.explain(new ExplainRequest(backend, null, "select file_write('x', '/tmp/p') from h2_probe", ExplainRequest.Mode.PLAN, List.of()))) + .hasMessageContaining("FILE_WRITE"); + } + + + + /******************************************************************************* + ** Without QUERY_STATISTICS the snapshot is partial until the action runs. + *******************************************************************************/ + @Test + void enableActionTurnsOnStatistics() throws Exception + { + DbPerfRuntime runtime = runtime(""); + String backend = runtime.getConfig().getBackendName(); + QRecord partial = new SnapshotCollector(runtime).collect(backend, true); + assertThat(partial.getValueString("status")).isEqualTo("PARTIAL"); + assertThat(partial.getValueString("errorMessage")).contains("QUERY_STATISTICS"); + assertThat(DbPerfStore.query("db_perf_table_stat", DbPerfStore.eq("backendName", backend))).isNotEmpty(); + + RunBackendStepOutput output = new RunBackendStepOutput(); + new DbPerfEnableStatisticsStep().run(new RunBackendStepInput(), output); + assertThat(String.valueOf(output.getValue("processResults"))).contains("FULL"); + assertThat(new DbPerfSettingsService(runtime).ensure(backend).getValueString("statsAccess")).isEqualTo("FULL"); + workload(runtime); + assertThat(new SnapshotCollector(runtime).collect(backend, false).getValueString("status")).isEqualTo("OK"); + } + + + + /******************************************************************************* + ** QQQ runtime on a fresh H2 backend with the given URL options. + *******************************************************************************/ + private static DbPerfRuntime runtime(String options) throws Exception + { + String name = "dbperfh2" + UUID.randomUUID().toString().replace("-", ""); + RDBMSBackendMetaData backend = new RDBMSBackendMetaData().withName(name).withVendor("h2").withHostName("mem").withDatabaseName(name).withUsername("sa"); + backend.setJdbcUrl("jdbc:h2:mem:" + name + ";CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1;MODE=PostgreSQL" + options); + return (DbPerfRuntime.of(DbPerfTestSupport.instance(backend))); + } + + + + /******************************************************************************* + ** A table with a duplicate-prone index and three identical lookups. + *******************************************************************************/ + private static void workload(DbPerfRuntime runtime) throws Exception + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName()); + try(Connection connection = DriverManager.getConnection(backend.getJdbcUrl(), "sa", ""); Statement statement = connection.createStatement()) + { + statement.execute("CREATE TABLE IF NOT EXISTS h2_probe (id INT PRIMARY KEY, name VARCHAR(40))"); + statement.execute("CREATE INDEX IF NOT EXISTS h2_probe_name ON h2_probe (name)"); + statement.execute("MERGE INTO h2_probe KEY (id) SELECT x, 'n' || x FROM SYSTEM_RANGE(1, 20)"); + for(int i = 0; i < 3; i++) + { + statement.executeQuery("SELECT * FROM h2_probe WHERE name = 'n7'").close(); + } + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/H2StatsProviderTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/H2StatsProviderTest.java new file mode 100644 index 0000000..df7a347 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/H2StatsProviderTest.java @@ -0,0 +1,366 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import com.kingsrook.qbits.dbperformance.provider.ConnectionCount; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; +import com.kingsrook.qbits.dbperformance.provider.H2StatsProvider; +import com.kingsrook.qbits.dbperformance.provider.IndexStats; +import com.kingsrook.qbits.dbperformance.provider.PostgresStatsProvider; +import com.kingsrook.qbits.dbperformance.provider.ProviderCapabilities; +import com.kingsrook.qbits.dbperformance.provider.SessionInfo; +import com.kingsrook.qbits.dbperformance.provider.StatementStats; +import com.kingsrook.qbits.dbperformance.provider.StatsFeature; +import com.kingsrook.qbits.dbperformance.provider.TableStats; +import com.kingsrook.qbits.dbperformance.provider.UnsupportedStatsProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** H2StatsProvider against real in-memory H2 databases: every read returns + ** rows produced by activity in the test, never fixtures. + *******************************************************************************/ +class H2StatsProviderTest +{ + private final H2StatsProvider provider = new H2StatsProvider(); + + + + /******************************************************************************* + ** The registry maps JDBC product names and saved names to providers. + *******************************************************************************/ + @Test + void registryChoosesByProductName() throws Exception + { + assertThat(DbStatsProviders.forProductName("H2")).isInstanceOf(H2StatsProvider.class); + assertThat(DbStatsProviders.forProductName("PostgreSQL")).isInstanceOf(PostgresStatsProvider.class); + assertThat(DbStatsProviders.forProductName("MySQL")).isInstanceOf(UnsupportedStatsProvider.class); + assertThat(DbStatsProviders.forProductName(null)).isInstanceOf(UnsupportedStatsProvider.class); + assertThat(DbStatsProviders.forProviderName("H2")).isInstanceOf(H2StatsProvider.class); + assertThat(DbStatsProviders.forProviderName("POSTGRES")).isInstanceOf(PostgresStatsProvider.class); + assertThat(DbStatsProviders.forProviderName(null)).isInstanceOf(UnsupportedStatsProvider.class); + try(Connection connection = open(false, "")) + { + assertThat(DbStatsProviders.forConnection(connection)).isInstanceOf(H2StatsProvider.class); + } + assertThat(DbStatsProviders.forProductName("ExampleDB")).isInstanceOf(UnsupportedStatsProvider.class); + DbStatsProviders.register("ExampleDB", H2StatsProvider::new); + assertThat(DbStatsProviders.forProductName("exampledb")).isInstanceOf(H2StatsProvider.class); + assertThat(new UnsupportedStatsProvider().capabilities(null).features()).isEmpty(); + } + + + + /******************************************************************************* + ** Every read returns real rows after activity, in default and MySQL mode. + *******************************************************************************/ + @ParameterizedTest + @ValueSource(strings = {"", ";MODE=MySQL", ";MODE=PostgreSQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE"}) + void readsReturnRealRowsAfterActivity(String mode) throws Exception + { + String url = url(true, mode); + try(Connection connection = DriverManager.getConnection(url, "sa", "")) + { + activity(connection); + ProviderCapabilities capabilities = provider.capabilities(connection); + assertThat(capabilities.supported()).isTrue(); + assertThat(capabilities.statsAccess()).isEqualTo("FULL"); + assertThat(capabilities.statements()).isTrue(); + assertThat(capabilities.readAllStats()).isTrue(); + assertThat(capabilities.version()).isGreaterThanOrEqualTo(2000000); + assertThat(capabilities.features()).contains(StatsFeature.STATEMENTS, StatsFeature.SESSIONS, StatsFeature.BLOCKING, StatsFeature.TABLE_ROWS, StatsFeature.EXPLAIN_TEXT) + .doesNotContain(StatsFeature.CACHE_COUNTERS, StatsFeature.DATABASE_COUNTERS, StatsFeature.INDEX_USAGE, StatsFeature.TABLE_MAINTENANCE, StatsFeature.MAX_CONNECTIONS, StatsFeature.EXPLAIN_JSON); + assertThat(provider.features()).containsAll(capabilities.features()); + + assertThat(provider.readDatabase(connection).counters()).isEmpty(); + assertThat(provider.readDatabase(connection).maxConnections()).isZero(); + assertThat(provider.readStatementResetInfo(connection).statsReset()).isNull(); + + provider.readTables(connection); + List statements = provider.readStatements(connection); + StatementStats lookup = statements.stream().filter(s -> s.queryText().contains("WHERE customer_id = 3")).findFirst().orElseThrow(); + assertThat(lookup.calls()).isEqualTo(5); + assertThat(lookup.rows()).isEqualTo(5 * 10); + assertThat(lookup.totalExecMs()).isGreaterThanOrEqualTo(lookup.maxExecMs()); + assertThat(lookup.dbUser()).isNull(); + assertThat(statements).noneMatch(s -> s.queryText().toUpperCase().contains("INFORMATION_SCHEMA") || s.queryText().contains("DB_PERF_PROBE") + || s.queryText().toUpperCase().startsWith("SET ") || s.queryText().contains("DISK_SPACE_USED") || s.queryText().equalsIgnoreCase("CALL DATABASE()")); + assertThat(provider.readStatements(connection)).anyMatch(s -> s.queryId() == lookup.queryId() && s.queryText().equals(lookup.queryText())); + + TableStats orders = provider.readTables(connection).stream().filter(t -> t.tableName().equalsIgnoreCase("orders")).findFirst().orElseThrow(); + assertThat(orders.counters()).containsEntry("nLiveTup", 50L).containsKeys("tableBytes", "totalBytes").doesNotContainKeys("seqScan", "nDeadTup"); + assertThat(orders.relid()).isPositive(); + + List indexes = provider.readIndexes(connection).stream().filter(i -> i.tableName().equalsIgnoreCase("orders")).toList(); + assertThat(indexes).anySatisfy(i -> assertThat(i.primary() && i.unique()).isTrue()); + assertThat(indexes).anySatisfy(i -> + { + assertThat(i.indexName()).isEqualToIgnoringCase("orders_status"); + assertThat(i.unique()).isFalse(); + assertThat(i.columns()).isEqualToIgnoringCase("status, created DESC"); + assertThat(i.definition()).contains("CREATE INDEX").doesNotContain("USING CONSTRAINT"); + }); + assertThat(indexes).anySatisfy(i -> assertThat(i.definition()).contains("USING CONSTRAINT")); + + try(Connection holder = DriverManager.getConnection(url, "sa", ""); Connection waiter = DriverManager.getConnection(url + ";LOCK_TIMEOUT=10000", "sa", "")) + { + holder.setAutoCommit(false); + try(Statement statement = holder.createStatement()) + { + statement.executeUpdate("UPDATE orders SET status = 'HELD' WHERE id = 1"); + } + CompletableFuture blocked = CompletableFuture.runAsync(() -> update(waiter)); + List sessions = waitForBlocked(connection); + SessionInfo idle = sessions.stream().filter(s -> s.state().equals("idle in transaction")).findFirst().orElseThrow(); + assertThat(idle.stateChange()).isNotNull(); + assertThat(idle.xactSeconds()).isNotNull(); + SessionInfo waiting = sessions.stream().filter(s -> !s.blockedBy().isEmpty()).findFirst().orElseThrow(); + assertThat(waiting.state()).isEqualTo("active"); + assertThat(waiting.waitEventType()).isEqualTo("Lock"); + assertThat(waiting.blockedBy()).containsExactly(idle.pid()); + assertThat(waiting.queryText()).contains("UPDATE orders"); + assertThat(waiting.queryId()).isNotNull(); + assertThat(waiting.querySeconds()).isNotNull(); + assertThat(provider.readConnections(connection).stream().mapToInt(ConnectionCount::connectionCount).sum()).isGreaterThanOrEqualTo(3); + assertThat(provider.readConnections(connection)).anyMatch(c -> "idle in transaction".equals(c.state())); + holder.rollback(); + blocked.get(); + } + + Map settings = provider.readSettings(connection); + assertThat(settings).containsEntry("QUERY_STATISTICS", "TRUE").containsKey("info.VERSION"); + } + } + + + + /******************************************************************************* + ** Without QUERY_STATISTICS statements are unavailable until enabled. + *******************************************************************************/ + @Test + void capabilitiesWithoutQueryStatisticsAndEnable() throws Exception + { + try(Connection connection = open(false, "")) + { + activity(connection); + ProviderCapabilities off = provider.capabilities(connection); + assertThat(off.supported()).isTrue(); + assertThat(off.statements()).isFalse(); + assertThat(off.statsAccess()).isEqualTo("NO_QUERY_STATISTICS"); + assertThat(off.message()).contains("QUERY_STATISTICS=TRUE"); + assertThat(off.has(StatsFeature.STATEMENTS)).isFalse(); + assertThat(off.has(StatsFeature.SESSIONS)).isTrue(); + assertThat(provider.readStatements(connection)).isEmpty(); + assertThat(provider.readTables(connection)).isNotEmpty(); + assertThat(provider.readSettings(connection)).containsEntry("QUERY_STATISTICS", "FALSE"); + + assertThat(provider.enableStatementStatistics(connection)).contains("JDBC URL"); + activity(connection); + ProviderCapabilities on = provider.capabilities(connection); + assertThat(on.statsAccess()).isEqualTo("FULL"); + assertThat(on.has(StatsFeature.STATEMENTS)).isTrue(); + assertThat(provider.readStatements(connection)).anyMatch(s -> s.queryText().contains("WHERE customer_id = 3") && s.calls() == 5); + assertThat(provider.enableStatementStatistics(connection)).contains("already on"); + } + } + + + + /******************************************************************************* + ** A non-admin user sees only its own session and cannot enable statistics. + *******************************************************************************/ + @Test + void nonAdminUser() throws Exception + { + String url = url(true, ""); + try(Connection admin = DriverManager.getConnection(url, "sa", "")) + { + try(Statement statement = admin.createStatement()) + { + statement.execute("CREATE USER app PASSWORD 'secret'"); + statement.execute("CREATE TABLE probe (id INT PRIMARY KEY)"); + statement.execute("GRANT SELECT ON probe TO app"); + } + try(Connection app = DriverManager.getConnection(url.replace(";DB_CLOSE_DELAY=-1", "").replace(";QUERY_STATISTICS=TRUE", ""), "app", "secret")) + { + ProviderCapabilities capabilities = provider.capabilities(app); + assertThat(capabilities.statsAccess()).isEqualTo("NOT_ADMIN"); + assertThat(capabilities.readAllStats()).isFalse(); + assertThat(capabilities.statements()).isTrue(); + assertThat(capabilities.has(StatsFeature.BLOCKING)).isFalse(); + assertThat(provider.readSessions(app)).isEmpty(); + assertThat(provider.remedy("NOT_ADMIN", "APP")).contains("ALTER USER APP ADMIN TRUE"); + try(Statement statement = admin.createStatement()) + { + statement.execute("SET QUERY_STATISTICS FALSE"); + } + assertThatThrownBy(() -> provider.enableStatementStatistics(app)).isInstanceOf(SQLException.class); + } + } + } + + + + /******************************************************************************* + ** EXPLAIN plans without executing; ANALYZE only for a non-admin SELECT; + ** side-effecting functions, aliases, DDL and multiple statements refused. + *******************************************************************************/ + @Test + void explainText() throws Exception + { + String url = url(true, ";MODE=MySQL"); + try(Connection connection = DriverManager.getConnection(url, "sa", "")) + { + activity(connection); + assertThat(provider.capabilities(connection).has(StatsFeature.EXPLAIN_ANALYZE)).isFalse(); + assertThat(provider.capabilities(connection).has(StatsFeature.EXPLAIN_TEXT)).isTrue(); + assertThat(provider.explainText(connection, "SELECT * FROM orders WHERE status = 'NEW'", List.of(), false, 5000)).containsIgnoringCase("orders_status").doesNotContain("scanCount"); + assertThat(provider.explainText(connection, "SELECT * FROM orders WHERE customer_id = ?1", List.of(), false, 5000)).contains("?1"); + assertThat(connection.getAutoCommit()).isTrue(); + assertThat(connection.isReadOnly()).isFalse(); + assertThatThrownBy(() -> provider.explainText(connection, "SELECT * FROM orders WHERE customer_id = ?1", List.of("3"), true, 5000)).hasMessageContaining("admin"); + assertThatThrownBy(() -> provider.explainText(connection, "SELECT FILE_WRITE('x', '/tmp/dbperf-h2-explain-probe') FROM orders", List.of(), false, 5000)).hasMessageContaining("FILE_WRITE"); + assertThatThrownBy(() -> provider.explainText(connection, "SELECT \"file_write\"('x', '/tmp/p')", List.of(), false, 5000)).hasMessageContaining("FILE_WRITE"); + assertThatThrownBy(() -> provider.explainText(connection, "DROP TABLE orders", List.of(), false, 5000)).hasMessageContaining("DROP"); + assertThatThrownBy(() -> provider.explainText(connection, "SELECT 1; SELECT 2", List.of(), false, 5000)).hasMessageContaining("one statement"); + assertThat(provider.explainText(connection, "SELECT 'drop; file_write' AS label FROM orders", List.of(), false, 5000)).containsIgnoringCase("orders"); + try(Statement statement = connection.createStatement()) + { + statement.execute("CREATE ALIAS my_side_effect FOR \"java.lang.System.nanoTime\""); + statement.execute("CREATE USER reader PASSWORD 'secret'"); + statement.execute("GRANT SELECT ON orders TO reader"); + } + assertThatThrownBy(() -> provider.explainText(connection, "SELECT my_side_effect() FROM orders", List.of(), false, 5000)).hasMessageContaining("MY_SIDE_EFFECT"); + try(Connection reader = DriverManager.getConnection(url.replace(";DB_CLOSE_DELAY=-1", "").replace(";QUERY_STATISTICS=TRUE", ""), "reader", "secret")) + { + assertThat(provider.capabilities(reader).has(StatsFeature.EXPLAIN_ANALYZE)).isTrue(); + assertThat(provider.explainText(reader, "SELECT * FROM orders WHERE customer_id = ?1", List.of("3"), true, 5000)).contains("scanCount"); + assertThatThrownBy(() -> provider.explainText(reader, "VALUES (1)", List.of(), true, 5000)).hasMessageContaining("single SELECT"); + } + } + assertThat(java.nio.file.Files.exists(java.nio.file.Path.of("/tmp/dbperf-h2-explain-probe"))).isFalse(); + } + + + + /******************************************************************************* + ** H2 session states use PostgreSQL's names. + *******************************************************************************/ + @Test + void stateMapping() throws Exception + { + assertThat(provider.setupChecks("NO_QUERY_STATISTICS", "m")).extracting(c -> c.check()).containsExactly("H2 QUERY_STATISTICS", "H2 admin user"); + assertThat(provider.remedy("FULL", "sa")).isEqualTo("Full statistics access"); + assertThat(provider.remedy("UNSUPPORTED", "sa")).contains("H2 2.0"); + assertThat(new PostgresStatsProvider().remedy("NO_READ_ALL_STATS", "sa")).isEqualTo("grant pg_read_all_stats to sa"); + assertThat(new PostgresStatsProvider().setupChecks("FULL", null)).extracting(c -> c.check()).containsExactly("pg_stat_statements", "pg_read_all_stats"); + } + + + + /******************************************************************************* + ** Tables, a secondary index, a foreign key and repeated queries. + *******************************************************************************/ + private static void activity(Connection connection) throws SQLException + { + try(Statement statement = connection.createStatement()) + { + statement.execute("CREATE TABLE IF NOT EXISTS customer (id INT PRIMARY KEY, name VARCHAR(50))"); + statement.execute("CREATE TABLE IF NOT EXISTS orders (id INT PRIMARY KEY, customer_id INT REFERENCES customer(id), status VARCHAR(20), created TIMESTAMP)"); + statement.execute("CREATE INDEX IF NOT EXISTS orders_status ON orders (status, created DESC)"); + statement.execute("MERGE INTO customer KEY (id) SELECT x, 'c' || x FROM SYSTEM_RANGE(1, 5)"); + statement.execute("MERGE INTO orders KEY (id) SELECT x, MOD(x, 5) + 1, 'NEW', CURRENT_TIMESTAMP FROM SYSTEM_RANGE(1, 50)"); + for(int i = 0; i < 5; i++) + { + statement.executeQuery("SELECT * FROM orders WHERE customer_id = 3").close(); + } + } + } + + + + /******************************************************************************* + ** Poll until the waiter shows as blocked. + *******************************************************************************/ + private List waitForBlocked(Connection connection) throws Exception + { + for(int i = 0; i < 100; i++) + { + List sessions = provider.readSessions(connection); + if(sessions.stream().anyMatch(s -> !s.blockedBy().isEmpty())) + { + return (sessions); + } + Thread.sleep(50); + } + throw (new AssertionError("no blocked session observed")); + } + + + + /******************************************************************************* + ** Blocking update for the waiter thread. + *******************************************************************************/ + private static void update(Connection waiter) + { + try(Statement statement = waiter.createStatement()) + { + statement.executeUpdate("UPDATE orders SET status = 'WAIT' WHERE id = 1"); + } + catch(SQLException e) + { + throw (new IllegalStateException(e)); + } + } + + + + /******************************************************************************* + ** A fresh in-memory database. + *******************************************************************************/ + private static Connection open(boolean statistics, String mode) throws SQLException + { + return (DriverManager.getConnection(url(statistics, mode), "sa", "")); + } + + + + /******************************************************************************* + ** URL of a fresh in-memory database. + *******************************************************************************/ + private static String url(boolean statistics, String mode) + { + return ("jdbc:h2:mem:h2stats" + UUID.randomUUID().toString().replace("-", "") + ";DB_CLOSE_DELAY=-1" + (statistics ? ";QUERY_STATISTICS=TRUE" : "") + mode); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/SnapshotCollectorTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/SnapshotCollectorTest.java index 3aa247d..e1fee8a 100644 --- a/src/test/java/com/kingsrook/qbits/dbperformance/SnapshotCollectorTest.java +++ b/src/test/java/com/kingsrook/qbits/dbperformance/SnapshotCollectorTest.java @@ -28,6 +28,7 @@ import java.util.List; import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; +import com.kingsrook.qbits.dbperformance.provider.UnsupportedStatsProvider; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -161,9 +162,17 @@ class SnapshotCollectorTest void unsupportedVendor() throws Exception { DbPerfRuntime runtime = DbPerfRuntime.of(DbPerfTestSupport.instance()); - QRecord snapshot = new SnapshotCollector(runtime).collect(runtime.getConfig().getBackendName(), false); + QRecord snapshot = new SnapshotCollector(runtime, Clock.systemUTC(), new UnsupportedStatsProvider()).collect(runtime.getConfig().getBackendName(), false); assertThat(snapshot.getValueString("status")).isEqualTo("PARTIAL"); assertThat(snapshot.getValueString("errorMessage")).contains("not supported"); + + ////////////////////////////////////////////////////////////////////// + // the H2 test backend itself is supported, partially without // + // QUERY_STATISTICS // + ////////////////////////////////////////////////////////////////////// + QRecord h2 = new SnapshotCollector(runtime).collect(runtime.getConfig().getBackendName(), false); + assertThat(h2.getValueString("status")).isEqualTo("PARTIAL"); + assertThat(h2.getValueString("errorMessage")).contains("QUERY_STATISTICS"); } -- 2.54.0 From f005c400832f50e98be708537b2dc2eca3499fa1 Mon Sep 17 00:00:00 2001 From: Forge Recovery Date: Mon, 2 Nov 2026 04:10:32 +0000 Subject: [PATCH 10/18] ci: add the identical QRun release caller Co-Authored-By: Codex --- .gitea/workflows/qrun-release.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .gitea/workflows/qrun-release.yaml diff --git a/.gitea/workflows/qrun-release.yaml b/.gitea/workflows/qrun-release.yaml new file mode 100644 index 0000000..122f7e9 --- /dev/null +++ b/.gitea/workflows/qrun-release.yaml @@ -0,0 +1,12 @@ +# Identical in every QRun Maven application: do not edit. It only delegates to the protected qrun-release-v1 channel. +name: qrun-release +on: + push: + branches: [main, develop, 'release/**', 'hotfix/**'] +permissions: + contents: read +jobs: + release: + uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-maven.yaml@qrun-release-v1 + with: + runner: linux-small -- 2.54.0 From 612f474e91341e21d5bc0afcd81ccdb2787ca94b Mon Sep 17 00:00:00 2001 From: Forge Recovery Date: Mon, 2 Nov 2026 06:17:47 +0000 Subject: [PATCH 11/18] ci: move the QRun release caller to the thin v2 contract Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Codex --- .gitea/workflows/qrun-release.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/qrun-release.yaml b/.gitea/workflows/qrun-release.yaml index 122f7e9..77794cf 100644 --- a/.gitea/workflows/qrun-release.yaml +++ b/.gitea/workflows/qrun-release.yaml @@ -1,12 +1,16 @@ -# Identical in every QRun Maven application: do not edit. It only delegates to the protected qrun-release-v1 channel. +# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository. name: qrun-release on: - push: - branches: [main, develop, 'release/**', 'hotfix/**'] + push: {} + workflow_dispatch: {} permissions: contents: read + actions: read jobs: release: - uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-maven.yaml@qrun-release-v1 + uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1 with: runner: linux-small + docker-runner: container-build + browsers: true + browser-runner: browser-build -- 2.54.0 From 346845333506854e1b80bf2c24eff5fca13ff4ca Mon Sep 17 00:00:00 2001 From: Forge Recovery Date: Mon, 2 Nov 2026 15:48:51 +0000 Subject: [PATCH 12/18] ci: add the isolated postpublish job to the QRun release caller Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Codex --- .gitea/workflows/qrun-release.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitea/workflows/qrun-release.yaml b/.gitea/workflows/qrun-release.yaml index 77794cf..5ebc530 100644 --- a/.gitea/workflows/qrun-release.yaml +++ b/.gitea/workflows/qrun-release.yaml @@ -14,3 +14,12 @@ jobs: docker-runner: container-build browsers: true browser-runner: browser-build + postpublish: + if: ${{ needs.release.outputs.capture == 'true' }} + needs: release + permissions: + contents: write + actions: write + uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1 + with: + runner: linux-small -- 2.54.0 From 5d82358d2bb36f6c48ca0e5c48d6a6fb325968e7 Mon Sep 17 00:00:00 2001 From: Forge Recovery Date: Mon, 2 Nov 2026 20:16:11 +0000 Subject: [PATCH 13/18] ci: enroll shared Gitea release-candidate lifecycle Co-Authored-By: Codex --- .gitea/workflows/release-rc.yaml | 72 ++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .gitea/workflows/release-rc.yaml diff --git a/.gitea/workflows/release-rc.yaml b/.gitea/workflows/release-rc.yaml new file mode 100644 index 0000000..12ec9fc --- /dev/null +++ b/.gitea/workflows/release-rc.yaml @@ -0,0 +1,72 @@ +name: Release candidate +'on': + workflow_dispatch: + inputs: + dry_run: + description: Plan a cut without changing release state + type: boolean + default: false + finalize: + description: Retry finalization of an existing cut; do not cut another RC + type: boolean + default: false + recover: + description: Recover the same published RC after a deployment timeout; never + reupload + type: boolean + default: false + circle_build: + description: Original release run id (recovery only) + type: string + default: '' + merge_sha: + description: Existing cut merge_sha (finalize retry only) + type: string + default: '' + bump_sha: + description: Existing cut bump_sha (finalize retry only) + type: string + default: '' + branch: + description: Existing cut branch (finalize retry only) + type: string + default: '' + version: + description: Existing cut version (finalize retry only) + type: string + default: '' + pull_request: + types: + - closed + branches: + - develop +permissions: + contents: read +jobs: + release-rc: + # The old qbit-rc-published repository_dispatch is replaced by the already-proven workflow_dispatch + # completion: the publisher's own ActionsWRITE job dispatches this workflow with finalize=true plus + # merge_sha/bump_sha/branch/version (same inputs as the manual finalize retry). + permissions: + contents: write + actions: read + pull-requests: write + if: github.event_name == 'workflow_dispatch' + || (github.event_name == 'pull_request' && github.event.pull_request.merged + == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name, + 'release:rc')) + uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qbit-release-rc.yaml@qrun-release-v1 + with: + dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false + }} + finalize: ${{ github.event_name == 'workflow_dispatch' && inputs.finalize }} + recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false + }} + circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build + || '' }} + merge_sha: ${{ inputs.merge_sha || '' }} + bump_sha: ${{ inputs.bump_sha || '' }} + branch: ${{ inputs.branch || '' }} + version: ${{ inputs.version || '' }} + secrets: + QRUN_TAG_SIGNING_KEY: ${{ secrets.QRUN_TAG_SIGNING_KEY }} -- 2.54.0 From 171b0d3fad17b91b04151ffae7b555ec896078bb Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 5 Nov 2026 22:04:26 +0000 Subject: [PATCH 14/18] ci: use shared ci-workflows Maven workflow Replace the platform-ci release callers with one caller of KofTwentyTwo/ci-workflows maven.yaml (qqq-orb port). Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build.yaml | 7 +++ .gitea/workflows/qrun-release.yaml | 25 ----------- .gitea/workflows/release-rc.yaml | 72 ------------------------------ 3 files changed, 7 insertions(+), 97 deletions(-) create mode 100644 .gitea/workflows/build.yaml delete mode 100644 .gitea/workflows/qrun-release.yaml delete mode 100644 .gitea/workflows/release-rc.yaml diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..ed42608 --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,7 @@ +name: build +on: + push: +jobs: + maven: + uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@main + secrets: inherit diff --git a/.gitea/workflows/qrun-release.yaml b/.gitea/workflows/qrun-release.yaml deleted file mode 100644 index 5ebc530..0000000 --- a/.gitea/workflows/qrun-release.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository. -name: qrun-release -on: - push: {} - workflow_dispatch: {} -permissions: - contents: read - actions: read -jobs: - release: - uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1 - with: - runner: linux-small - docker-runner: container-build - browsers: true - browser-runner: browser-build - postpublish: - if: ${{ needs.release.outputs.capture == 'true' }} - needs: release - permissions: - contents: write - actions: write - uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1 - with: - runner: linux-small diff --git a/.gitea/workflows/release-rc.yaml b/.gitea/workflows/release-rc.yaml deleted file mode 100644 index 12ec9fc..0000000 --- a/.gitea/workflows/release-rc.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: Release candidate -'on': - workflow_dispatch: - inputs: - dry_run: - description: Plan a cut without changing release state - type: boolean - default: false - finalize: - description: Retry finalization of an existing cut; do not cut another RC - type: boolean - default: false - recover: - description: Recover the same published RC after a deployment timeout; never - reupload - type: boolean - default: false - circle_build: - description: Original release run id (recovery only) - type: string - default: '' - merge_sha: - description: Existing cut merge_sha (finalize retry only) - type: string - default: '' - bump_sha: - description: Existing cut bump_sha (finalize retry only) - type: string - default: '' - branch: - description: Existing cut branch (finalize retry only) - type: string - default: '' - version: - description: Existing cut version (finalize retry only) - type: string - default: '' - pull_request: - types: - - closed - branches: - - develop -permissions: - contents: read -jobs: - release-rc: - # The old qbit-rc-published repository_dispatch is replaced by the already-proven workflow_dispatch - # completion: the publisher's own ActionsWRITE job dispatches this workflow with finalize=true plus - # merge_sha/bump_sha/branch/version (same inputs as the manual finalize retry). - permissions: - contents: write - actions: read - pull-requests: write - if: github.event_name == 'workflow_dispatch' - || (github.event_name == 'pull_request' && github.event.pull_request.merged - == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name, - 'release:rc')) - uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qbit-release-rc.yaml@qrun-release-v1 - with: - dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false - }} - finalize: ${{ github.event_name == 'workflow_dispatch' && inputs.finalize }} - recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false - }} - circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build - || '' }} - merge_sha: ${{ inputs.merge_sha || '' }} - bump_sha: ${{ inputs.bump_sha || '' }} - branch: ${{ inputs.branch || '' }} - version: ${{ inputs.version || '' }} - secrets: - QRUN_TAG_SIGNING_KEY: ${{ secrets.QRUN_TAG_SIGNING_KEY }} -- 2.54.0 From aff84d5addaa17d0cf7421fc50acbb348c015e2d Mon Sep 17 00:00:00 2001 From: CI Date: Thu, 5 Nov 2026 16:24:46 -0600 Subject: [PATCH 15/18] ci: rerun shared workflow -- 2.54.0 From cf106d43019e05632be3d93ed3d2b906333363f0 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 14:08:36 +0000 Subject: [PATCH 16/18] ci: pin shared workflow to v1 Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index ed42608..ce00351 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -3,5 +3,5 @@ on: push: jobs: maven: - uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@main + uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1 secrets: inherit -- 2.54.0 From 725446709bdda22689efe78ed8a5ce6a11684efd Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 7 Nov 2026 17:59:16 +0000 Subject: [PATCH 17/18] build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26) pom.xml: parent -> 2.2.1 pom.xml: qqq.snapshot.version: 4.1.0-RC.13 -> 4.1.0-RC.26 pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import) Co-Authored-By: Claude Opus 5.5 --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ffed9fa..435c5c5 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ + 4.1.0-RC.26 0.1.0-SNAPSHOT 21 21 @@ -28,7 +29,7 @@ com.kingsrook qbit-build-parent - 2.0.0 + 2.2.1 com.kingsrook.qbits @@ -248,7 +249,7 @@ true - 4.1.0-RC.13 + 4.1.0-RC.26 -- 2.54.0 From f0ecacc3e938f4377d4fbb5874ea59628e437c98 Mon Sep 17 00:00:00 2001 From: James Maes <6+james.maes@noreply.git.kof22.com> Date: Sat, 7 Nov 2026 12:30:25 -0600 Subject: [PATCH 18/18] test: regenerate OpenAPI contract for QQQ 4.1.0-RC.26 QQQ RC.26 adds the shared BasicErrorResponseV1 schema to extension models; regenerate the committed fragment and allow that core schema. Co-Authored-By: Claude Opus 5.5 --- .../openapi/qqq-v1/db-performance.yaml | 186 ++++++++++++++++++ .../dbperformance/DbPerfApiContractTest.java | 2 +- 2 files changed, 187 insertions(+), 1 deletion(-) diff --git a/src/main/resources/openapi/qqq-v1/db-performance.yaml b/src/main/resources/openapi/qqq-v1/db-performance.yaml index 6c47106..e5570c1 100644 --- a/src/main/resources/openapi/qqq-v1/db-performance.yaml +++ b/src/main/resources/openapi/qqq-v1/db-performance.yaml @@ -1,5 +1,44 @@ components: schemas: + BasicErrorResponseV1: + description: "The error body of every UI API operation (core and extensions)." + properties: + error: + description: "The error message (the same as message; kept for clients of\ + \ UI API 1.0 to 1.5)." + type: "string" + code: + description: "A stable, machine-readable code. Well-known values: BAD_REQUEST\ + \ (400), UNAUTHENTICATED (401),\nPERMISSION_DENIED (403), NOT_FOUND (404),\ + \ CONFLICT (409), RATE_LIMITED (429), INTERNAL_ERROR (500),\nERROR (other).\ + \ Extensions may add codes; treat an unknown code by the HTTP status.\ + \ Since 1.6.0." + type: "string" + message: + description: "A message to show the user. Since 1.6.0." + type: "string" + fieldErrors: + description: "Errors about particular request fields, when known. Since\ + \ 1.6.0." + items: + properties: + field: + description: "The field (or parameter) name." + type: "string" + message: + description: "The error about that field." + type: "string" + code: + description: "An optional machine-readable code, e.g. REQUIRED, TOO_LONG,\ + \ PATTERN." + type: "string" + type: "object" + type: "array" + traceId: + description: "Identifies this request in the server logs (also the X-QQQ-Trace-Id\ + \ response header); quote it when reporting a problem. Since 1.6.0." + type: "string" + type: "object" DbPerformanceActivityV1: properties: blockingChains: @@ -256,9 +295,42 @@ components: type: "object" securitySchemes: sessionUuidCookie: + description: "The session cookie set by POST /qqq/v1/manageSession (web clients)." in: "cookie" name: "sessionUUID" type: "apiKey" + bearerAuth: + bearerFormat: "JWT or session UUID" + description: "`Authorization: Bearer `, where the token is either\n-\ + \ an OAuth2 access token (a signed JWT) issued by the app's identity provider\ + \ to the app's client,\n or to its native client (`values.nativeClientId`\ + \ in GET /qqq/v1/metaData/authentication), for\n example from the device-authorization\ + \ grant or auth-code + PKCE. It is verified against the\n provider's keys\ + \ (issuer, audience, expiry, signature) and gives the same user and permissions\n\ + \ as signing in on the web. When it expires (401), refresh it at the provider\ + \ and retry; or\n- a session UUID returned by POST /qqq/v1/manageSession,\ + \ which the server keeps refreshed." + scheme: "bearer" + type: "http" + oauth2: + description: "The app's OAuth2 / OpenID Connect provider. The real endpoints\ + \ come from the provider's discovery\ndocument: `{values.baseUrl}.well-known/openid-configuration`,\ + \ with values from\nGET /qqq/v1/metaData/authentication. Native clients use\ + \ `values.nativeClientId` (a public client)\nwith auth-code + PKCE (loopback\ + \ or custom-scheme redirect URI registered at the provider) or the\ndevice-authorization\ + \ grant (`device_authorization_endpoint`, RFC 8628), then send the access\ + \ token\nas `Authorization: Bearer`." + flows: + authorizationCode: + authorizationUrl: "/authorize" + scopes: + email: "Email address" + offline_access: "A refresh token, to renew the access token without signing\ + \ in again" + openid: "OpenID Connect sign-in" + profile: "Name, username and groups (permissions come from the groups)" + tokenUrl: "/token" + type: "oauth2" info: contact: email: "contact@kingsrook.com" @@ -303,10 +375,37 @@ paths: schema: $ref: "#/components/schemas/DbPerformanceOverviewV1" description: "Successful Response" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Not found, or not visible to this user (code NOT_FOUND)." + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Conflict with the current state, e.g. an edit conflict or\ + \ an invalid transition (code CONFLICT)." + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." security: - sessionUuidCookie: - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" summary: "Database Performance overview" + tags: + - "db-performance" /qqq/v1/qbit/db-performance/activity: get: description: "Requires db_performance.view and uses the caller's session." @@ -343,10 +442,37 @@ paths: schema: $ref: "#/components/schemas/DbPerformanceActivityV1" description: "Successful Response" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Not found, or not visible to this user (code NOT_FOUND)." + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Conflict with the current state, e.g. an edit conflict or\ + \ an invalid transition (code CONFLICT)." + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." security: - sessionUuidCookie: - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" summary: "Database Performance activity" + tags: + - "db-performance" /qqq/v1/qbit/db-performance/explain: post: description: "PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze.\ @@ -383,10 +509,37 @@ paths: schema: $ref: "#/components/schemas/DbPerformanceExplainResultV1" description: "Successful Response" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Not found, or not visible to this user (code NOT_FOUND)." + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Conflict with the current state, e.g. an edit conflict or\ + \ an invalid transition (code CONFLICT)." + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." security: - sessionUuidCookie: - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" summary: "Explain a query (PostgreSQL JSON or H2 text plan)" + tags: + - "db-performance" /qqq/v1/qbit/db-performance/statements/lookup: get: description: "Requires db_performance.view and uses the caller's session." @@ -428,7 +581,40 @@ paths: schema: $ref: "#/components/schemas/DbPerformanceLookupV1" description: "Successful Response" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Not found, or not visible to this user (code NOT_FOUND)." + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Conflict with the current state, e.g. an edit conflict or\ + \ an invalid transition (code CONFLICT)." + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." security: - sessionUuidCookie: - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" summary: "Database Performance lookup" + tags: + - "db-performance" +servers: +- description: "The QQQ server that serves this document (paths include /qqq/v1)." + url: "/" +tags: +- description: "Operations of the db-performance extension." + name: "db-performance" diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java index a63aeb3..bd74f89 100644 --- a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java @@ -56,7 +56,7 @@ class DbPerfApiContractTest } assertThat(UiApiContract.check(generated, Files.readString(path), null, 1, "mvnw -Dtest=DbPerfApiContractTest -Ddbperf.openapi.update=true test")).isEmpty(); var model = version.generateExtensionOpenAPIModel("qqq", extension); - assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> name.startsWith("DbPerformance") && name.endsWith("V1")); + assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> (name.startsWith("DbPerformance") || name.equals("BasicErrorResponseV1")) && name.endsWith("V1")); assertThat(model.getPaths().keySet()).containsExactlyInAnyOrder("/qqq/v1/qbit/db-performance/overview", "/qqq/v1/qbit/db-performance/activity", "/qqq/v1/qbit/db-performance/explain", "/qqq/v1/qbit/db-performance/statements/lookup"); assertThat(extension.getSemanticVersion()).isEqualTo("1.0.0"); -- 2.54.0