From f8d25d1f7c37bd4a5e2e29de1b83e05f1cf9c8d3 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 15 Oct 2026 12:06:37 -0500 Subject: [PATCH] feat(ui): plan viewer, db-performance UI API extension, app home widgets, help bundle and docs Query plan record action and POST explain: plain or generic EXPLAIN, and EXPLAIN (ANALYZE, BUFFERS) for single SELECTs in a rolled-back READ ONLY transaction with statement_timeout, behind explain_analyze, always audited, ANALYZE plans and parameters never stored. UI API extension /qqq/v1/qbit/db-performance (overview, activity with live text only for view_live_text, explain, statements/lookup) with a committed OpenAPI fragment and contract test. Eight full-width typed home widgets. Data-driven help bundle with a coverage test. README, CHANGELOG and UPGRADE. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +- CHANGELOG.md | 13 +- CLAUDE.md | 6 +- README.md | 150 ++- UPGRADE.md | 13 + pom.xml | 27 +- .../dbperformance/DbPerfQBitProducer.java | 20 + .../qbits/dbperformance/api/ActivitySpec.java | 67 ++ .../dbperformance/api/DbPerformanceData.java | 349 +++++++ .../api/DbPerformanceJsonSpec.java | 100 ++ .../api/DbPerformanceSchemas.java | 151 +++ .../dbperformance/api/DbPerformanceUiApi.java | 65 ++ .../qbits/dbperformance/api/ExplainSpec.java | 152 +++ .../qbits/dbperformance/api/LookupSpec.java | 86 ++ .../qbits/dbperformance/api/OverviewSpec.java | 67 ++ .../dbperformance/explain/ExplainRequest.java | 42 + .../dbperformance/explain/ExplainResult.java | 34 + .../dbperformance/explain/ExplainService.java | 312 ++++++ .../dbperformance/explain/ExplainSql.java | 257 +++++ .../explain/PlanTextRenderer.java | 102 ++ .../explain/QueryPlanProcess.java | 144 +++ .../dbperformance/metadata/DbPerfHelp.java | 94 ++ .../widgets/DbPerfTimeChart.java | 44 + .../widgets/DbPerfWidgetRenderer.java | 363 +++++++ .../dbperformance/widgets/DbPerfWidgets.java | 81 ++ src/main/resources/help/project-version.txt | 1 + .../resources/help/qbit-db-performance.yaml | 946 ++++++++++++++++++ .../openapi/qqq-v1/db-performance.yaml | 434 ++++++++ .../dbperformance/DbPerfApiContractTest.java | 66 ++ .../dbperformance/DbPerfHelpBundleTest.java | 76 ++ .../dbperformance/DbPerfPresentationTest.java | 292 ++++++ .../dbperformance/ExplainConnectionTest.java | 322 ++++++ .../dbperformance/ExplainServiceTest.java | 239 +++++ .../qbits/dbperformance/ExplainSqlTest.java | 86 ++ 34 files changed, 5153 insertions(+), 54 deletions(-) create mode 100644 UPGRADE.md create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/ActivitySpec.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceData.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceJsonSpec.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceSchemas.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceUiApi.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/LookupSpec.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/api/OverviewSpec.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/explain/QueryPlanProcess.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfHelp.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfTimeChart.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java create mode 100644 src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgets.java create mode 100644 src/main/resources/help/project-version.txt create mode 100644 src/main/resources/help/qbit-db-performance.yaml create mode 100644 src/main/resources/openapi/qqq-v1/db-performance.yaml create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHelpBundleTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/ExplainConnectionTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/ExplainServiceTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dbperformance/ExplainSqlTest.java diff --git a/AGENTS.md b/AGENTS.md index 5b5f1ab..8fbb781 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This file provides guidance to Codex when working with code in this repository. ## Project Overview -**QBit DB Performance**: Database performance for QQQ apps: slow and expensive queries from pg_stat_statements with delta snapshots, table scan and index analysis, live activity and blocking chains, connection capacity, maintenance, QQQ query attribution with N+1 detection, and ranked suggestions, all as standard QQQ tables. Design: docs/SPEC.md in this repository. Implementation in progress. +**QBit DB Performance**: Database performance for QQQ apps in standard secured tables: PostgreSQL statement deltas, table and index analysis, live activity and blocking, connection capacity, audited read-only query plans, typed full-width dashboards, a versioned UI API and data-driven help. Collection and presentation are implemented; attribution, findings, alerts and health runtime integrations are developed separately. Design: docs/SPEC.md. ## Build Commands @@ -21,7 +21,7 @@ This file provides guidance to Codex when working with code in this repository. ## Naming (owner rule) - Tables are lowercase snake_case prefixed with the full qbit name: `db_perf_`. The QQQ table name and the backend (SQL) name are identical; possible-value sources take the table name. -- Permissions use the full qbit name: `db_perf.` (for example `db_perf.view`, `db_perf.admin`). +- Permissions use the full qbit name: `db_performance.` (for example `db_performance.view`, `db_performance.manage_settings`). - Qbit data lives in standard, visible QQQ tables, secured by permissions and record security locks, never hidden. - Processes run as table or record actions, never in the nav. @@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository. ## Dependencies - Java 21. -- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.10` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1. +- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.11` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1. - Tests: JUnit 5, AssertJ. ## Release diff --git a/CHANGELOG.md b/CHANGELOG.md index 3919c93..4a5b524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ # Changelog -All notable changes to this project are documented here. The format follows Keep a Changelog; versions follow semantic versioning. +All notable changes are documented here. This project follows Keep a Changelog and semantic versioning. -## [0.1.0] - Unreleased +## [Unreleased] ### Added -- Repository scaffold: `DbPerfQBitConfig` and `DbPerfQBitProducer` (table prefix `db_perf_`, permissions `db_perf.*`), build, CI and repository standard files. + +- Secured database-performance tables, PostgreSQL statistics snapshots, daily rollups and configurable retention. +- Audited PLAN/GENERIC and read-only ANALYZE query viewer with SQL validation, timeout, rollback and transient parameter handling. +- Query plan statement action with readable plan text, cost, SQL Workbench link and a sanitized assistant prompt. +- UI API extension 1.0.0 with overview, live activity, explain and statement lookup, plus its committed OpenAPI contract. +- Eight full-width typed home panels with pagination, record links, guided empty states and permission-gated live activity. +- Maven-versioned QBIT help bundle, complete metadata coverage checks, rule threshold guidance and adoption documentation. +- PostgreSQL security acceptance tests and H2 API/widget fixtures. diff --git a/CLAUDE.md b/CLAUDE.md index 9645895..baab3f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code when working with code in this reposi ## Project Overview -**QBit DB Performance**: Database performance for QQQ apps: slow and expensive queries from pg_stat_statements with delta snapshots, table scan and index analysis, live activity and blocking chains, connection capacity, maintenance, QQQ query attribution with N+1 detection, and ranked suggestions, all as standard QQQ tables. Design: docs/SPEC.md in this repository. Implementation in progress. +**QBit DB Performance**: Database performance for QQQ apps in standard secured tables: PostgreSQL statement deltas, table and index analysis, live activity and blocking, connection capacity, audited read-only query plans, typed full-width dashboards, a versioned UI API and data-driven help. Collection and presentation are implemented; attribution, findings, alerts and health runtime integrations are developed separately. Design: docs/SPEC.md. ## Build Commands @@ -21,7 +21,7 @@ This file provides guidance to Claude Code when working with code in this reposi ## Naming (owner rule) - Tables are lowercase snake_case prefixed with the full qbit name: `db_perf_`. The QQQ table name and the backend (SQL) name are identical; possible-value sources take the table name. -- Permissions use the full qbit name: `db_perf.` (for example `db_perf.view`, `db_perf.admin`). +- Permissions use the full qbit name: `db_performance.` (for example `db_performance.view`, `db_performance.manage_settings`). - Qbit data lives in standard, visible QQQ tables, secured by permissions and record security locks, never hidden. - Processes run as table or record actions, never in the nav. @@ -35,7 +35,7 @@ This file provides guidance to Claude Code when working with code in this reposi ## Dependencies - Java 21. -- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.10` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1. +- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.11` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1. - Tests: JUnit 5, AssertJ. ## Release diff --git a/README.md b/README.md index ab73abe..7a94b16 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,133 @@ # QBit DB Performance -[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://www.apache.org/licenses/LICENSE-2.0) -[![Java](https://img.shields.io/badge/java-21+-blue.svg)](https://adoptium.net/) - -> Database performance for QQQ apps: slow and expensive queries from pg_stat_statements with delta snapshots, table scan and index analysis, live activity and blocking chains, connection capacity, maintenance, QQQ query attribution with N+1 detection, and ranked suggestions, all as standard QQQ tables. - -**Status:** Phase A is implemented: schema and secured metadata, PostgreSQL statistics collection, daily rollups and retention. Attribution listeners, suggestions, explain actions, dashboard/API integration and help are later phases. Nothing is published yet. Approved design: [docs/SPEC.md](docs/SPEC.md). +Database performance for QQQ apps: PostgreSQL statement deltas, table and index statistics, live sessions and blocking, connection capacity, audited query plans, and typed home dashboards in standard secured QQQ tables. Approved design: [docs/SPEC.md](docs/SPEC.md). This branch implements collection and the presentation/plan-viewer phase; attribution listeners, findings evaluation, alerts and health are developed in a separate phase. No release has been published yet. ## Requirements -- Java 21. -- QQQ 4.1 (built against `4.1.0-RC.11`). -- PostgreSQL 13+ for collection; PostgreSQL 17 and H2 2.2 are migration test targets. +- Java 21 and QQQ **4.1.0-RC.11+**. +- PostgreSQL **13+**; **16+** for parameterized generic plans, with the pgJDBC driver (built against 42.7.14). +- `pg_stat_statements` preloaded, created in the monitored database, and readable by the application role. +- `pg_read_all_stats` granted to that role for complete statistics and other sessions' visibility. +- A host scheduler for automatic collection and rollups. Docker is required for the PostgreSQL acceptance tests. ## Install -Not yet published. Once released, add `com.kingsrook.qbits:qbit-db-performance` and produce the qbit into your `QInstance`: +Add the dependency when the first release is available (use your locally installed snapshot during development): + +```xml + + com.kingsrook.qbits + qbit-db-performance + 0.1.0-SNAPSHOT + +``` + +Register a QQQ `PostgreSQLBackendMetaData` backend and the scheduler first, then produce the qbit: ```java new DbPerfQBitProducer() .withQBitConfig(new DbPerfQBitConfig() .withBackendName("rdbms") - .withSchedulerName("scheduler")) // omit to register no schedules + .withSchedulerName("scheduler") + .withParentAppName("operations")) .produce(qInstance); ``` -The backend must already exist. For PostgreSQL, use QQQ's -`PostgreSQLBackendMetaData` from `qqq-backend-module-postgres` so table actions -use the PostgreSQL SQL dialect. The qbit reads statistics through the backend's -connection manager and writes its records through QQQ actions as the system user. +The optional parent must already exist. Include the migration in the host's Liquibase master: -Include `db/liquibase/qbit-db-performance/db_performance.changelog.sql` in the -host's Liquibase changelog. It creates 17 tables and seeds 14 suggestion rules -without overwriting administrator edits. The runtime creates its Settings row -on first use. It does not run migrations itself. +```xml + +``` -Enable `pg_stat_statements` in PostgreSQL's `shared_preload_libraries`, create -the extension in the monitored database, and grant `pg_read_all_stats` to the -application database role. Missing statistics access produces a PARTIAL snapshot -and a Settings message; unsupported database vendors remain usable for metadata. +The changelog creates 17 tables and seeds 14 rules without overwriting administrator edits. The qbit never runs migrations itself. Declare the database role grant through your normal database provisioning: -## Naming +```sql +GRANT pg_read_all_stats TO your_application_role; +``` -Tables are `db_perf_` (lowercase snake_case, backend name equal to the table name). Permissions are `db_performance.`; the complete names are in `DbPerfPermissions.ALL`. +Configure `shared_preload_libraries` with `pg_stat_statements` and create the extension in the monitored database through your normal deployment process. Missing access produces reduced-capability status and setup guidance. -The app and tables require `db_performance.view`. Settings updates, rule changes -and Collect now require `db_performance.manage_settings`. Other tables are -read-only to users. Settings and rules have no AI exposure. +With `schedulerName`, `dbPerfCollectSnapshot` runs every five minutes and `dbPerfRollupAndPurge` daily at 03:15 (UTC by default). Omitting it registers no schedules. Use **Collect now** on Snapshots for the first baseline and a second collection for deltas. Table and index statistics normally update hourly. The scheduled processes are durable-eligible when the host has durable jobs enabled. -Collection defaults to every five minutes, with table/index samples hourly. -New statements establish a baseline instead of booking historical counts into -one interval. Snapshot totals include all statement deltas; only the top 200 -details are stored by default. Therefore a statement's 24-hour totals and daily -rollup reflect its retained details, while its share uses the full database total. +UI hosts must include `qqq-middleware-javalin` (an optional dependency of this qbit). The producer registers UI API extension `db-performance` version `1.0.0` under `/qqq/v1/qbit/db-performance`. Core-only hosts can use tables and collection without middleware. -The daily process rolls up yesterday and missing days in the preceding week, -then purges details after 30 days and daily rows after 365 days. These thresholds -are editable in Settings. Days use the configured time zone, UTC by default. -Processes are scheduled or exposed as table actions, never navigation children. +## Tables + +All table names and SQL names are identical; all data is visible through standard QQQ tables. + +| Tables | Purpose | Retention | +|---|---|---| +| `db_perf_settings`, `db_perf_rule` | Per-backend settings and rule thresholds | Indefinite | +| `db_perf_snapshot` | Collection status, interval totals and capacity | 30 days | +| `db_perf_statement` | Normalized query inventory and retained 24-hour totals | Stale after 365 days | +| `db_perf_query_snapshot`, `db_perf_query_daily` | Statement interval details and daily aggregates | 30 / 365 days | +| `db_perf_table_stat`, `db_perf_index_stat` | Current table/index inventory and maintenance | Stale after 365 days | +| `db_perf_table_snapshot`, `db_perf_table_daily` | Table history and growth | 30 / 365 days | +| `db_perf_activity_sample`, `db_perf_connection_sample` | Notable activity and connection counts | 30 days | +| `db_perf_attribution`, `db_perf_attribution_daily`, `db_perf_repeat_pattern` | QQQ units of work and repeated statements | 30 / 365 / 30 days | +| `db_perf_finding` | Ranked suggestions and evidence | Open retained; stale closed after 365 days | +| `db_perf_explain` | Every plan request and eligible saved plans | 30 days | + +The home page stacks Setup, Database time, Top queries, Findings, Live activity, Connections, QQQ attribution and Tables at full width. Tables paginate; widgets refresh every 30–60 seconds. Statement records include Query Plans and snapshot/daily/finding child lists. Attribution and findings panels remain guided empty states until their producing runtime is installed and has collected data. + +Snapshot totals include all observed statement deltas. Only the top configured details are retained, so statement totals and rollups can undercount queries outside that selection. The database-time share denominator includes all observed statement deltas. Reset counters start new baselines rather than creating spikes. + +## Permissions + +| Permission | Access | +|---|---| +| `db_performance.view` | App, read-only tables, widgets, overview, lookup and normalized activity | +| `db_performance.view_live_text` | Raw SQL in the transient activity API only; also requires view | +| `db_performance.explain` | PLAN/GENERIC requests and Query plan record action | +| `db_performance.explain_analyze` | Execute the guarded ANALYZE service; the record action also needs explain | +| `db_performance.manage_settings` | Settings/rule edits and Collect now | +| `db_performance.manage_findings` | Finding status actions supplied by the findings phase | + +Hosts can change `permissionBaseName`; grant the corresponding complete names. Persistent collector and explain-audit writes run as the system user, retaining the requesting session's user ID for plans. Dashboard and API reads retain the viewer's session. + +## Configuration + +| Host option | Default / meaning | +|---|---| +| `backendName` | Required; monitored database and qbit storage backend | +| `schedulerName` | Optional; host-registered scheduler | +| `appName`, `parentAppName` | `dbPerformance`; optional existing parent app | +| `permissionBaseName` | `db_performance` | +| `attributionEnabled` | `true`; attribution listener integration is a separate phase | +| `sqlWorkbenchRunnerPath` | `/app/sqlWorkbenchRunner` | +| `podName` | `HOSTNAME`, or `local` | +| `snapshotCron`, `analyzeCron`, `rollupCron` | `0 */5 * * * ?`, `0 7 * * * ?`, `0 15 3 * * ?`; analysis schedule is owned by the findings phase | +| `timeZone` | `UTC` | + +Runtime values live in Settings: 5-minute collection, hourly tables, 200 top statement details, 30-day detail retention, 365-day rollups, 10-second explain timeout, 30-second long-query and 60-second idle-transaction thresholds. Attribution and alert settings are documented in the shipped help bundle, as are every rule's threshold units and seeded values. Existing administrator settings are never overwritten on startup. + +## Query plans and UI API + +**Query plan** on one statement presents PLAN/ANALYZE and optional parameter values (one per line). The Result screen shows selectable plan text, cost, a SQL Workbench link and an assistant prompt. There is no action on the Query Plans audit table. + +- `GET /overview?backend=`: setup status, latest snapshot, top ten queries and open findings. +- `GET /activity?backend=`: current provider sessions and transitive blocking chains; raw text is permission-gated. +- `POST /explain`: `{ "backend": "rdbms", "sql": "select * from orders where id = $1", "mode": "PLAN" }`. Supply either `statementId` or `sql`. For ANALYZE, add `"parameters": ["123"]` and obtain its separate permission. +- `GET /statements/lookup?backend=&sql=`: normalized fingerprint matches with `/app//db_perf_statement/` history paths. + +Paths above are relative to `/qqq/v1/qbit/db-performance`. The [committed OpenAPI fragment](src/main/resources/openapi/qqq-v1/db-performance.yaml) is generated from endpoint specs and verified by a contract test. The server never calls an AI model. + +## Security and PHI + +Stored SQL is normalized and fingerprinted. Raw `pg_stat_activity` text is never persisted and appears only in the activity API with `view_live_text`; home widgets always use normalized text. SQL literals and parameter values must not be used as identifiers: normalization preserves quoted identifiers. Settings and rules are excluded from AI exposure. + +PLAN executes EXPLAIN in a read-only transaction with a local timeout and rollback. Parameterized PLAN uses PostgreSQL 16+ GENERIC_PLAN. Only eligible literal-free PLAN/GENERIC plans are stored. A user's SQL must equal its own normalized form to qualify; SQL loaded from the normalized statement table also qualifies. + +ANALYZE accepts one SELECT or a non-modifying WITH, rejects INTO and row locks, binds supplied strings as unknown JDBC parameters, and executes inside a read-only transaction. Functions are still constrained by PostgreSQL READ ONLY. The EXPLAIN command uses pgJDBC simple-query execution with JDBC-managed parameter quoting, so PostgreSQL can coerce unknown-typed EXECUTE arguments. The transaction always rolls back and the prepared statement is explicitly deallocated before the connection returns to its pool. SQLSTATE 57014 records TIMEOUT. The service stores only a sanitized audit, never ANALYZE plans or parameter values. Returned ANALYZE plans are transient and can contain values; the assistant prompt redacts literals in plan expressions. Review transient plans before copying them elsewhere. ## Adopting in another app -Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/docs/migration/adoption-checklist.adoc) once the first release is published. +1. Add compatible QQQ, PostgreSQL backend and qbit dependencies; include the changelog once. +2. Preload/create pg_stat_statements and grant the database statistics role through your provisioning. +3. Register the backend, optional parent app and scheduler, then produce the qbit. +4. Grant the exact permissions to intended operators; reserve ANALYZE and raw live text for appropriate roles. +5. Run Collect now twice, check Setup, and confirm the API extension in `/qqq/v1/openapi.yaml`. +6. Optionally configure SQL Workbench links; install attribution/findings runtime integrations when available. Review and commit proposed database changes through the host's migration workflow. ## Build @@ -69,12 +135,8 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/ ./mvnw -o verify ``` -Always use the Maven wrapper (Maven 3.9.11); Maven 3.10 cannot read the parent POM. -On Windows use `./mvnw.cmd -o verify`. Validation includes main/test checkstyle, -JUnit tests and JaCoCo gates of 70% instructions and 90% classes. PostgreSQL -Testcontainers tests (including `PostgresStatsProviderIT`) run with the suite and -skip when Docker is unavailable. +On Windows use `./mvnw.cmd -o verify`. Always use the wrapper; Maven 3.10 cannot read the parent POM. Offline mode requires dependencies already cached. Verification runs main/test Checkstyle, JUnit including PostgreSQL 17 Testcontainers, and JaCoCo gates of 70% instructions and 90% classes. Docker must be accessible to the build process. Regenerate the API fragment with `./mvnw -o -Dtest=DbPerfApiContractTest -Ddbperf.openapi.update=true test`. ## License -Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). +[Apache License 2.0](LICENSE). diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000..4b956e6 --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,13 @@ +# Upgrading QBit DB Performance + +## First release + +There is no previous published release. See [CHANGELOG.md](CHANGELOG.md) for additions and [README.md](README.md) for complete installation. + +Use QQQ 4.1.0-RC.11 or newer, Java 21, and PostgreSQL 13 or newer (16+ for generic plans). Include `db/liquibase/qbit-db-performance/db_performance.changelog.sql` once in your application's Liquibase changelog. Run migrations through the host deployment before collecting statistics; the qbit does not apply them automatically. + +Preload and create `pg_stat_statements`, then grant `pg_read_all_stats` to the application's database role through your normal provisioning. Without the role the collector reports reduced visibility. Preserve existing rule and settings edits; seeding is insert-only. + +Register the producer and a host scheduler. Grant `db_performance.view` and the narrower operator permissions appropriate to each role; ANALYZE and raw live text require separate grants. Existing deployments using an early scaffold's `db_perf.*` grants must use the actual `db_performance.*` names (or the configured permission base). + +Collect twice to establish useful deltas. Inspect Setup for access and schedule status. UI hosts need the optional `qqq-middleware-javalin` dependency so the four endpoints register in the standard UI API. The help bundle version is filtered from the Maven project version during the build. diff --git a/pom.xml b/pom.xml index 402710d..0cd031c 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ 21 21 0.8.15 + 5.23.0 @@ -67,6 +68,17 @@ + + org.mockito + mockito-core + ${mockito.version} + test + + + com.kingsrook.qqq + qqq-middleware-javalin + true + com.kingsrook.qqq qqq-backend-module-postgres @@ -92,7 +104,7 @@ org.postgresql postgresql 42.7.14 - test + true org.testcontainers @@ -126,12 +138,25 @@ + + + src/main/resources + false + help/** + + + src/main/resources + true + help/** + + org.apache.maven.plugins maven-surefire-plugin 3.2.5 + @{argLine} -javaagent:"${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar" **/*Test.java **/*IT.java diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/DbPerfQBitProducer.java b/src/main/java/com/kingsrook/qbits/dbperformance/DbPerfQBitProducer.java index 94d40ab..5bbc0a8 100644 --- a/src/main/java/com/kingsrook/qbits/dbperformance/DbPerfQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/dbperformance/DbPerfQBitProducer.java @@ -138,6 +138,26 @@ public class DbPerfQBitProducer implements QBitMetaDataProducer snapshots() throws QException + { + requireView(null); + return (DbPerfStore.query("db_perf_snapshot", backendFilter().withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN_OR_EQUALS, Instant.now().minusSeconds(86400))) + .withOrderBy(new QFilterOrderBy("capturedAt", true)))); + } + + + + /******************************************************************************* + ** Ranked statements with literal-free text and canonical history links. + *******************************************************************************/ + public List> statements(int limit, String fingerprint) throws QException + { + requireView(null); + QQueryFilter filter = backendFilter().withOrderBy(new QFilterOrderBy("totalMs24h", false)).withLimit(limit); + if(fingerprint != null) + { + filter.withCriteria(new QFilterCriteria("fingerprint", QCriteriaOperator.EQUALS, fingerprint)); + } + List> results = new ArrayList<>(); + for(QRecord record : DbPerfStore.query("db_perf_statement", filter)) + { + Map item = fields(record, "id", "calls24h", "meanMs24h", "totalMs24h", "shareOfDbTime24h"); + String text = SqlNormalizer.normalize(record.getValueString("queryText")); + item.put("queryText", fingerprint == null ? text.substring(0, Math.min(300, text.length())) : text); + item.put("historyPath", recordPath("db_perf_statement", record.getValueInteger("id"))); + item.put("hitRatio", hitRatio(record)); + results.add(item); + } + return (results); + } + + + + /******************************************************************************* + ** 24-hour cache hit ratio uses deltas rather than cumulative statement counters. + *******************************************************************************/ + private BigDecimal hitRatio(QRecord record) throws QException + { + List rows = DbPerfStore.query("db_perf_query_snapshot", DbPerfStore.eq("statementId", record.getValueInteger("id")) + .withCriteria(new QFilterCriteria("capturedAt", QCriteriaOperator.GREATER_THAN_OR_EQUALS, Instant.now().minusSeconds(86400)))); + double hits = rows.stream().mapToDouble(row -> number(row, "sharedBlksHit")).sum(); + double reads = rows.stream().mapToDouble(row -> number(row, "sharedBlksRead")).sum(); + return (hits + reads == 0 ? null : BigDecimal.valueOf(hits / (hits + reads))); + } + + + + /******************************************************************************* + ** Open and acknowledged findings stay actionable and are ordered by score. + *******************************************************************************/ + public List findings(int limit) throws QException + { + requireView(null); + return (DbPerfStore.query("db_perf_finding", backendFilter() + .withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, List.of("OPEN", "ACKNOWLEDGED"))) + .withOrderBy(new QFilterOrderBy("score", false)).withLimit(limit))); + } + + + + /******************************************************************************* + ** Small overview response; last snapshot metrics are interval deltas. + *******************************************************************************/ + public Map overview(String backend) throws QException + { + requireView(backend); + Map counts = new LinkedHashMap<>(); + List> top = new ArrayList<>(); + for(QRecord record : findings(Integer.MAX_VALUE)) + { + counts.merge(record.getValueString("severity"), 1, Integer::sum); + if(top.size() < 5) + { + top.add(fields(record, "id", "title", "severity", "score")); + } + } + return (map("settings", fields(settings(), "provider", "statsAccess", "statsAccessMessage", "lastSnapshotAt"), + "snapshot", fields(latest(), "id", "capturedAt", "status", "connectionsTotal", "maxConnections", "cacheHitRatio", "tempBytes", "deadlocks", "statementTotalMs", "statementCalls"), + "topStatements", statements(10, null), "findingsBySeverity", counts, "topFindings", top)); + } + + + + /******************************************************************************* + ** Link arbitrary SQL shapes back to their normalized statement history. + *******************************************************************************/ + public Map lookup(String backend, String sql) throws QException + { + requireView(backend); + if(sql == null || sql.isBlank()) + { + throw (new QBadRequestException("Supply SQL to look up.")); + } + String normalized = SqlNormalizer.normalize(sql); + String fingerprint = SqlNormalizer.fingerprint(normalized); + return (map("normalizedSql", normalized, "fingerprint", fingerprint, "matches", statements(100, fingerprint))); + } + + + + /******************************************************************************* + ** Live activity is read now, not from saved samples, and is never persisted. + *******************************************************************************/ + public Map activity(String backend, boolean allowRawText) throws Exception + { + requireView(backend); + try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName()))) + { + DbStatsProvider provider = DbStatsProviders.forConnection(connection); + return (activity(provider.readSessions(connection), allowRawText)); + } + catch(java.sql.SQLException e) + { + throw (new QException("Live activity is unavailable. Check the backend and statistics access.")); + } + } + + + + /******************************************************************************* + ** Pure projection also used by provider fixture tests; widgets always pass false. + *******************************************************************************/ + public Map activity(List sessions, boolean allowRawText) throws QException + { + requireView(null); + boolean redacted = !allowRawText || !QContext.getQSession().hasPermission(runtime.getConfig().getPermissionBaseName() + ".view_live_text"); + Map> blocked = new LinkedHashMap<>(); + for(SessionInfo session : sessions) + { + for(Integer blocker : session.blockedBy()) + { + blocked.computeIfAbsent(blocker, key -> new HashSet<>()).add(session.pid()); + } + } + List> rows = new ArrayList<>(); + List> chains = new ArrayList<>(); + Set allBlocked = new HashSet<>(); + blocked.values().forEach(allBlocked::addAll); + for(SessionInfo session : sessions) + { + String normalized = SqlNormalizer.normalize(session.queryText()); + rows.add(map("pid", session.pid(), "user", session.dbUser(), "application", session.applicationName(), "state", session.state(), + "waitEvent", session.waitEvent(), "querySeconds", session.querySeconds(), "xactSeconds", session.xactSeconds(), + "blockedBy", session.blockedBy(), "blockingCount", blocked.getOrDefault(session.pid(), Set.of()).size(), + "fingerprint", SqlNormalizer.fingerprint(normalized), "normalizedText", normalized, "query", redacted ? null : session.queryText(), "liveTextRedacted", redacted)); + } + for(Integer root : blocked.keySet()) + { + if(!allBlocked.contains(root)) + { + Set descendants = new java.util.TreeSet<>(); + descendants(root, blocked, descendants); + descendants.remove(root); + chains.add(map("rootPid", root, "blockedPids", new ArrayList<>(descendants))); + } + } + return (map("sessions", rows, "blockingChains", chains, "liveTextRedacted", redacted)); + } + + + + /******************************************************************************* + ** Traverse transitive blocking with cycle protection. + *******************************************************************************/ + private static void descendants(Integer pid, Map> blocked, Set result) + { + for(Integer child : blocked.getOrDefault(pid, Set.of())) + { + if(result.add(child)) + { + descendants(child, blocked, result); + } + } + } + + + + /******************************************************************************* + ** One record path builder for API and widgets, including custom app names. + *******************************************************************************/ + public String recordPath(String table, Integer id) + { + return ("/app/" + runtime.getConfig().getAppName() + "/" + table + "/" + id); + } + + + + /******************************************************************************* + ** Explicit allowlist prevents returning raw counters, configuration or future fields. + *******************************************************************************/ + public static Map fields(QRecord record, String... names) + { + Map result = new LinkedHashMap<>(); + for(String name : names) + { + result.put(name, record == null ? null : record.getValue(name)); + } + return (result); + } + + + + /******************************************************************************* + ** Nullable JSON object with stable property order. + *******************************************************************************/ + public static Map map(Object... pairs) + { + Map result = new LinkedHashMap<>(); + for(int i = 0; i < pairs.length; i += 2) + { + result.put((String) pairs[i], pairs[i + 1]); + } + return (result); + } + + + + /******************************************************************************* + ** Nullable statistics arithmetic. + *******************************************************************************/ + public static double number(QRecord row, String field) + { + return (row == null || row.getValue(field) == null ? 0 : ((Number) row.getValue(field)).doubleValue()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceJsonSpec.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceJsonSpec.java new file mode 100644 index 0000000..89cc778 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceJsonSpec.java @@ -0,0 +1,100 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.api; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dbperformance.DbPerfRuntime; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec; +import com.kingsrook.qqq.openapi.model.In; +import com.kingsrook.qqq.openapi.model.Method; +import com.kingsrook.qqq.openapi.model.Parameter; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; +import io.javalin.http.HttpStatus; + + +/******************************************************************************* + ** Common schemas and error contract, with no alternate route or permission pipeline. + *******************************************************************************/ +public abstract class DbPerformanceJsonSpec extends AbstractJsonEndpointSpec +{ + + + + /******************************************************************************* + ** Stable operation identifiers for generated clients. + *******************************************************************************/ + @Override + public Method defineMethod() + { + return (super.defineMethod().withOperationId("dbPerformance" + getClass().getSimpleName().replace("Spec", ""))); + } + + + + /******************************************************************************* + ** Publish all nested response and request component schemas. + *******************************************************************************/ + @Override + public Map defineComponentSchemas() + { + return (DbPerformanceSchemas.all()); + } + + + + /******************************************************************************* + ** Backend defaults to the qbit's single configured monitored backend for reads. + *******************************************************************************/ + @Override + public List defineRequestParameters() + { + return (List.of(new Parameter().withName("backend").withIn(In.QUERY).withRequired(false) + .withSchema(new Schema().withType(Type.STRING)).withDescription("Configured monitored backend; defaults to the qbit backend."))); + } + + + + /******************************************************************************* + ** Framework exceptions map to the same status in every UI client. + *******************************************************************************/ + @Override + public List defineAdditionalBasicResponses() + { + return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Invalid request", "DbPerformanceErrorV1"), + new BasicResponse(HttpStatus.UNAUTHORIZED, "Sign in required", "DbPerformanceErrorV1"), + new BasicResponse(HttpStatus.FORBIDDEN, "Permission denied", "DbPerformanceErrorV1"))); + } + + + + /******************************************************************************* + ** Resolve per-request runtime from the authenticated QContext. + *******************************************************************************/ + protected DbPerformanceData data() + { + return (new DbPerformanceData(DbPerfRuntime.of(QContext.getQInstance()))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceSchemas.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceSchemas.java new file mode 100644 index 0000000..04bc566 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceSchemas.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.api; + + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + + +/******************************************************************************* + ** Explicit versioned schemas; every public property is described for generated clients. + *******************************************************************************/ +public final class DbPerformanceSchemas +{ + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private DbPerformanceSchemas() + { + } + + + + /******************************************************************************* + ** Shared schema components referenced by the four independently versioned operations. + *******************************************************************************/ + public static Map all() + { + Map schemas = new TreeMap<>(); + schemas.put("DbPerformanceErrorV1", object("error", text())); + schemas.put("DbPerformanceSettingsV1", object("provider", text(), "statsAccess", text(), "statsAccessMessage", text(), "lastSnapshotAt", text())); + schemas.put("DbPerformanceSnapshotV1", object("id", integer(), "capturedAt", text(), "status", text(), "connectionsTotal", integer(), "maxConnections", integer(), + "cacheHitRatio", number(), "tempBytes", integer(), "deadlocks", integer(), "statementTotalMs", number(), "statementCalls", integer())); + schemas.put("DbPerformanceStatementV1", object("id", integer(), "queryText", text().withDescription("Normalized SQL, truncated to 300 characters."), + "calls24h", integer(), "meanMs24h", number(), "totalMs24h", number(), "shareOfDbTime24h", number(), "hitRatio", number(), "historyPath", text())); + schemas.put("DbPerformanceFindingV1", object("id", integer(), "title", text(), "severity", text(), "score", number())); + schemas.put("DbPerformanceOverviewV1", object("settings", ref("Settings"), "snapshot", ref("Snapshot"), "topStatements", array(ref("Statement")), + "findingsBySeverity", new Schema().withType(Type.OBJECT).withAdditionalProperties(integer()), "topFindings", array(ref("Finding")))); + schemas.put("DbPerformanceSessionV1", object("pid", integer(), "user", text(), "application", text(), "state", text(), "waitEvent", text(), + "querySeconds", number(), "xactSeconds", number(), "blockedBy", array(integer()), "blockingCount", integer(), "fingerprint", text(), "normalizedText", text(), + "query", text().withDescription("Raw live SQL only with view_live_text; never persisted."), "liveTextRedacted", bool())); + schemas.put("DbPerformanceBlockingChainV1", object("rootPid", integer(), "blockedPids", array(integer()))); + schemas.put("DbPerformanceActivityV1", object("sessions", array(ref("Session")), "blockingChains", array(ref("BlockingChain")), "liveTextRedacted", bool())); + schemas.put("DbPerformanceLookupV1", object("normalizedSql", text(), "fingerprint", text(), "matches", array(ref("Statement")))); + schemas.put("DbPerformanceExplainRequestV1", object("backend", text(), "statementId", integer(), "sql", text(), + "mode", new Schema().withType(Type.STRING).withEnumValues(List.of("PLAN", "ANALYZE")), "parameters", array(text())) + .withDescription("Supply exactly one of statementId or sql. Parameters are strings, accepted only for ANALYZE and never stored.")); + schemas.put("DbPerformanceExplainResultV1", object("mode", text(), "planJson", text(), "planText", text(), "totalCost", number(), "actualTotalMs", number(), + "warnings", array(text()), "explainId", integer(), "sqlWorkbenchUrl", text(), "assistantPrompt", text())); + return (schemas); + } + + + + /******************************************************************************* + ** Nullable scalar for metrics that have not yet been collected. + *******************************************************************************/ + private static Schema text() + { + return (new Schema().withType(Type.STRING).withNullable(true)); + } + + + + /******************************************************************************* + ** Integral count or identifier. + *******************************************************************************/ + private static Schema integer() + { + return (new Schema().withType(Type.INTEGER).withNullable(true)); + } + + + + /******************************************************************************* + ** Fractional duration, cost or ratio. + *******************************************************************************/ + private static Schema number() + { + return (new Schema().withType(Type.NUMBER).withNullable(true)); + } + + + + /******************************************************************************* + ** Required boolean flag. + *******************************************************************************/ + private static Schema bool() + { + return (new Schema().withType(Type.BOOLEAN)); + } + + + + /******************************************************************************* + ** Stable component reference. + *******************************************************************************/ + public static Schema ref(String suffix) + { + return (new Schema().withRefToSchema("DbPerformance" + suffix + "V1")); + } + + + + /******************************************************************************* + ** A list with an explicitly typed item. + *******************************************************************************/ + private static Schema array(Schema item) + { + return (new Schema().withType(Type.ARRAY).withItems(item)); + } + + + + /******************************************************************************* + ** Deterministically ordered properties for stable OpenAPI generation. + *******************************************************************************/ + private static Schema object(Object... pairs) + { + Map properties = new TreeMap<>(); + for(int i = 0; i < pairs.length; i += 2) + { + properties.put((String) pairs[i], (Schema) pairs[i + 1]); + } + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceUiApi.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceUiApi.java new file mode 100644 index 0000000..0133f3a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/DbPerformanceUiApi.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.api; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData; + + +/******************************************************************************* + ** Discoverable UI API extension, never raw Javalin routes. + *******************************************************************************/ +public final class DbPerformanceUiApi +{ + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private DbPerformanceUiApi() + { + } + + + + /******************************************************************************* + ** Add the versioned extension once per host instance. + *******************************************************************************/ + public static void register(QInstance instance) + { + MiddlewareSpecExtensionsMetaData.ofOrWithNew(instance).withExtension(extension()); + } + + + + /******************************************************************************* + ** Public contract used for routing, discovery and committed OpenAPI generation. + *******************************************************************************/ + public static MiddlewareSpecExtension extension() + { + return (new MiddlewareSpecExtension().withName("db-performance").withSemanticVersion("1.0.0") + .withDescription("Database performance, live activity and audited query plans") + .withSpec(OverviewSpec.class).withSpec(ActivitySpec.class).withSpec(ExplainSpec.class).withSpec(LookupSpec.class)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java new file mode 100644 index 0000000..d615036 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/ExplainSpec.java @@ -0,0 +1,152 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.api; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dbperformance.DbPerfRuntime; +import com.kingsrook.qbits.dbperformance.explain.ExplainRequest; +import com.kingsrook.qbits.dbperformance.explain.ExplainService; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation; +import com.kingsrook.qqq.openapi.model.HttpMethod; +import com.kingsrook.qqq.openapi.model.Parameter; +import com.kingsrook.qqq.openapi.model.Schema; + + +/******************************************************************************* + ** Audited plan execution. Service denial is a core permission exception (HTTP 403). + *******************************************************************************/ +public class ExplainSpec extends DbPerformanceJsonSpec +{ + + + + /******************************************************************************* + ** JSON-only POST, with stronger permission for ANALYZE. + *******************************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/db-performance/explain") + .withShortSummary("Explain a PostgreSQL query").withLongDescription("PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze. Values and ANALYZE plans are never persisted.")); + } + + + + /******************************************************************************* + ** All inputs are in the body. + *******************************************************************************/ + @Override + public List defineRequestParameters() + { + return (List.of()); + } + + + + /******************************************************************************* + ** Versioned response shape. + *******************************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("DbPerformanceExplainResultV1"); + } + + + + /******************************************************************************* + ** Versioned request shape. + *******************************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (DbPerformanceSchemas.ref("ExplainRequest")); + } + + + + /******************************************************************************* + ** Validate JSON types without copying arbitrary values to errors or logs. + *******************************************************************************/ + @Override + public Object handle(JsonInput input) throws Exception + { + Map body = input.getBody(); + ExplainRequest.Mode mode = null; + try + { + mode = ExplainRequest.Mode.valueOf(string(body, "mode")); + } + catch(IllegalArgumentException | NullPointerException e) + { + // The service audits invalid modes before returning a user-facing error. + } + Integer id = null; + if(body.get("statementId") != null) + { + try + { + id = new java.math.BigDecimal(body.get("statementId").toString()).intValueExact(); + } + catch(ArithmeticException | NumberFormatException e) + { + throw (new QBadRequestException("statementId must be an integer.")); + } + } + List parameters = new ArrayList<>(); + if(body.get("parameters") != null) + { + if(!(body.get("parameters") instanceof List values)) + { + throw (new QBadRequestException("parameters must be an array of strings.")); + } + for(Object value : values) + { + if(!(value instanceof String)) + { + throw (new QBadRequestException("parameters must be an array of strings.")); + } + parameters.add((String) value); + } + } + return (new ExplainService(DbPerfRuntime.of(QContext.getQInstance())).explain(new ExplainRequest(string(body, "backend"), id, string(body, "sql"), mode, parameters))); + } + + + + /******************************************************************************* + ** Accept strings only, rather than coercing objects into executable SQL. + *******************************************************************************/ + private static String string(Map body, String key) throws QBadRequestException + { + Object value = body.get(key); + if(value != null && !(value instanceof String)) + { + throw (new QBadRequestException(key + " must be a string.")); + } + return ((String) value); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/LookupSpec.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/LookupSpec.java new file mode 100644 index 0000000..4d7b323 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/LookupSpec.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.api; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation; +import com.kingsrook.qqq.openapi.model.HttpMethod; +import com.kingsrook.qqq.openapi.model.In; +import com.kingsrook.qqq.openapi.model.Parameter; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + + +/******************************************************************************* + ** Database Performance lookup under the authenticated UI API. + *******************************************************************************/ +public class LookupSpec extends DbPerformanceJsonSpec +{ + + + + /******************************************************************************* + ** Method and path relative to /qqq/v1. + *******************************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/db-performance/statements/lookup") + .withShortSummary("Database Performance lookup").withLongDescription("Requires db_performance.view and uses the caller's session.")); + } + + + + /******************************************************************************* + ** Versioned response shape. + *******************************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("DbPerformanceLookupV1"); + } + + + + /******************************************************************************* + ** Execute in the caller's context; also directly testable without transport. + *******************************************************************************/ + @Override + public Object handle(JsonInput input) throws Exception + { + return (data().lookup(input.getQueryParam("backend"), input.getQueryParam("sql"))); + } + + + + /******************************************************************************* + ** Lookup requires a SQL shape in addition to the optional backend. + *******************************************************************************/ + @Override + public List defineRequestParameters() + { + List parameters = new ArrayList<>(super.defineRequestParameters()); + parameters.add(new Parameter().withName("sql").withIn(In.QUERY).withRequired(true).withSchema(new Schema().withType(Type.STRING))); + return (parameters); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/api/OverviewSpec.java b/src/main/java/com/kingsrook/qbits/dbperformance/api/OverviewSpec.java new file mode 100644 index 0000000..00020e9 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/api/OverviewSpec.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.api; + + +import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation; +import com.kingsrook.qqq.openapi.model.HttpMethod; + + +/******************************************************************************* + ** Database Performance overview under the authenticated UI API. + *******************************************************************************/ +public class OverviewSpec extends DbPerformanceJsonSpec +{ + + + + /******************************************************************************* + ** Method and path relative to /qqq/v1. + *******************************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/db-performance/overview") + .withShortSummary("Database Performance overview").withLongDescription("Requires db_performance.view and uses the caller's session.")); + } + + + + /******************************************************************************* + ** Versioned response shape. + *******************************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("DbPerformanceOverviewV1"); + } + + + + /******************************************************************************* + ** Execute in the caller's context; also directly testable without transport. + *******************************************************************************/ + @Override + public Object handle(JsonInput input) throws Exception + { + return (data().overview(input.getQueryParam("backend"))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainRequest.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainRequest.java new file mode 100644 index 0000000..df0bb57 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainRequest.java @@ -0,0 +1,42 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.explain; + + +import java.util.List; + + +/******************************************************************************* + ** Transient explain input. Parameters must never be persisted or logged. + *******************************************************************************/ +public record ExplainRequest(String backendName, Integer statementId, String sql, Mode mode, List parameters) +{ + + + + /******************************************************************************* + ** Requested execution mode; GENERIC is selected automatically for PLAN. + *******************************************************************************/ + public enum Mode + { + PLAN, ANALYZE + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainResult.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainResult.java new file mode 100644 index 0000000..567f8d1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainResult.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.explain; + + +import java.math.BigDecimal; +import java.util.List; + + +/******************************************************************************* + ** Transient response; ANALYZE plans are returned only to the authorized caller. + *******************************************************************************/ +public record ExplainResult(String mode, String planJson, String planText, BigDecimal totalCost, BigDecimal actualTotalMs, + List warnings, Integer explainId, String sqlWorkbenchUrl, String assistantPrompt) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java new file mode 100644 index 0000000..820e017 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainService.java @@ -0,0 +1,312 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.explain; + + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import com.kingsrook.qbits.dbperformance.DbPerfRuntime; +import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; +import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.postgresql.core.QueryExecutor; +import org.postgresql.jdbc.PgStatement; + + +/******************************************************************************* + ** Audited, read-only plan execution on a dedicated monitored-backend connection. + ** Neither database error messages nor supplied values enter the persistent audit. + *******************************************************************************/ +public class ExplainService +{ + private final DbPerfRuntime runtime; + + + + /******************************************************************************* + ** Use the registered host runtime. + *******************************************************************************/ + public ExplainService(DbPerfRuntime runtime) + { + this.runtime = runtime; + } + + + + /******************************************************************************* + ** Audit every request, including validation and authorization failures. + *******************************************************************************/ + public ExplainResult explain(ExplainRequest request) throws Exception + { + long started = System.nanoTime(); + QSession session = QContext.getQSession(); + QRecord audit = new QRecord().withValue("requestedAt", Instant.now()).withValue("outcome", "ERROR") + .withValue("requestedBy", session == null || session.getUser() == null ? null : session.getUser().getIdReference()) + .withValue("mode", request != null && request.mode() == ExplainRequest.Mode.ANALYZE ? "ANALYZE" : "PLAN") + .withValue("parameterCount", request == null || request.parameters() == null ? 0 : request.parameters().size()); + String sql = request == null ? "" : request.sql(); + String json = null; + PlanTextRenderer.Plan plan = null; + QException failure = null; + List warnings = List.of(); + try + { + String permission = runtime.getConfig().getPermissionBaseName() + ("ANALYZE".equals(audit.getValueString("mode")) ? ".explain_analyze" : ".explain"); + if(session == null || !session.hasPermission(permission)) + { + throw (new QPermissionDeniedException("Query plan permission required: " + permission)); + } + if(request == null || request.mode() == null) + { + throw (new QBadRequestException("Choose PLAN or ANALYZE.")); + } + if(!runtime.getConfig().getBackendName().equals(request.backendName())) + { + throw (new QBadRequestException("Choose the configured monitored backend.")); + } + if((request.statementId() == null) == (sql == null || sql.isBlank())) + { + throw (new QBadRequestException("Supply either a statementId or SQL, not both.")); + } + boolean storedStatement = request.statementId() != null; + if(storedStatement) + { + QRecord row = runtime.asSystem(() -> DbPerfStore.first("db_perf_statement", DbPerfStore.eq("id", request.statementId()))); + if(row == null || !request.backendName().equals(row.getValueString("backendName"))) + { + throw (new QBadRequestException("The statement was not found on the monitored backend.")); + } + audit.setValue("statementId", request.statementId()); + sql = row.getValueString("queryText"); + } + ExplainSql.Parsed parsed = ExplainSql.parse(sql); + List parameters = request.parameters() == null ? List.of() : request.parameters(); + audit.setValue("parameterCount", parsed.parameterCount()); + boolean analyze = request.mode() == ExplainRequest.Mode.ANALYZE; + if(analyze) + { + parsed.requireSelect(); + if(parameters.size() != parsed.parameterCount()) + { + throw (new QBadRequestException("Supply exactly " + parsed.parameterCount() + " parameter values for ANALYZE.")); + } + } + else if(!parameters.isEmpty()) + { + throw (new QBadRequestException("Parameter values are accepted only for ANALYZE.")); + } + boolean generic = !analyze && parsed.parameterCount() > 0; + audit.setValue("mode", analyze ? "ANALYZE" : generic ? "GENERIC" : "PLAN"); + QRecord settings = new DbPerfSettingsService(runtime).ensure(request.backendName()); + json = execute(parsed.sql(), parameters, analyze, generic, SnapshotCollector.positive(settings, "explainTimeoutMs", 10000)); + plan = PlanTextRenderer.render(json); + if(!analyze && (storedStatement || sql.equals(SqlNormalizer.normalize(sql)))) + { + audit.setValue("planJson", json); + audit.setValue("planText", plan.text()); + audit.setValue("totalCost", plan.totalCost()); + } + else + { + warnings = List.of("This plan is transient; only a literal-free request audit is stored."); + } + audit.setValue("outcome", "OK"); + } + catch(QPermissionDeniedException e) + { + audit.setValue("outcome", "DENIED"); + failure = e; + } + catch(QBadRequestException e) + { + failure = e; + } + catch(Exception e) + { + boolean timeout = isTimeout(e); + audit.setValue("outcome", timeout ? "TIMEOUT" : "ERROR"); + failure = new QException(timeout ? "Query plan timed out. Reduce the query scope or ask an administrator to review the explain timeout." + : "Query plan failed. Check SQL, parameter types and database access. Writes are not allowed."); + } + finally + { + String normalized = SqlNormalizer.normalize(sql); + audit.setValue("sqlText", normalized); + audit.setValue("sqlFingerprint", SqlNormalizer.fingerprint(normalized)); + audit.setValue("durationMs", (System.nanoTime() - started) / 1000000); + audit.setValue("errorMessage", failure == null ? null : failure.getMessage()); + QRecord inserted = runtime.asSystem(() -> DbPerfStore.insert("db_perf_explain", List.of(audit)).getFirst()); + audit.setValue("id", inserted.getValueInteger("id")); + } + if(failure != null) + { + throw (failure); + } + return (new ExplainResult(audit.getValueString("mode"), json, plan.text(), plan.totalCost(), plan.actualTotalMs(), warnings, + audit.getValueInteger("id"), runtime.getConfig().getSqlWorkbenchRunnerPath() + "?sql=" + encode("EXPLAIN " + sql) + "&backend=" + encode(request.backendName()), + "Explain this PostgreSQL query plan and suggest improvements.\n\nQuery:\n" + SqlNormalizer.normalize(sql) + "\n\nPlan:\n" + plan.safeText())); + } + + + + /******************************************************************************* + ** Roll back even on parse, execution or timeout failure, deallocate after rollback, + ** and restore pool state before releasing this dedicated connection. + *******************************************************************************/ + private String execute(String sql, List parameters, boolean analyze, boolean generic, int timeout) throws Exception + { + try(Connection connection = openConnection()) + { + if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName())) + { + throw (new QBadRequestException("Query plans currently require PostgreSQL.")); + } + if(generic && connection.getMetaData().getDatabaseMajorVersion() < 16) + { + throw (new QBadRequestException("Generic plans require PostgreSQL 16 or later. Use ANALYZE with parameter values if authorized.")); + } + boolean autoCommit = connection.getAutoCommit(); + boolean readOnly = connection.isReadOnly(); + String preparedName = "dbperf_" + UUID.randomUUID().toString().replace("-", ""); + boolean prepared = false; + try + { + connection.setAutoCommit(false); + try(Statement statement = connection.createStatement()) + { + statement.execute("SET TRANSACTION READ ONLY"); + statement.execute("SET LOCAL statement_timeout = " + timeout); + statement.execute("SET LOCAL standard_conforming_strings = on"); + if(!analyze) + { + // GENERIC_PLAN must not be parsed as an external JDBC bind request. + statement.unwrap(PgStatement.class).executeWithFlags("EXPLAIN (" + (generic ? "GENERIC_PLAN, " : "") + "FORMAT JSON) " + sql, QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); + try(ResultSet rows = statement.getResultSet()) + { + rows.next(); + return (rows.getString(1)); + } + } + statement.execute("PREPARE " + preparedName + " AS " + sql); + prepared = true; + } + String arguments = parameters.isEmpty() ? "" : "(" + String.join(", ", Collections.nCopies(parameters.size(), "?")) + ")"; + try(PreparedStatement statement = connection.prepareStatement("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE " + preparedName + arguments)) + { + for(int i = 0; i < parameters.size(); i++) + { + statement.setObject(i + 1, parameters.get(i), Types.OTHER); + } + // EXECUTE's arguments are utility expressions: unknown bind types cannot + // be inferred in extended Parse. Keep JDBC binding/escaping, but send this + // one command in simple mode so PostgreSQL coerces the quoted strings. + statement.unwrap(PgStatement.class).executeWithFlags(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); + try(ResultSet rows = statement.getResultSet()) + { + rows.next(); + return (rows.getString(1)); + } + } + } + finally + { + try + { + connection.rollback(); + if(prepared) + { + try(Statement cleanup = connection.createStatement()) + { + cleanup.execute("DEALLOCATE " + preparedName); + } + finally + { + connection.rollback(); + } + } + } + finally + { + connection.setReadOnly(readOnly); + connection.setAutoCommit(autoCommit); + } + } + } + } + + + + /******************************************************************************* + ** Always obtain a dedicated connection, independent of any caller QQQ transaction. + ** Protected for contract tests that verify pool state restoration on driver errors. + *******************************************************************************/ + protected Connection openConnection() throws SQLException + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName()); + return (ConnectionManager.getConnection(backend)); + } + + + + /******************************************************************************* + ** SQLSTATE survives JDBC wrapper exceptions without exposing their messages. + *******************************************************************************/ + private static boolean isTimeout(Throwable exception) + { + for(Throwable current = exception; current != null; current = current.getCause()) + { + if(current instanceof SQLException sql && "57014".equals(sql.getSQLState())) + { + return (true); + } + } + return (false); + } + + + + /******************************************************************************* + ** UTF-8 query parameter encoding. + *******************************************************************************/ + private static String encode(String value) + { + return (URLEncoder.encode(value, StandardCharsets.UTF_8)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java new file mode 100644 index 0000000..dbe12f6 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/ExplainSql.java @@ -0,0 +1,257 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.explain; + + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; + + +/******************************************************************************* + ** Lex executable SQL without changing literals, identifiers or parameter positions. + ** This is a conservative SELECT guard; PostgreSQL READ ONLY is the write boundary. + *******************************************************************************/ +public final class ExplainSql +{ + private static final Pattern DOLLAR_QUOTE = Pattern.compile("\\$(?:[A-Za-z_][A-Za-z_0-9]*)?\\$"); + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private ExplainSql() + { + } + + + + /******************************************************************************* + ** Executable SQL with contiguous PostgreSQL bind slots and unquoted tokens. + *******************************************************************************/ + public record Parsed(String sql, List tokens, int parameterCount) + { + /******************************************************************************* + ** Reject writes, SELECT INTO and every locking SELECT, including nested CTEs. + *******************************************************************************/ + public void requireSelect() throws QBadRequestException + { + if(tokens.isEmpty() || !Set.of("SELECT", "WITH").contains(tokens.getFirst()) + || tokens.stream().anyMatch(Set.of("INSERT", "UPDATE", "DELETE", "MERGE", "INTO", "SHARE")::contains)) + { + throw (new QBadRequestException("ANALYZE requires a single SELECT without writes, INTO or row locks.")); + } + } + } + + + + /******************************************************************************* + ** Skip SQL comments and quoted regions, reject multiple statements, and convert ?. + ** Mixing JDBC and PostgreSQL parameter styles is rejected as ambiguous. + *******************************************************************************/ + public static Parsed parse(String sql) throws QBadRequestException + { + if(sql == null || sql.isBlank() || sql.length() > 100000) + { + throw (new QBadRequestException("Supply one SQL statement of at most 100000 characters.")); + } + StringBuilder result = new StringBuilder(); + List tokens = new ArrayList<>(); + Set positions = new HashSet<>(); + int jdbc = 0; + int i = 0; + boolean ended = false; + while(i < sql.length()) + { + int start = i; + char c = sql.charAt(i); + if(Character.isWhitespace(c)) + { + result.append(c); + i++; + continue; + } + if(sql.startsWith("--", i)) + { + int end = sql.indexOf('\n', i); + i = end < 0 ? sql.length() : end; + result.append(' '); + continue; + } + if(sql.startsWith("/*", i)) + { + int depth = 1; + i += 2; + while(i < sql.length() && depth > 0) + { + if(sql.startsWith("/*", i)) + { + depth++; + i += 2; + } + else if(sql.startsWith("*/", i)) + { + depth--; + i += 2; + } + else + { + i++; + } + } + if(depth != 0) + { + throw (new QBadRequestException("Unterminated SQL comment.")); + } + result.append(' '); + continue; + } + if(ended) + { + throw (new QBadRequestException("Only one SQL statement is allowed.")); + } + if(c == ';') + { + ended = true; + i++; + continue; + } + boolean escaped = (c == 'e' || c == 'E') && i + 1 < sql.length() && sql.charAt(i + 1) == '\''; + if(c == '\'' || c == '"' || escaped) + { + char quote = escaped ? '\'' : c; + i = quotedEnd(sql, i + (escaped ? 1 : 0), quote, escaped); + result.append(sql, start, i); + tokens.add(""); + continue; + } + if(c == '$') + { + Matcher delimiter = DOLLAR_QUOTE.matcher(sql).region(i, sql.length()); + if(delimiter.lookingAt()) + { + int end = sql.indexOf(delimiter.group(), delimiter.end()); + if(end < 0) + { + throw (new QBadRequestException("Unterminated dollar-quoted SQL string.")); + } + i = end + delimiter.group().length(); + result.append(sql, start, i); + tokens.add(""); + continue; + } + i++; + while(i < sql.length() && Character.isDigit(sql.charAt(i))) + { + i++; + } + if(i > start + 1) + { + try + { + int position = Integer.parseInt(sql.substring(start + 1, i)); + if(position < 1 || position > 65535) + { + throw (new NumberFormatException()); + } + positions.add(position); + } + catch(NumberFormatException e) + { + throw (new QBadRequestException("SQL parameter positions must be between 1 and 65535.")); + } + } + result.append(sql, start, i); + continue; + } + if(c == '?') + { + // PostgreSQL's multi-character JSON operators are not JDBC bind slots. + if(i + 1 < sql.length() && "|&?".indexOf(sql.charAt(i + 1)) >= 0) + { + result.append(sql, i, i + 2); + i += 2; + } + else + { + result.append('$').append(++jdbc); + i++; + } + continue; + } + if(Character.isLetter(c) || c == '_') + { + i++; + while(i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || "_$".indexOf(sql.charAt(i)) >= 0)) + { + i++; + } + tokens.add(sql.substring(start, i).toUpperCase(Locale.ROOT)); + result.append(sql, start, i); + continue; + } + result.append(c); + i++; + } + int maximum = positions.stream().mapToInt(Integer::intValue).max().orElse(0); + if(jdbc > 0 && maximum > 0 || maximum != positions.size()) + { + throw (new QBadRequestException("Use either ? parameters or contiguous $1 through $n parameters.")); + } + return (new Parsed(result.toString().trim(), List.copyOf(tokens), Math.max(jdbc, maximum))); + } + + + + /******************************************************************************* + ** A quoted region cannot introduce executable keywords or statement delimiters. + *******************************************************************************/ + private static int quotedEnd(String sql, int start, char quote, boolean escaped) throws QBadRequestException + { + int i = start + 1; + while(i < sql.length()) + { + if(escaped && sql.charAt(i) == '\\') + { + i += 2; + } + else if(sql.charAt(i++) == quote) + { + if(i < sql.length() && sql.charAt(i) == quote) + { + i++; + } + else + { + return (i); + } + } + } + throw (new QBadRequestException("Unterminated SQL quote.")); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java new file mode 100644 index 0000000..bd94a3f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/PlanTextRenderer.java @@ -0,0 +1,102 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.explain; + + +import java.math.BigDecimal; +import java.util.List; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer; + + +/******************************************************************************* + ** Readable PostgreSQL JSON plan tree, with a literal-free prompt variant. + *******************************************************************************/ +public final class PlanTextRenderer +{ + private static final ObjectMapper MAPPER = new ObjectMapper(); + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private PlanTextRenderer() + { + } + + + + /******************************************************************************* + ** A parsed plan and its top-level measures. + *******************************************************************************/ + public record Plan(String text, String safeText, BigDecimal totalCost, BigDecimal actualTotalMs) + { + } + + + + /******************************************************************************* + ** PostgreSQL emits an array containing the plan and optional execution timing. + *******************************************************************************/ + public static Plan render(String json) throws Exception + { + JsonNode document = MAPPER.readTree(json); + JsonNode root = document.isArray() ? document.path(0) : document; + JsonNode plan = root.path("Plan"); + if(!plan.isObject()) + { + throw (new IllegalArgumentException("PostgreSQL did not return a plan tree.")); + } + StringBuilder text = new StringBuilder(); + StringBuilder safe = new StringBuilder(); + append(plan, 0, text, false); + append(plan, 0, safe, true); + JsonNode actual = root.has("Execution Time") ? root.get("Execution Time") : plan.get("Actual Total Time"); + return (new Plan(text.toString().stripTrailing(), safe.toString().stripTrailing(), + plan.has("Total Cost") ? plan.get("Total Cost").decimalValue() : null, actual == null ? null : actual.decimalValue())); + } + + + + /******************************************************************************* + ** Preserve tree structure and planner metrics; redact expression literals for AI. + *******************************************************************************/ + private static void append(JsonNode node, int depth, StringBuilder out, boolean redact) + { + out.append(" ".repeat(depth)).append(node.path("Node Type").asText("Unknown")); + for(String field : List.of("Relation Name", "Index Name", "Startup Cost", "Total Cost", "Plan Rows", "Actual Total Time", "Actual Rows", "Filter", "Index Cond", "Hash Cond", "Join Filter", "Recheck Cond")) + { + if(node.has(field)) + { + String value = node.get(field).asText(); + boolean expression = field.endsWith("Cond") || field.endsWith("Filter"); + out.append(" | ").append(field).append(": ").append(redact && expression ? SqlNormalizer.normalize(value) : value); + } + } + out.append('\n'); + for(JsonNode child : node.path("Plans")) + { + append(child, depth + 1, out, redact); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/explain/QueryPlanProcess.java b/src/main/java/com/kingsrook/qbits/dbperformance/explain/QueryPlanProcess.java new file mode 100644 index 0000000..51c90d7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/explain/QueryPlanProcess.java @@ -0,0 +1,144 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.explain; + + +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qbits.dbperformance.DbPerfPermissions; +import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig; +import com.kingsrook.qbits.dbperformance.DbPerfRuntime; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; + + +/******************************************************************************* + ** The one-record Query plan action; user-supplied values remain transient. + *******************************************************************************/ +public class QueryPlanProcess implements BackendStep +{ + public static final String NAME = "dbPerfQueryPlan"; + + + + /******************************************************************************* + ** Register form, execution and copyable result fields, without a navigation item. + *******************************************************************************/ + public static void register(QInstance instance, DbPerfQBitConfig config) + { + instance.addPossibleValueSource(new QPossibleValueSource().withName("db_perf_ExplainRequestMode").withType(QPossibleValueSourceType.ENUM) + .withEnumValues(List.of(new QPossibleValue<>("PLAN", "PLAN"), new QPossibleValue<>("ANALYZE", "ANALYZE")))); + instance.addProcess(new QProcessMetaData().withName(NAME).withLabel("Query plan").withTableName("db_perf_statement") + .withMinInputRecords(1).withMaxInputRecords(1) + .withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".explain")) + .withStep(new QFrontendStepMetaData().withName("review").withLabel("Review") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormFields(List.of(new QFieldMetaData("mode", QFieldType.STRING).withLabel("Mode").withPossibleValueSourceName("db_perf_ExplainRequestMode").withDefaultValue("PLAN").withIsRequired(true), + new QFieldMetaData("parameterValues", QFieldType.TEXT).withLabel("Parameter values (ANALYZE only, one per line)").withIsSensitive(true)))) + .withStep(new QBackendStepMetaData().withName("explain").withCode(new QCodeReference(QueryPlanProcess.class)) + .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName("db_perf_statement")))) + .withStep(new QFrontendStepMetaData().withName("result").withLabel("Result") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withViewField(code("planText", "Query plan")) + .withViewField(new QFieldMetaData("totalCost", QFieldType.DECIMAL).withLabel("Total cost")) + .withViewField(new QFieldMetaData("sqlWorkbenchUrl", QFieldType.STRING).withLabel("Open in SQL Workbench") + .withFieldAdornment(new FieldAdornment(AdornmentType.LINK))) + .withViewField(code("assistantPrompt", "Copy to assistant")))); + } + + + + /******************************************************************************* + ** Monospace text that remains selectable and copyable. + *******************************************************************************/ + private static QFieldMetaData code(String name, String label) + { + return (new QFieldMetaData(name, QFieldType.TEXT).withLabel(label) + .withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("text")))); + } + + + + /******************************************************************************* + ** Re-check permissions inside the service even when invoked without UI middleware. + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + try + { + if(input.getRecords() == null || input.getRecords().size() != 1) + { + throw (new QException("Select exactly one statement.")); + } + DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance()); + String values = input.getValueString("parameterValues"); + List parameters = values == null || values.isEmpty() ? List.of() : Arrays.asList(values.split("\\R", -1)); + ExplainRequest.Mode mode; + try + { + mode = ExplainRequest.Mode.valueOf(input.getValueString("mode") == null ? "PLAN" : input.getValueString("mode")); + } + catch(IllegalArgumentException e) + { + mode = null; + } + ExplainResult result = new ExplainService(runtime).explain(new ExplainRequest(runtime.getConfig().getBackendName(), + input.getRecords().getFirst().getValueInteger("id"), null, mode, parameters)); + output.addValue("planText", result.planText()); + output.addValue("totalCost", result.totalCost()); + output.addValue("sqlWorkbenchUrl", result.sqlWorkbenchUrl()); + output.addValue("assistantPrompt", result.assistantPrompt()); + } + catch(QException e) + { + throw (e); + } + catch(Exception e) + { + throw (new QException("Unable to complete the query plan request.")); + } + finally + { + input.getValues().remove("parameterValues"); + output.getValues().remove("parameterValues"); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfHelp.java b/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfHelp.java new file mode 100644 index 0000000..e933007 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/metadata/DbPerfHelp.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.metadata; + + +import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; +import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundle; +import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundleEntry; +import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles; + + +/******************************************************************************* + ** Data-driven app/table help, also visible when the host has no help-content tables. + *******************************************************************************/ +public final class DbPerfHelp +{ + public static final String RESOURCE = "help/qbit-db-performance.yaml"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private DbPerfHelp() + { + } + + + + /******************************************************************************* + ** Register the Maven-versioned bundle and fill only unclaimed help slots. + *******************************************************************************/ + public static void register(QInstance instance, DbPerfQBitConfig config) + { + HelpBundles.register(instance, RESOURCE); + HelpBundle bundle = HelpBundles.of(instance).getBundles().getLast(); + for(HelpBundleEntry entry : bundle.getEntries()) + { + String key = entry.getKey(); + if(key.startsWith("app:dbPerformance;")) + { + entry.setKey(key.replace("app:dbPerformance;", "app:" + config.getAppName() + ";")); + } + if(key.startsWith("table:") && key.contains(";slot:")) + { + String name = key.substring(6, key.indexOf(';')); + String slot = key.substring(key.indexOf(";slot:") + 6); + var table = instance.getTable(name); + if(table != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(slot))) + { + table.withHelpContent(slot, content(entry)); + } + } + else if(key.equals("app:dbPerformance;slot:help")) + { + var app = instance.getApp(config.getAppName()); + if(app.getHelpContent() == null || !app.getHelpContent().containsKey("help")) + { + app.withHelpContent("help", content(entry)); + } + } + } + } + + + + /******************************************************************************* + ** Render a concise summary followed by the full data-driven markdown explanation. + *******************************************************************************/ + private static QHelpContent content(HelpBundleEntry entry) + { + return (new QHelpContent().withContentAsMarkdown(entry.getSummary() + "\n\n" + (entry.getContent() == null ? "" : entry.getContent()))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfTimeChart.java b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfTimeChart.java new file mode 100644 index 0000000..543d2f4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfTimeChart.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.widgets; + + +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; + + +/******************************************************************************* + ** Snapshot time series rendered by the standard typed line-chart component. + *******************************************************************************/ +public class DbPerfTimeChart extends ChartData +{ + + + + /******************************************************************************* + ** Match the registered metadata type. + *******************************************************************************/ + @Override + public String getType() + { + return (WidgetType.LINE_CHART.getType()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java new file mode 100644 index 0000000..1059e3e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgetRenderer.java @@ -0,0 +1,363 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.widgets; + + +import java.sql.Connection; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qbits.dbperformance.DbPerfRuntime; +import com.kingsrook.qbits.dbperformance.api.DbPerformanceData; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.collect.SnapshotCollector; +import com.kingsrook.qbits.dbperformance.provider.ConnectionCount; +import com.kingsrook.qbits.dbperformance.provider.DbStatsProviders; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ParentWidgetData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockLink; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.fields; +import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.map; +import static com.kingsrook.qbits.dbperformance.api.DbPerformanceData.number; + + +/******************************************************************************* + ** Typed data only: paginated tables, full-width panels and explicit empty guidance. + *******************************************************************************/ +public class DbPerfWidgetRenderer extends AbstractWidgetRenderer +{ + + + + /******************************************************************************* + ** No privileged read context: both widget and service enforce the caller's view grant. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + DbPerfRuntime runtime = DbPerfRuntime.of(QContext.getQInstance()); + DbPerformanceData data = new DbPerformanceData(runtime); + data.requireView(null); + try + { + QWidgetData result = switch(input.getWidgetMetaData().getName()) + { + case "dbPerfSetup" -> setup(runtime, data); + case "dbPerfDatabaseTime" -> databaseTime(data); + case "dbPerfTopQueries" -> topQueries(data); + case "dbPerfFindings" -> findings(data); + case "dbPerfLiveActivity" -> liveActivity(data); + case "dbPerfConnections" -> connections(); + case "dbPerfConnectionStatistics" -> connectionStatistics(data); + case "dbPerfConnectionsByApplication" -> connectionsByApplication(runtime); + case "dbPerfAttribution" -> attribution(); + case "dbPerfTables" -> tables(data); + default -> throw new QException("Unknown Database Performance widget."); + }; + return (new RenderWidgetOutput(result)); + } + catch(QException e) + { + throw (e); + } + catch(Exception e) + { + throw (new QException("Database Performance panel unavailable. Check the backend and statistics access.")); + } + } + + + + /******************************************************************************* + ** Guided checks do not write settings or require a first successful snapshot. + *******************************************************************************/ + public TableData setup(DbPerfRuntime runtime, DbPerformanceData data) throws Exception + { + QRecord settings = data.settings(); + String access = settings == null ? "NOT_COLLECTED" : settings.getValueString("statsAccess"); + Instant last = settings == null ? null : settings.getValueInstant("lastSnapshotAt"); + boolean attribution = settings == null ? Boolean.TRUE.equals(runtime.getConfig().getAttributionEnabled()) : Boolean.TRUE.equals(settings.getValueBoolean("attributionEnabled")); + List> rows = new ArrayList<>(); + rows.add(check("Monitored backend", runtime.getConfig().getBackendName(), "Review Settings for this backend.")); + rows.add(check("Provider", settings == null ? "Not detected yet" : settings.getValueString("provider"), "PostgreSQL 13+ is supported.")); + rows.add(check("pg_stat_statements", access, "Preload and create pg_stat_statements; then run Collect now.")); + rows.add(check("pg_read_all_stats", access, settings == null ? "Grant pg_read_all_stats to the application's database role." : settings.getValueString("statsAccessMessage"))); + rows.add(check("Last snapshot", last == null ? "Nothing collected yet" : Duration.between(last, Instant.now()).toSeconds() + " seconds ago", + last == null ? "Run Collect now from Snapshots, or wait for the next scheduled collection." : "Collection normally runs every " + SnapshotCollector.positive(settings, "snapshotIntervalMinutes", 5) + " minutes.")); + rows.add(check("Attribution", attribution ? "On" : "Off", attribution ? "Attribution starts after the next flush." : "Enable attribution in Settings and the host configuration.")); + rows.add(check("Schedules", runtime.getConfig().getSchedulerName() == null ? "Not configured" : "Configured", "Set schedulerName on the qbit and register that scheduler in the host.")); + return (table("Run Collect now to start collecting database performance.", rows, "Check", "State", "Next step")); + } + + + + /******************************************************************************* + ** Plain text check row, never markup. + *******************************************************************************/ + private static Map check(String name, String state, String next) + { + return (map("Check", name, "State", state, "Next step", next)); + } + + + + /******************************************************************************* + ** Separate labeled time and call series from each snapshot's interval deltas. + *******************************************************************************/ + public ChartData databaseTime(DbPerformanceData data) throws QException + { + List snapshots = data.snapshots(); + List labels = snapshots.stream().map(row -> row.getValueString("capturedAt")).toList(); + List milliseconds = snapshots.stream().map(row -> (Number) row.getValue("statementTotalMs")).toList(); + List calls = snapshots.stream().map(row -> (Number) row.getValue("statementCalls")).toList(); + return (new DbPerfTimeChart().withDescription(snapshots.isEmpty() ? "Database time appears after the next snapshot." : "Last 24 hours; each point is one collection interval.") + .withChartData(new ChartData.Data().withLabels(labels).withDatasets(List.of( + new ChartData.Data.Dataset().withLabel("Database time (ms)").withData(milliseconds).withBackgroundColor("#2563eb"), + new ChartData.Data.Dataset().withLabel("Calls").withData(calls).withBackgroundColor("#16a34a"))))); + } + + + + /******************************************************************************* + ** Truncated query shapes link to their record and paginate rather than scroll. + *******************************************************************************/ + public TableData topQueries(DbPerformanceData data) throws QException + { + List> rows = new ArrayList<>(); + for(Map statement : data.statements(100, null)) + { + rows.add(map("Query", link((String) statement.get("queryText"), (String) statement.get("historyPath")), "Calls", statement.get("calls24h"), + "Mean ms", statement.get("meanMs24h"), "Total ms", statement.get("totalMs24h"), "% DB time", statement.get("shareOfDbTime24h"), "Hit ratio", statement.get("hitRatio"))); + } + TableData table = table("Top queries appear after two snapshots establish interval deltas.", rows, "Query", "Calls", "Mean ms", "Total ms", "% DB time", "Hit ratio"); + table.getColumns().getFirst().setType("composite"); + return (table); + } + + + + /******************************************************************************* + ** Show actionable findings from the parallel findings engine, without changing them. + *******************************************************************************/ + public TableData findings(DbPerformanceData data) throws QException + { + List> rows = new ArrayList<>(); + for(QRecord row : data.findings(100)) + { + rows.add(map("Severity", row.getValueString("severity"), "Title", link(row.getValueString("title"), data.recordPath("db_perf_finding", row.getValueInteger("id"))), + "Rule", row.getValueString("ruleCode"), "Last seen", row.getValue("lastSeenAt"))); + } + TableData table = table("Findings appear after the next analysis of collected snapshots.", rows, "Severity", "Title", "Rule", "Last seen"); + table.getColumns().get(1).setType("composite"); + return (table); + } + + + + /******************************************************************************* + ** Even callers with live-text permission see only fingerprinted text on home. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public TableData liveActivity(DbPerformanceData data) throws Exception + { + QRecord settings = data.settings(); + int longRunning = settings == null ? 30 : SnapshotCollector.positive(settings, "activityLongRunningSeconds", 30); + int idle = settings == null ? 60 : SnapshotCollector.positive(settings, "idleInTransactionSeconds", 60); + List> sessions = (List>) data.activity((String) null, false).get("sessions"); + List> rows = sessions.stream().filter(row -> !((List) row.get("blockedBy")).isEmpty() || ((Number) row.get("blockingCount")).intValue() > 0 + || "active".equals(row.get("state")) && numeric(row.get("querySeconds")) >= longRunning + || Objects.toString(row.get("state"), "").startsWith("idle in transaction") && numeric(row.get("xactSeconds")) >= idle).toList(); + return (table("No long-running or blocked sessions right now", rows, "pid", "user", "application", "state", "querySeconds", "xactSeconds", "blockedBy", "blockingCount", "normalizedText")); + } + + + + /******************************************************************************* + ** Capacity and breakdown are stacked, each occupying the whole row. + *******************************************************************************/ + public ParentWidgetData connections() + { + return (new ParentWidgetData().withChildWidgetNameList(List.of("dbPerfConnectionStatistics", "dbPerfConnectionsByApplication"))); + } + + + + /******************************************************************************* + ** Last observed capacity plus summed 24-hour delta metrics, never cumulative totals. + *******************************************************************************/ + public MultiStatisticsData connectionStatistics(DbPerformanceData data) throws QException + { + QRecord latest = data.latest(); + List snapshots = data.snapshots(); + List values = new ArrayList<>(); + values.add(stat("Total connections", latest == null ? null : number(latest, "connectionsTotal"))); + values.add(stat("Maximum connections", latest == null ? null : number(latest, "maxConnections"))); + values.add(stat("Active", latest == null ? null : number(latest, "connectionsActive"))); + values.add(stat("Idle", latest == null ? null : number(latest, "connectionsIdle"))); + values.add(stat("Idle in transaction", latest == null ? null : number(latest, "connectionsIdleInTx"))); + values.add(stat("Cache hit %", latest == null ? null : 100 * number(latest, "cacheHitRatio"))); + values.add(stat("Temp MB (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "tempBytes")).sum() / 1048576)); + values.add(stat("Deadlocks (24h)", snapshots.isEmpty() ? null : snapshots.stream().mapToDouble(row -> number(row, "deadlocks")).sum())); + return (new MultiStatisticsData(null, List.of(new MultiStatisticsData.StatisticsGroupData().withHeader("Connection capacity") + .withSubheader(latest == null ? "Capacity appears after the next snapshot." : "Latest snapshot; percentages and MB rounded to whole numbers.").withStatisticList(values)))); + } + + + + /******************************************************************************* + ** Current connection counts by application, user and state, directly from provider. + *******************************************************************************/ + public TableData connectionsByApplication(DbPerfRuntime runtime) throws Exception + { + List> rows = new ArrayList<>(); + try(Connection connection = ConnectionManager.getConnection((RDBMSBackendMetaData) runtime.getInstance().getBackend(runtime.getConfig().getBackendName()))) + { + for(ConnectionCount count : DbStatsProviders.forConnection(connection).readConnections(connection)) + { + rows.add(map("Application", count.applicationName(), "User", count.dbUser(), "State", count.state(), "Connections", count.connectionCount())); + } + } + return (table("No connection counts available for this provider.", rows, "Application", "User", "State", "Connections")); + } + + + + /******************************************************************************* + ** Aggregate units over 24 hours and show repeated-statement patterns below them. + *******************************************************************************/ + public MultiTableData attribution() throws QException + { + List records = DbPerfStore.query("db_perf_attribution", since("windowEnd")); + Map> grouped = new LinkedHashMap<>(); + Map> tableTimes = new LinkedHashMap<>(); + double total = records.stream().mapToDouble(row -> number(row, "totalMs")).sum(); + for(QRecord row : records) + { + String key = row.getValueString("unitKind") + "\n" + row.getValueString("unitName"); + Map group = grouped.computeIfAbsent(key, ignored -> map("Kind", row.getValueString("unitKind"), "Unit", row.getValueString("unitName"), "Statements", 0.0, "Total ms", 0.0)); + group.put("Statements", numeric(group.get("Statements")) + number(row, "statements")); + group.put("Total ms", numeric(group.get("Total ms")) + number(row, "totalMs")); + tableTimes.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).merge(Objects.toString(row.getValueString("tableName"), "Unknown"), number(row, "totalMs"), Double::sum); + } + grouped.forEach((key, group) -> + { + group.put("Share %", total == 0 ? 0 : 100 * numeric(group.get("Total ms")) / total); + group.put("Top table", tableTimes.get(key).entrySet().stream().max(Map.Entry.comparingByValue()).orElseThrow().getKey()); + }); + List> units = grouped.values().stream().sorted(Comparator.comparingDouble((Map row) -> numeric(row.get("Total ms"))).reversed()).toList(); + List> repeats = DbPerfStore.query("db_perf_repeat_pattern", since("windowEnd").withOrderBy(new QFilterOrderBy("maxRepeats", false)).withLimit(100)).stream() + .map(row -> fields(row, "unitKind", "unitName", "fingerprint", "unitsAffected", "maxRepeats", "totalRepeats")).toList(); + return (new MultiTableData(List.of(table("Attribution starts after the next flush", units, "Kind", "Unit", "Statements", "Total ms", "Share %", "Top table"), + table("Repeated statements appear after the next attribution flush when the repetition threshold is reached.", repeats, "unitKind", "unitName", "fingerprint", "unitsAffected", "maxRepeats", "totalRepeats")))); + } + + + + /******************************************************************************* + ** Sequential-scan pressure and seven-day size growth are separate ranked tables. + *******************************************************************************/ + public MultiTableData tables(DbPerformanceData data) throws QException + { + List scans = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("seqScanShare", false)).withLimit(100)); + List growing = DbPerfStore.query("db_perf_table_stat", data.backendFilter().withOrderBy(new QFilterOrderBy("growthBytes7d", false)).withLimit(100)); + return (new MultiTableData(List.of(table("Table scans appear after the next table snapshot (normally hourly).", scans.stream().map(row -> fields(row, "schemaName", "tableName", "seqScan", "idxScan", "seqScanShare")).toList(), + "schemaName", "tableName", "seqScan", "idxScan", "seqScanShare"), + table("Growth appears after table history has accumulated.", growing.stream().map(row -> fields(row, "schemaName", "tableName", "totalBytes", "growthBytes7d")).toList(), + "schemaName", "tableName", "totalBytes", "growthBytes7d")))); + } + + + + /******************************************************************************* + ** The core TABLE renderer paginates when fixedHeight is absent. + *******************************************************************************/ + private static TableData table(String empty, List> rows, String... columns) + { + return (new TableData(null, Arrays.stream(columns).map(name -> new TableData.Column("default", name, name, null, "left")).toList(), rows) + .withRowsPerPage(10).withHidePaginationDropdown(false).withNoRowsFoundHTML(empty)); + } + + + + /******************************************************************************* + ** Typed record link, never an HTML fragment. + *******************************************************************************/ + private static CompositeWidgetData link(String text, String path) + { + return (new CompositeWidgetData().withLayout(CompositeWidgetData.Layout.FLEX_COLUMN) + .withBlock(new TextBlockData().withValues(new TextValues(text)).withLink(new BlockLink(path)))); + } + + + + /******************************************************************************* + ** Trailing-day filter for time-window records. + *******************************************************************************/ + private static QQueryFilter since(String field) + { + return (new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, Instant.now().minusSeconds(86400)))); + } + + + + /******************************************************************************* + ** Core multi-statistics values are nullable integers. + *******************************************************************************/ + private static MultiStatisticsData.StatisticsGroupData.Statistic stat(String label, Double value) + { + return (new MultiStatisticsData.StatisticsGroupData.Statistic(label, value == null ? null : (int) Math.min(Integer.MAX_VALUE, Math.round(value)), null)); + } + + + + /******************************************************************************* + ** Numbers in the shared read model retain their numeric type. + *******************************************************************************/ + private static double numeric(Object value) + { + return (value == null ? 0 : ((Number) value).doubleValue()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgets.java b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgets.java new file mode 100644 index 0000000..af8d85f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dbperformance/widgets/DbPerfWidgets.java @@ -0,0 +1,81 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance.widgets; + + +import java.util.List; +import com.kingsrook.qbits.dbperformance.DbPerfPermissions; +import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; + + +/******************************************************************************* + ** Full-width typed home panels, with one panel per row and bounded refresh rates. + *******************************************************************************/ +public final class DbPerfWidgets +{ + public static final List HOME = List.of("dbPerfSetup", "dbPerfDatabaseTime", "dbPerfTopQueries", "dbPerfFindings", + "dbPerfLiveActivity", "dbPerfConnections", "dbPerfAttribution", "dbPerfTables"); + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private DbPerfWidgets() + { + } + + + + /******************************************************************************* + ** Additive registration, preserving phase A record-child widgets and app sections. + *******************************************************************************/ + public static void register(QInstance instance, DbPerfQBitConfig config) + { + add(instance, config, HOME.get(0), "Setup", WidgetType.TABLE, 60); + add(instance, config, HOME.get(1), "Database time", WidgetType.LINE_CHART, 60); + add(instance, config, HOME.get(2), "Top queries", WidgetType.TABLE, 60); + add(instance, config, HOME.get(3), "Findings", WidgetType.TABLE, 60); + add(instance, config, HOME.get(4), "Live activity", WidgetType.TABLE, 30); + add(instance, config, HOME.get(5), "Connections", WidgetType.PARENT_WIDGET, 30); + add(instance, config, "dbPerfConnectionStatistics", "Connection capacity", WidgetType.MULTI_STATISTICS, 30); + add(instance, config, "dbPerfConnectionsByApplication", "Connections by application and user", WidgetType.TABLE, 30); + add(instance, config, HOME.get(6), "QQQ attribution", WidgetType.MULTI_TABLE, 60); + add(instance, config, HOME.get(7), "Tables", WidgetType.MULTI_TABLE, 60); + instance.getApp(config.getAppName()).withWidgets(HOME); + } + + + + /******************************************************************************* + ** Permission and responsive width also apply when a widget is requested directly. + *******************************************************************************/ + private static void add(QInstance instance, DbPerfQBitConfig config, String name, String label, WidgetType type, int refresh) + { + instance.addWidget(new QWidgetMetaData().withName(name).withLabel(label).withType(type.getType()).withGridColumns(12) + .withCodeReference(new QCodeReference(DbPerfWidgetRenderer.class)).withDefaultValue("refreshSeconds", refresh) + .withPermissionRules(DbPerfPermissions.rules(config.getPermissionBaseName() + ".view"))); + } +} diff --git a/src/main/resources/help/project-version.txt b/src/main/resources/help/project-version.txt new file mode 100644 index 0000000..ad96e7c --- /dev/null +++ b/src/main/resources/help/project-version.txt @@ -0,0 +1 @@ +${project.version} diff --git a/src/main/resources/help/qbit-db-performance.yaml b/src/main/resources/help/qbit-db-performance.yaml new file mode 100644 index 0000000..72273f2 --- /dev/null +++ b/src/main/resources/help/qbit-db-performance.yaml @@ -0,0 +1,946 @@ +bundle: qbit-db-performance +version: ${project.version} +layer: QBIT +entries: +- key: app:dbPerformance;slot:help + summary: "Find expensive queries, inspect query plans and review database capacity." + content: | + Start with Setup. Run {{label:dbPerfCollectNow}} from Snapshots to establish the first baseline. Two collections produce useful statement deltas. Table and index data normally refresh hourly; the dashboard refreshes every 30 to 60 seconds. + + Use {{label:dbPerfQueryPlan}} on a query to inspect its plan. PLAN does not execute the query; generic plans require PostgreSQL 16+. ANALYZE executes a read-only SELECT with a timeout and always rolls back. Parameters and ANALYZE plans are never stored. Raw live SQL is permission-gated and never persisted. + + All data lives in standard QQQ tables. The host owns scheduler registration and permissions. Attribution and findings require their corresponding runtime collectors; empty panels explain when data will arrive. +- key: table:db_perf_activity_sample;slot:help + summary: "Notable session samples with normalized text, never raw query text." + content: | + The collector samples long-running, blocked and idle-in-transaction sessions every collection. Quiet databases can remain empty. Retained for detailRetentionDays (30). Use Live activity for current sessions. +- key: table:db_perf_activity_sample;slot:emptyState + summary: "Notable session samples with normalized text, never raw query text." + content: | + The collector samples long-running, blocked and idle-in-transaction sessions every collection. Quiet databases can remain empty. Retained for detailRetentionDays (30). Use Live activity for current sessions. +- key: table:db_perf_activity_sample;field:snapshotId + summary: "Reference to the related snapshot record." +- key: table:db_perf_activity_sample;field:capturedAt + summary: "Time this database sample was captured." +- key: table:db_perf_activity_sample;field:pid + summary: "Pid reported for this activity sample record." +- key: table:db_perf_activity_sample;field:dbUser + summary: "Database role associated with this statement, session or connection group." +- key: table:db_perf_activity_sample;field:applicationName + summary: "Client application name reported by PostgreSQL." +- key: table:db_perf_activity_sample;field:clientAddr + summary: "Client address reported by the statistics provider." +- key: table:db_perf_activity_sample;field:state + summary: "State reported for this activity sample record." +- key: table:db_perf_activity_sample;field:waitEventType + summary: "PostgreSQL category of the current wait, such as Lock or IO." +- key: table:db_perf_activity_sample;field:waitEvent + summary: "PostgreSQL event on which this backend is currently waiting." +- key: table:db_perf_activity_sample;field:backendType + summary: "Backend type reported for this activity sample record." +- key: table:db_perf_activity_sample;field:xactStart + summary: "Xact start reported for this activity sample record." +- key: table:db_perf_activity_sample;field:queryStart + summary: "Query start reported for this activity sample record." +- key: table:db_perf_activity_sample;field:stateChange + summary: "State change reported for this activity sample record." +- key: table:db_perf_activity_sample;field:querySeconds + summary: "Age of the current query in seconds." +- key: table:db_perf_activity_sample;field:xactSeconds + summary: "Age of the current transaction in seconds." +- key: table:db_perf_activity_sample;field:blockedBy + summary: "Blocking process identifiers reported by PostgreSQL." +- key: table:db_perf_activity_sample;field:blockingCount + summary: "Number of sessions directly blocked by this session." +- key: table:db_perf_activity_sample;field:reason + summary: "Reason reported for this activity sample record." +- key: table:db_perf_activity_sample;field:queryId + summary: "PostgreSQL query identifier; fingerprint provides cross-source normalized matching." +- key: table:db_perf_activity_sample;field:statementId + summary: "Reference to the related statement record." +- key: table:db_perf_activity_sample;field:queryFingerprint + summary: "Query fingerprint reported for this activity sample record." +- key: table:db_perf_activity_sample;field:queryTextNormalized + summary: "Query text normalized reported for this activity sample record." +- key: table:db_perf_attribution;slot:help + summary: "QQQ request, process and job database time by statement shape." + content: | + The host attribution listener flushes one batch per attributionFlushSeconds (60 by default). Retained for detailRetentionDays (30). Attribution starts after the next flush; enable the listener in the host if empty. +- key: table:db_perf_attribution;slot:emptyState + summary: "QQQ request, process and job database time by statement shape." + content: | + The host attribution listener flushes one batch per attributionFlushSeconds (60 by default). Retained for detailRetentionDays (30). Attribution starts after the next flush; enable the listener in the host if empty. +- key: table:db_perf_attribution;field:windowStart + summary: "Beginning of the attribution aggregation window." +- key: table:db_perf_attribution;field:windowEnd + summary: "End of the attribution aggregation window." +- key: table:db_perf_attribution;field:podName + summary: "Application pod that observed and flushed this attribution window." +- key: table:db_perf_attribution;field:unitKind + summary: "Kind of QQQ unit of work, such as route, process or scheduled job." +- key: table:db_perf_attribution;field:unitName + summary: "Name of the QQQ unit responsible for database work." +- key: table:db_perf_attribution;field:processName + summary: "QQQ process responsible for the attributed work, when known." +- key: table:db_perf_attribution;field:tableName + summary: "Table name reported for this attribution record." +- key: table:db_perf_attribution;field:operation + summary: "Normalized SQL operation, such as SELECT, INSERT, UPDATE or DELETE." +- key: table:db_perf_attribution;field:fingerprint + summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded." +- key: table:db_perf_attribution;field:sqlText + summary: "Normalized SQL shape without literal values. Parameter values are never stored." +- key: table:db_perf_attribution;field:units + summary: "Number of sampled QQQ units contributing to this aggregate." +- key: table:db_perf_attribution;field:statements + summary: "Number of attributed SQL executions during the window." +- key: table:db_perf_attribution;field:totalMs + summary: "Total ms in milliseconds for this measurement." +- key: table:db_perf_attribution;field:maxMs + summary: "Max ms in milliseconds for this measurement." +- key: table:db_perf_attribution;field:rowsCount + summary: "Rows processed during the measured execution window." +- key: table:db_perf_attribution;field:errors + summary: "Number of failed SQL executions observed in the attribution window." +- key: table:db_perf_attribution;field:sampleRate + summary: "Fraction of units sampled in this attribution window." +- key: table:db_perf_attribution_daily;slot:help + summary: "Daily database-time attribution for QQQ units of work." + content: | + The daily rollup aggregates flushed attribution windows at 03:15 by default. Retained for rollupRetentionDays (365). Wait for a flush and the next daily rollup when empty. +- key: table:db_perf_attribution_daily;slot:emptyState + summary: "Daily database-time attribution for QQQ units of work." + content: | + The daily rollup aggregates flushed attribution windows at 03:15 by default. Retained for rollupRetentionDays (365). Wait for a flush and the next daily rollup when empty. +- key: table:db_perf_attribution_daily;field:day + summary: "Calendar day in the configured qbit time zone for this rollup." +- key: table:db_perf_attribution_daily;field:rollupKey + summary: "Stable identity used to replace a daily aggregate idempotently." +- key: table:db_perf_attribution_daily;field:unitKind + summary: "Kind of QQQ unit of work, such as route, process or scheduled job." +- key: table:db_perf_attribution_daily;field:unitName + summary: "Name of the QQQ unit responsible for database work." +- key: table:db_perf_attribution_daily;field:processName + summary: "QQQ process responsible for the attributed work, when known." +- key: table:db_perf_attribution_daily;field:tableName + summary: "Table name reported for this attribution daily record." +- key: table:db_perf_attribution_daily;field:operation + summary: "Normalized SQL operation, such as SELECT, INSERT, UPDATE or DELETE." +- key: table:db_perf_attribution_daily;field:fingerprint + summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded." +- key: table:db_perf_attribution_daily;field:sqlText + summary: "Normalized SQL shape without literal values. Parameter values are never stored." +- key: table:db_perf_attribution_daily;field:units + summary: "Number of sampled QQQ units contributing to this aggregate." +- key: table:db_perf_attribution_daily;field:statements + summary: "Number of attributed SQL executions during the window." +- key: table:db_perf_attribution_daily;field:totalMs + summary: "Total ms in milliseconds for this measurement." +- key: table:db_perf_attribution_daily;field:maxMs + summary: "Max ms in milliseconds for this measurement." +- key: table:db_perf_attribution_daily;field:rowsCount + summary: "Rows processed during the measured execution window." +- key: table:db_perf_attribution_daily;field:errors + summary: "Number of failed SQL executions observed in the attribution window." +- key: table:db_perf_connection_sample;slot:help + summary: "Connection counts grouped by application, user and state for each snapshot." + content: | + The collector writes these rows every snapshot. Retained for detailRetentionDays (30). Run Collect now or wait for the next scheduled pass when empty. +- key: table:db_perf_connection_sample;slot:emptyState + summary: "Connection counts grouped by application, user and state for each snapshot." + content: | + The collector writes these rows every snapshot. Retained for detailRetentionDays (30). Run Collect now or wait for the next scheduled pass when empty. +- key: table:db_perf_connection_sample;field:snapshotId + summary: "Reference to the related snapshot record." +- key: table:db_perf_connection_sample;field:capturedAt + summary: "Time this database sample was captured." +- key: table:db_perf_connection_sample;field:dbUser + summary: "Database role associated with this statement, session or connection group." +- key: table:db_perf_connection_sample;field:applicationName + summary: "Client application name reported by PostgreSQL." +- key: table:db_perf_connection_sample;field:state + summary: "State reported for this connection sample record." +- key: table:db_perf_connection_sample;field:connectionCount + summary: "Connection count reported for this connection sample record." +- key: table:db_perf_explain;slot:help + summary: "Audited plan requests, including denied, failed and timed-out attempts." + content: | + Query plan and the explain UI API write one audit row per service request. Only literal-free PLAN or GENERIC results may be saved; ANALYZE plans and all parameter values remain transient. Retained for detailRetentionDays (30). Run Query plan on a statement when empty. +- key: table:db_perf_explain;slot:emptyState + summary: "Audited plan requests, including denied, failed and timed-out attempts." + content: | + Query plan and the explain UI API write one audit row per service request. Only literal-free PLAN or GENERIC results may be saved; ANALYZE plans and all parameter values remain transient. Retained for detailRetentionDays (30). Run Query plan on a statement when empty. +- key: table:db_perf_explain;field:statementId + summary: "Reference to the related statement record." +- key: table:db_perf_explain;field:requestedBy + summary: "Session user identifier that requested the plan, even for a denied request." +- key: table:db_perf_explain;field:requestedAt + summary: "When the plan request was received." +- key: table:db_perf_explain;field:mode + summary: "PLAN, automatically selected GENERIC, or explicitly authorized ANALYZE." +- key: table:db_perf_explain;field:sqlFingerprint + summary: "SHA-256 of the normalized query in this explain audit." +- key: table:db_perf_explain;field:sqlText + summary: "Normalized SQL shape without literal values. Parameter values are never stored." +- key: table:db_perf_explain;field:parameterCount + summary: "Number of positional parameter slots; values are never stored." +- key: table:db_perf_explain;field:durationMs + summary: "Duration ms in milliseconds for this measurement." +- key: table:db_perf_explain;field:outcome + summary: "OK, ERROR, DENIED or TIMEOUT; every explain service request is audited." +- key: table:db_perf_explain;field:errorMessage + summary: "Sanitized failure description. Database SQL errors and parameter values are excluded." +- key: table:db_perf_explain;field:planJson + summary: "Saved PostgreSQL JSON plan, only for eligible literal-free PLAN or GENERIC requests." +- key: table:db_perf_explain;field:planText + summary: "Readable indented plan tree; never stored for ANALYZE." +- key: table:db_perf_explain;field:totalCost + summary: "Total cost reported for this explain record." +- key: table:db_perf_finding;slot:help + summary: "Ranked, deduplicated suggestions with supporting evidence and proposed fixes." + content: | + The findings engine analyzes collected data hourly. OPEN and ACKNOWLEDGED findings remain actionable; resolved or ignored stale findings are purged after rollupRetentionDays (365). Wait for collection and the next analysis when empty. Proposed migrations require developer review; this qbit never applies them. +- key: table:db_perf_finding;slot:emptyState + summary: "Ranked, deduplicated suggestions with supporting evidence and proposed fixes." + content: | + The findings engine analyzes collected data hourly. OPEN and ACKNOWLEDGED findings remain actionable; resolved or ignored stale findings are purged after rollupRetentionDays (365). Wait for collection and the next analysis when empty. Proposed migrations require developer review; this qbit never applies them. +- key: table:db_perf_finding;field:backendName + summary: "The configured QQQ backend whose database is monitored." +- key: table:db_perf_finding;field:ruleId + summary: "Reference to the related rule record." +- key: table:db_perf_finding;field:ruleCode + summary: "Rule code reported for this finding record." +- key: table:db_perf_finding;field:dedupeKey + summary: "Stable rule-plus-subject identity that updates an existing finding instead of creating duplicates." +- key: table:db_perf_finding;field:status + summary: "Current lifecycle state. Review the status-specific guidance and related messages." +- key: table:db_perf_finding;field:severity + summary: "Configured importance of a finding or rule; contributes to the ranking score." +- key: table:db_perf_finding;field:score + summary: "Severity weight multiplied by normalized impact; larger scores rank first." +- key: table:db_perf_finding;field:title + summary: "Title reported for this finding record." +- key: table:db_perf_finding;field:summary + summary: "Summary reported for this finding record." +- key: table:db_perf_finding;field:evidence + summary: "Measurements supporting the finding, without raw SQL values." +- key: table:db_perf_finding;field:recommendation + summary: "Suggested developer or operator action; the qbit never applies it automatically." +- key: table:db_perf_finding;field:migrationSnippet + summary: "Proposed Liquibase migration text for review and source control; never executed by this qbit." +- key: table:db_perf_finding;field:subjectTable + summary: "Subject table associated with the finding." +- key: table:db_perf_finding;field:statementId + summary: "Reference to the related statement record." +- key: table:db_perf_finding;field:tableStatId + summary: "Reference to the related table stat record." +- key: table:db_perf_finding;field:indexStatId + summary: "Reference to the related index stat record." +- key: table:db_perf_finding;field:unitName + summary: "Name of the QQQ unit responsible for database work." +- key: table:db_perf_finding;field:firstSeenAt + summary: "First recorded observation of this statement or finding." +- key: table:db_perf_finding;field:lastSeenAt + summary: "Most recent observation of this statement, table, index or finding." +- key: table:db_perf_finding;field:occurrences + summary: "Number of analyses in which this finding has been observed." +- key: table:db_perf_finding;field:resolvedAt + summary: "When this finding was marked resolved." +- key: table:db_perf_finding;field:statusNote + summary: "Human explanation for the latest finding status change." +- key: table:db_perf_finding;field:statusChangedBy + summary: "User identifier responsible for the latest manual status change." +- key: table:db_perf_index_stat;slot:help + summary: "Current index definitions, usage, size and uniqueness." + content: | + The collector upserts indexes with the hourly table pass. Stale indexes are purged after rollupRetentionDays (365). Run Collect now to force collection when empty. +- key: table:db_perf_index_stat;slot:emptyState + summary: "Current index definitions, usage, size and uniqueness." + content: | + The collector upserts indexes with the hourly table pass. Stale indexes are purged after rollupRetentionDays (365). Run Collect now to force collection when empty. +- key: table:db_perf_index_stat;field:backendName + summary: "The configured QQQ backend whose database is monitored." +- key: table:db_perf_index_stat;field:tableStatId + summary: "Reference to the related table stat record." +- key: table:db_perf_index_stat;field:schemaName + summary: "Schema name reported for this index stat record." +- key: table:db_perf_index_stat;field:tableName + summary: "Table name reported for this index stat record." +- key: table:db_perf_index_stat;field:indexName + summary: "Index name reported for this index stat record." +- key: table:db_perf_index_stat;field:relid + summary: "Relid reported for this index stat record." +- key: table:db_perf_index_stat;field:definition + summary: "Definition reported for this index stat record." +- key: table:db_perf_index_stat;field:columns + summary: "Columns reported for this index stat record." +- key: table:db_perf_index_stat;field:isUnique + summary: "Whether this index enforces uniqueness." +- key: table:db_perf_index_stat;field:isPrimary + summary: "Whether this index enforces a primary key." +- key: table:db_perf_index_stat;field:isValid + summary: "Whether PostgreSQL considers the index valid for queries." +- key: table:db_perf_index_stat;field:idxScan + summary: "Idx scan reported for this index stat record." +- key: table:db_perf_index_stat;field:idxTupRead + summary: "Idx tup read reported for this index stat record." +- key: table:db_perf_index_stat;field:idxTupFetch + summary: "Idx tup fetch reported for this index stat record." +- key: table:db_perf_index_stat;field:sizeBytes + summary: "Size bytes; size measured in bytes." +- key: table:db_perf_index_stat;field:writeCost + summary: "Write cost reported for this index stat record." +- key: table:db_perf_index_stat;field:scansSinceReset + summary: "Scans since reset reported for this index stat record." +- key: table:db_perf_index_stat;field:lastSeenAt + summary: "Most recent observation of this statement, table, index or finding." +- key: table:db_perf_query_daily;slot:help + summary: "Daily totals from retained statement snapshot details." + content: | + Roll up and purge writes completed days, normally at 03:15 in the configured time zone. Retained for rollupRetentionDays (365). Wait for the next daily run after query snapshots exist. +- key: table:db_perf_query_daily;slot:emptyState + summary: "Daily totals from retained statement snapshot details." + content: | + Roll up and purge writes completed days, normally at 03:15 in the configured time zone. Retained for rollupRetentionDays (365). Wait for the next daily run after query snapshots exist. +- key: table:db_perf_query_daily;field:statementId + summary: "Reference to the related statement record." +- key: table:db_perf_query_daily;field:day + summary: "Calendar day in the configured qbit time zone for this rollup." +- key: table:db_perf_query_daily;field:calls + summary: "Calls reported for this query daily record." +- key: table:db_perf_query_daily;field:totalMs + summary: "Total ms in milliseconds for this measurement." +- key: table:db_perf_query_daily;field:meanMs + summary: "Mean ms in milliseconds for this measurement." +- key: table:db_perf_query_daily;field:maxMs + summary: "Max ms in milliseconds for this measurement." +- key: table:db_perf_query_daily;field:rowsCount + summary: "Rows processed during the measured execution window." +- key: table:db_perf_query_daily;field:hitRatio + summary: "Shared block hits divided by hits plus reads, from 0 to 1; null when no blocks were accessed." +- key: table:db_perf_query_daily;field:shareOfDbTime + summary: "Percentage of all database statement time in this interval, from 0 to 100." +- key: table:db_perf_query_daily;field:snapshots + summary: "Snapshots reported for this query daily record." +- key: table:db_perf_query_snapshot;slot:help + summary: "Interval deltas for the highest-cost statements in each collection." + content: | + The collector writes the top configured number of statement deltas every collection. New or reset counters establish baselines instead. Retained for detailRetentionDays (30). Collect twice when empty. +- key: table:db_perf_query_snapshot;slot:emptyState + summary: "Interval deltas for the highest-cost statements in each collection." + content: | + The collector writes the top configured number of statement deltas every collection. New or reset counters establish baselines instead. Retained for detailRetentionDays (30). Collect twice when empty. +- key: table:db_perf_query_snapshot;field:snapshotId + summary: "Reference to the related snapshot record." +- key: table:db_perf_query_snapshot;field:statementId + summary: "Reference to the related statement record." +- key: table:db_perf_query_snapshot;field:capturedAt + summary: "Time this database sample was captured." +- key: table:db_perf_query_snapshot;field:calls + summary: "Calls reported for this query snapshot record." +- key: table:db_perf_query_snapshot;field:totalMs + summary: "Total ms in milliseconds for this measurement." +- key: table:db_perf_query_snapshot;field:meanMs + summary: "Mean ms in milliseconds for this measurement." +- key: table:db_perf_query_snapshot;field:maxMs + summary: "Max ms in milliseconds for this measurement." +- key: table:db_perf_query_snapshot;field:rowsCount + summary: "Rows processed during the measured execution window." +- key: table:db_perf_query_snapshot;field:sharedBlksHit + summary: "Shared blks hit reported for this query snapshot record." +- key: table:db_perf_query_snapshot;field:sharedBlksRead + summary: "Shared blks read reported for this query snapshot record." +- key: table:db_perf_query_snapshot;field:hitRatio + summary: "Shared block hits divided by hits plus reads, from 0 to 1; null when no blocks were accessed." +- key: table:db_perf_query_snapshot;field:tempBlksWritten + summary: "Temp blks written reported for this query snapshot record." +- key: table:db_perf_query_snapshot;field:shareOfDbTime + summary: "Percentage of all database statement time in this interval, from 0 to 100." +- key: table:db_perf_repeat_pattern;slot:help + summary: "Repeated statement shapes within a QQQ unit of work, indicating possible N+1 queries." + content: | + The attribution listener flushes patterns that reach the configured repetition threshold. Retained for detailRetentionDays (30). Empty is normal when no repeated-statement pattern meets the threshold. +- key: table:db_perf_repeat_pattern;slot:emptyState + summary: "Repeated statement shapes within a QQQ unit of work, indicating possible N+1 queries." + content: | + The attribution listener flushes patterns that reach the configured repetition threshold. Retained for detailRetentionDays (30). Empty is normal when no repeated-statement pattern meets the threshold. +- key: table:db_perf_repeat_pattern;field:windowStart + summary: "Beginning of the attribution aggregation window." +- key: table:db_perf_repeat_pattern;field:windowEnd + summary: "End of the attribution aggregation window." +- key: table:db_perf_repeat_pattern;field:podName + summary: "Application pod that observed and flushed this attribution window." +- key: table:db_perf_repeat_pattern;field:unitKind + summary: "Kind of QQQ unit of work, such as route, process or scheduled job." +- key: table:db_perf_repeat_pattern;field:unitName + summary: "Name of the QQQ unit responsible for database work." +- key: table:db_perf_repeat_pattern;field:processName + summary: "QQQ process responsible for the attributed work, when known." +- key: table:db_perf_repeat_pattern;field:tableName + summary: "Table name reported for this repeat pattern record." +- key: table:db_perf_repeat_pattern;field:fingerprint + summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded." +- key: table:db_perf_repeat_pattern;field:sqlText + summary: "Normalized SQL shape without literal values. Parameter values are never stored." +- key: table:db_perf_repeat_pattern;field:unitsAffected + summary: "Number of sampled units that exceeded the repeated-statement threshold." +- key: table:db_perf_repeat_pattern;field:maxRepeats + summary: "Largest repetition count for one fingerprint in a single sampled unit of work." +- key: table:db_perf_repeat_pattern;field:totalRepeats + summary: "Total repeats reported for this repeat pattern record." +- key: table:db_perf_rule;slot:help + summary: "Editable thresholds and severity for each suggestion rule." + content: | + Seeded insert-only by the Liquibase changelog; existing edits are preserved. Changes affect the next analysis. Rules are retained indefinitely. Apply the changelog when the list is empty. + + Threshold guide (stored labels are authoritative): + - DUPLICATE_INDEX: no numeric thresholds; compares index definitions for identical or prefix-redundant coverage. + - SLOW_QUERY: threshold 1 = Mean execution time (ms) (default 500); threshold 2 = Database time (%) (default 5). + - MISSING_INDEX: threshold 1 = Sequential scan share (%) (default 50); threshold 2 = Tuples read per "day" (default 1000000). + - UNUSED_INDEX: threshold 1 = Days since statistics reset (default 14); threshold 2 = Index size (MB) (default 1). + - N_PLUS_ONE: threshold 1 = Repeats per unit (default 10); threshold 2 = Affected units (default 5). + - HOT_JOB: threshold 1 = Database time (%) (default 10); threshold 2 is unused. + - OVER_AUDITED: threshold 1 = Audit inserts per "day" (default 50000); threshold 2 is unused. + - BLOAT: threshold 1 = Dead tuple ratio (%) (default 20); threshold 2 = Dead tuples (default 10000). + - VACUUM_STALE: threshold 1 = Days since vacuum (default 7); threshold 2 = Transaction freeze age (default 150000000). + - POOL_SIZING: threshold 1 = Connections used (%) (default 80); threshold 2 = Idle connections (%) (default 50). + - IDLE_IN_TRANSACTION: threshold 1 = Idle seconds (default 60); threshold 2 is unused. + - CACHE_HIT_LOW: threshold 1 = Cache hit ratio (%) (default 95); threshold 2 is unused. + - TEMP_SPILL: threshold 1 = Temporary data per "day" (MB) (default 100); threshold 2 is unused. + - DEADLOCKS: threshold 1 = Deadlocks per "day" (default 1); threshold 2 is unused. +- key: table:db_perf_rule;slot:emptyState + summary: "Editable thresholds and severity for each suggestion rule." + content: | + Seeded insert-only by the Liquibase changelog; existing edits are preserved. Changes affect the next analysis. Rules are retained indefinitely. Apply the changelog when the list is empty. +- key: table:db_perf_rule;field:code + summary: "Code reported for this rule record." +- key: table:db_perf_rule;field:label + summary: "Label reported for this rule record." +- key: table:db_perf_rule;field:description + summary: "Description reported for this rule record." +- key: table:db_perf_rule;field:enabled + summary: "Whether collection or this suggestion rule participates in the next scheduled run." +- key: table:db_perf_rule;field:severity + summary: "Configured importance of a finding or rule; contributes to the ranking score." +- key: table:db_perf_rule;field:threshold1 + summary: "Numeric rule threshold; its label defines the unit and meaning. See the rule threshold guide." + content: | + Rule thresholds use the units named on each rule row. + + - SLOW_QUERY: threshold 1 = Mean execution time (ms) (default 500); threshold 2 = Database time (%) (default 5). + - MISSING_INDEX: threshold 1 = Sequential scan share (%) (default 50); threshold 2 = Tuples read per "day" (default 1000000). + - UNUSED_INDEX: threshold 1 = Days since statistics reset (default 14); threshold 2 = Index size (MB) (default 1). + - N_PLUS_ONE: threshold 1 = Repeats per unit (default 10); threshold 2 = Affected units (default 5). + - HOT_JOB: threshold 1 = Database time (%) (default 10); threshold 2 is unused. + - OVER_AUDITED: threshold 1 = Audit inserts per "day" (default 50000); threshold 2 is unused. + - BLOAT: threshold 1 = Dead tuple ratio (%) (default 20); threshold 2 = Dead tuples (default 10000). + - VACUUM_STALE: threshold 1 = Days since vacuum (default 7); threshold 2 = Transaction freeze age (default 150000000). + - POOL_SIZING: threshold 1 = Connections used (%) (default 80); threshold 2 = Idle connections (%) (default 50). + - IDLE_IN_TRANSACTION: threshold 1 = Idle seconds (default 60); threshold 2 is unused. + - CACHE_HIT_LOW: threshold 1 = Cache hit ratio (%) (default 95); threshold 2 is unused. + - TEMP_SPILL: threshold 1 = Temporary data per "day" (MB) (default 100); threshold 2 is unused. + - DEADLOCKS: threshold 1 = Deadlocks per "day" (default 1); threshold 2 is unused. +- key: table:db_perf_rule;field:threshold2 + summary: "Numeric rule threshold; its label defines the unit and meaning. See the rule threshold guide." + content: | + Rule thresholds use the units named on each rule row. + + - SLOW_QUERY: threshold 1 = Mean execution time (ms) (default 500); threshold 2 = Database time (%) (default 5). + - MISSING_INDEX: threshold 1 = Sequential scan share (%) (default 50); threshold 2 = Tuples read per "day" (default 1000000). + - UNUSED_INDEX: threshold 1 = Days since statistics reset (default 14); threshold 2 = Index size (MB) (default 1). + - N_PLUS_ONE: threshold 1 = Repeats per unit (default 10); threshold 2 = Affected units (default 5). + - HOT_JOB: threshold 1 = Database time (%) (default 10); threshold 2 is unused. + - OVER_AUDITED: threshold 1 = Audit inserts per "day" (default 50000); threshold 2 is unused. + - BLOAT: threshold 1 = Dead tuple ratio (%) (default 20); threshold 2 = Dead tuples (default 10000). + - VACUUM_STALE: threshold 1 = Days since vacuum (default 7); threshold 2 = Transaction freeze age (default 150000000). + - POOL_SIZING: threshold 1 = Connections used (%) (default 80); threshold 2 = Idle connections (%) (default 50). + - IDLE_IN_TRANSACTION: threshold 1 = Idle seconds (default 60); threshold 2 is unused. + - CACHE_HIT_LOW: threshold 1 = Cache hit ratio (%) (default 95); threshold 2 is unused. + - TEMP_SPILL: threshold 1 = Temporary data per "day" (MB) (default 100); threshold 2 is unused. + - DEADLOCKS: threshold 1 = Deadlocks per "day" (default 1); threshold 2 is unused. +- key: table:db_perf_rule;field:threshold1Label + summary: "Threshold1 label reported for this rule record." +- key: table:db_perf_rule;field:threshold2Label + summary: "Threshold2 label reported for this rule record." +- key: table:db_perf_settings;slot:help + summary: "Per-backend collection, retention, attribution and query-plan settings." + content: | + Created on the first collection or explain request. Administrators edit these values; dashboard reads do not create settings. Settings are retained until removed by the host. Run Collect now when empty. +- key: table:db_perf_settings;slot:emptyState + summary: "Per-backend collection, retention, attribution and query-plan settings." + content: | + Created on the first collection or explain request. Administrators edit these values; dashboard reads do not create settings. Settings are retained until removed by the host. Run Collect now when empty. +- key: table:db_perf_settings;field:backendName + summary: "The configured QQQ backend whose database is monitored." +- key: table:db_perf_settings;field:enabled + summary: "Whether collection or this suggestion rule participates in the next scheduled run." +- key: table:db_perf_settings;field:provider + summary: "Detected statistics provider; unsupported vendors report a setup message." +- key: table:db_perf_settings;field:snapshotIntervalMinutes + summary: "Minimum minutes between scheduled snapshots; default 5. Collect now forces collection." +- key: table:db_perf_settings;field:tableSnapshotMinutes + summary: "Minutes between table and index collection passes; default 60." +- key: table:db_perf_settings;field:detailRetentionDays + summary: "Age in days before detailed samples and plan audits are purged; default 30." +- key: table:db_perf_settings;field:rollupRetentionDays + summary: "Retention in days for daily rows and stale inventory or closed findings; default 365." +- key: table:db_perf_settings;field:topStatementsPerSnapshot + summary: "Maximum statement details saved per pass, ranked by database time; default 200." +- key: table:db_perf_settings;field:activityLongRunningSeconds + summary: "Active query age in seconds that makes a session notable; default 30." +- key: table:db_perf_settings;field:idleInTransactionSeconds + summary: "Idle transaction age in seconds that makes a session notable; default 60." +- key: table:db_perf_settings;field:attributionEnabled + summary: "Enable QQQ statement attribution; the host listener must also be enabled." +- key: table:db_perf_settings;field:attributionSampleRate + summary: "Fraction of QQQ units sampled, from 0 to 1; default 1 captures every unit." +- key: table:db_perf_settings;field:attributionFlushSeconds + summary: "Seconds between attribution batch flushes; default 60. A crash can lose one window." +- key: table:db_perf_settings;field:attributionMaxKeys + summary: "Maximum in-memory attribution groups per pod; overflow goes to an other group. Default 10000." +- key: table:db_perf_settings;field:nPlusOneThreshold + summary: "Repeated occurrences of a statement shape in one unit before recording a pattern; default 10." +- key: table:db_perf_settings;field:explainTimeoutMs + summary: "Maximum query-plan execution time in milliseconds; default 10000 for PLAN and ANALYZE." +- key: table:db_perf_settings;field:alertsEnabled + summary: "Enable database performance alert emission when the alert integration is installed." +- key: table:db_perf_settings;field:spikeFactor + summary: "Multiplier above the seven-day interval median needed for a query-spike alert; default 3." +- key: table:db_perf_settings;field:spikeMinTotalMs + summary: "Minimum interval query time in milliseconds before a spike can alert; default 5000." +- key: table:db_perf_settings;field:connectionAlertPercent + summary: "Percentage of max connections at which saturation can alert; default 80." +- key: table:db_perf_settings;field:blockedAlertSeconds + summary: "Blocked session duration in seconds required for a blocking-chain alert; default 30." +- key: table:db_perf_settings;field:lastSnapshotAt + summary: "Time of the latest collection tracked by the backend settings." +- key: table:db_perf_settings;field:lastTableSnapshotAt + summary: "Time of the latest table and index collection pass." +- key: table:db_perf_settings;field:statsAccess + summary: "Statistics capability: FULL, reduced access, missing extension, or unsupported provider." +- key: table:db_perf_settings;field:statsAccessMessage + summary: "The collector explains missing capabilities and the next grant or setup step." +- key: table:db_perf_snapshot;slot:help + summary: "One collection pass and its database-wide interval totals." + content: | + The collector writes this table every five minutes by default, including partial and failed passes. The first pass establishes a baseline. Retained for detailRetentionDays (30 by default). Run Collect now when empty. +- key: table:db_perf_snapshot;slot:emptyState + summary: "One collection pass and its database-wide interval totals." + content: | + The collector writes this table every five minutes by default, including partial and failed passes. The first pass establishes a baseline. Retained for detailRetentionDays (30 by default). Run Collect now when empty. +- key: table:db_perf_snapshot;field:backendName + summary: "The configured QQQ backend whose database is monitored." +- key: table:db_perf_snapshot;field:capturedAt + summary: "Time this database sample was captured." +- key: table:db_perf_snapshot;field:intervalSeconds + summary: "Interval seconds reported for this snapshot record." +- key: table:db_perf_snapshot;field:status + summary: "Current lifecycle state. Review the status-specific guidance and related messages." +- key: table:db_perf_snapshot;field:durationMs + summary: "Duration ms in milliseconds for this measurement." +- key: table:db_perf_snapshot;field:errorMessage + summary: "Sanitized failure description. Database SQL errors and parameter values are excluded." +- key: table:db_perf_snapshot;field:statsResetDetected + summary: "Counters restarted or decreased; that pass establishes a new baseline." +- key: table:db_perf_snapshot;field:includesTables + summary: "Includes tables reported for this snapshot record." +- key: table:db_perf_snapshot;field:xactCommit + summary: "Xact commit reported for this snapshot record." +- key: table:db_perf_snapshot;field:xactRollback + summary: "Xact rollback reported for this snapshot record." +- key: table:db_perf_snapshot;field:blksHit + summary: "Blks hit reported for this snapshot record." +- key: table:db_perf_snapshot;field:blksRead + summary: "Blks read reported for this snapshot record." +- key: table:db_perf_snapshot;field:cacheHitRatio + summary: "Database cache hits divided by hits plus reads, from 0 to 1." +- key: table:db_perf_snapshot;field:tempFiles + summary: "Temp files reported for this snapshot record." +- key: table:db_perf_snapshot;field:tempBytes + summary: "Temp bytes; size measured in bytes." +- key: table:db_perf_snapshot;field:deadlocks + summary: "Deadlocks reported for this snapshot record." +- key: table:db_perf_snapshot;field:conflicts + summary: "Conflicts reported for this snapshot record." +- key: table:db_perf_snapshot;field:connectionsTotal + summary: "Connections total reported for this snapshot record." +- key: table:db_perf_snapshot;field:connectionsActive + summary: "Connections active reported for this snapshot record." +- key: table:db_perf_snapshot;field:connectionsIdle + summary: "Connections idle reported for this snapshot record." +- key: table:db_perf_snapshot;field:connectionsIdleInTx + summary: "Connections idle in tx reported for this snapshot record." +- key: table:db_perf_snapshot;field:maxConnections + summary: "Max connections reported for this snapshot record." +- key: table:db_perf_snapshot;field:databaseSizeBytes + summary: "Database size bytes; size measured in bytes." +- key: table:db_perf_snapshot;field:statementCount + summary: "Statement count reported for this snapshot record." +- key: table:db_perf_snapshot;field:statementCalls + summary: "Statement calls reported for this snapshot record." +- key: table:db_perf_snapshot;field:statementTotalMs + summary: "Statement total ms in milliseconds for this measurement." +- key: table:db_perf_snapshot;field:storedStatementCount + summary: "Stored statement count reported for this snapshot record." +- key: table:db_perf_snapshot;field:rawXactCommit + summary: "Previous cumulative xact commit counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawXactRollback + summary: "Previous cumulative xact rollback counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawBlksHit + summary: "Previous cumulative blks hit counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawBlksRead + summary: "Previous cumulative blks read counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawTempFiles + summary: "Previous cumulative temp files counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawTempBytes + summary: "Previous cumulative temp bytes counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawDeadlocks + summary: "Previous cumulative deadlocks counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawConflicts + summary: "Previous cumulative conflicts counter used for reset-safe deltas." +- key: table:db_perf_snapshot;field:rawStatsReset + summary: "Previous database counter reset timestamp used for delta calculations." +- key: table:db_perf_snapshot;field:rawStatementStatsReset + summary: "Previous provider reset marker used to detect counter resets, not raw SQL." +- key: table:db_perf_statement;slot:help + summary: "Normalized query shapes with the latest counters and retained 24-hour totals." + content: | + The collector upserts statements observed in pg_stat_statements each collection. Two snapshots establish deltas. Retained while observed; stale records are purged after rollupRetentionDays (365 by default). Enable pg_stat_statements and collect twice when empty. +- key: table:db_perf_statement;slot:emptyState + summary: "Normalized query shapes with the latest counters and retained 24-hour totals." + content: | + The collector upserts statements observed in pg_stat_statements each collection. Two snapshots establish deltas. Retained while observed; stale records are purged after rollupRetentionDays (365 by default). Enable pg_stat_statements and collect twice when empty. +- key: table:db_perf_statement;field:backendName + summary: "The configured QQQ backend whose database is monitored." +- key: table:db_perf_statement;field:queryId + summary: "PostgreSQL query identifier; fingerprint provides cross-source normalized matching." +- key: table:db_perf_statement;field:dbUser + summary: "Database role associated with this statement, session or connection group." +- key: table:db_perf_statement;field:toplevel + summary: "Toplevel reported for this statement record." +- key: table:db_perf_statement;field:fingerprint + summary: "SHA-256 of the normalized SQL shape; literals and bind values are excluded." +- key: table:db_perf_statement;field:queryText + summary: "Normalized query text with literals replaced by placeholders; never raw live text." +- key: table:db_perf_statement;field:operation + summary: "Normalized SQL operation, such as SELECT, INSERT, UPDATE or DELETE." +- key: table:db_perf_statement;field:mainTable + summary: "Best-effort main table extracted from normalized SQL." +- key: table:db_perf_statement;field:firstSeenAt + summary: "First recorded observation of this statement or finding." +- key: table:db_perf_statement;field:lastSeenAt + summary: "Most recent observation of this statement, table, index or finding." +- key: table:db_perf_statement;field:lastCalls + summary: "Last calls reported for this statement record." +- key: table:db_perf_statement;field:lastTotalExecMs + summary: "Last total exec ms in milliseconds for this measurement." +- key: table:db_perf_statement;field:lastRows + summary: "Last rows reported for this statement record." +- key: table:db_perf_statement;field:lastSharedBlksHit + summary: "Last shared blks hit reported for this statement record." +- key: table:db_perf_statement;field:lastSharedBlksRead + summary: "Last shared blks read reported for this statement record." +- key: table:db_perf_statement;field:lastTempBlksWritten + summary: "Last temp blks written reported for this statement record." +- key: table:db_perf_statement;field:lastMaxExecMs + summary: "Latest cumulative maximum execution time in milliseconds; not an interval maximum." +- key: table:db_perf_statement;field:lastStatsSince + summary: "Per-statement counter start time when the PostgreSQL version supplies it." +- key: table:db_perf_statement;field:calls24h + summary: "Calls in retained statement details from the last 24 hours." +- key: table:db_perf_statement;field:totalMs24h + summary: "Execution time in milliseconds in retained statement details from the last 24 hours." +- key: table:db_perf_statement;field:meanMs24h + summary: "Total retained execution milliseconds divided by retained calls over 24 hours." +- key: table:db_perf_statement;field:shareOfDbTime24h + summary: "Percentage of database time over 24 hours; numerator uses retained statement details." +- key: table:db_perf_table_daily;slot:help + summary: "Daily table scan deltas and ending size used to track growth." + content: | + The daily rollup writes these rows from retained table samples, normally at 03:15. Retained for rollupRetentionDays (365). Wait for the daily rollup after table snapshots exist. +- key: table:db_perf_table_daily;slot:emptyState + summary: "Daily table scan deltas and ending size used to track growth." + content: | + The daily rollup writes these rows from retained table samples, normally at 03:15. Retained for rollupRetentionDays (365). Wait for the daily rollup after table snapshots exist. +- key: table:db_perf_table_daily;field:tableStatId + summary: "Reference to the related table stat record." +- key: table:db_perf_table_daily;field:day + summary: "Calendar day in the configured qbit time zone for this rollup." +- key: table:db_perf_table_daily;field:seqScan + summary: "Seq scan reported for this table daily record." +- key: table:db_perf_table_daily;field:idxScan + summary: "Idx scan reported for this table daily record." +- key: table:db_perf_table_daily;field:seqTupRead + summary: "Seq tup read reported for this table daily record." +- key: table:db_perf_table_daily;field:nTupIns + summary: "PostgreSQL n tup ins tuple statistic for the sampled table." +- key: table:db_perf_table_daily;field:nTupUpd + summary: "PostgreSQL n tup upd tuple statistic for the sampled table." +- key: table:db_perf_table_daily;field:nTupDel + summary: "PostgreSQL n tup del tuple statistic for the sampled table." +- key: table:db_perf_table_daily;field:nLiveTup + summary: "PostgreSQL n live tup tuple statistic for the sampled table." +- key: table:db_perf_table_daily;field:nDeadTup + summary: "PostgreSQL n dead tup tuple statistic for the sampled table." +- key: table:db_perf_table_daily;field:totalBytes + summary: "Total bytes; size measured in bytes." +- key: table:db_perf_table_snapshot;slot:help + summary: "Historical samples of table scans, tuples, size and maintenance." + content: | + The collector writes table snapshots every tableSnapshotMinutes (60). Retained for detailRetentionDays (30). Run Collect now or wait for the next hourly table pass. +- key: table:db_perf_table_snapshot;slot:emptyState + summary: "Historical samples of table scans, tuples, size and maintenance." + content: | + The collector writes table snapshots every tableSnapshotMinutes (60). Retained for detailRetentionDays (30). Run Collect now or wait for the next hourly table pass. +- key: table:db_perf_table_snapshot;field:snapshotId + summary: "Reference to the related snapshot record." +- key: table:db_perf_table_snapshot;field:tableStatId + summary: "Reference to the related table stat record." +- key: table:db_perf_table_snapshot;field:capturedAt + summary: "Time this database sample was captured." +- key: table:db_perf_table_snapshot;field:seqScan + summary: "Seq scan reported for this table snapshot record." +- key: table:db_perf_table_snapshot;field:seqTupRead + summary: "Seq tup read reported for this table snapshot record." +- key: table:db_perf_table_snapshot;field:idxScan + summary: "Idx scan reported for this table snapshot record." +- key: table:db_perf_table_snapshot;field:idxTupFetch + summary: "Idx tup fetch reported for this table snapshot record." +- key: table:db_perf_table_snapshot;field:nTupIns + summary: "PostgreSQL n tup ins tuple statistic for the sampled table." +- key: table:db_perf_table_snapshot;field:nTupUpd + summary: "PostgreSQL n tup upd tuple statistic for the sampled table." +- key: table:db_perf_table_snapshot;field:nTupDel + summary: "PostgreSQL n tup del tuple statistic for the sampled table." +- key: table:db_perf_table_snapshot;field:nTupHotUpd + summary: "PostgreSQL n tup hot upd tuple statistic for the sampled table." +- key: table:db_perf_table_snapshot;field:nLiveTup + summary: "PostgreSQL n live tup tuple statistic for the sampled table." +- key: table:db_perf_table_snapshot;field:nDeadTup + summary: "PostgreSQL n dead tup tuple statistic for the sampled table." +- key: table:db_perf_table_snapshot;field:totalBytes + summary: "Total bytes; size measured in bytes." +- key: table:db_perf_table_snapshot;field:tableBytes + summary: "Table bytes; size measured in bytes." +- key: table:db_perf_table_snapshot;field:indexBytes + summary: "Index bytes; size measured in bytes." +- key: table:db_perf_table_snapshot;field:autovacuumCount + summary: "Autovacuum count reported for this table snapshot record." +- key: table:db_perf_table_snapshot;field:autoanalyzeCount + summary: "Autoanalyze count reported for this table snapshot record." +- key: table:db_perf_table_snapshot;field:freezeAge + summary: "Transaction-ID age used to assess vacuum and wraparound risk." +- key: table:db_perf_table_stat;slot:help + summary: "Current table size, scans, tuple counts and maintenance statistics." + content: | + The collector upserts table data every tableSnapshotMinutes (60). Stale records are purged after rollupRetentionDays (365). Run Collect now to force table collection when empty. +- key: table:db_perf_table_stat;slot:emptyState + summary: "Current table size, scans, tuple counts and maintenance statistics." + content: | + The collector upserts table data every tableSnapshotMinutes (60). Stale records are purged after rollupRetentionDays (365). Run Collect now to force table collection when empty. +- key: table:db_perf_table_stat;field:backendName + summary: "The configured QQQ backend whose database is monitored." +- key: table:db_perf_table_stat;field:schemaName + summary: "Schema name reported for this table stat record." +- key: table:db_perf_table_stat;field:tableName + summary: "Table name reported for this table stat record." +- key: table:db_perf_table_stat;field:relid + summary: "Relid reported for this table stat record." +- key: table:db_perf_table_stat;field:seqScan + summary: "Seq scan reported for this table stat record." +- key: table:db_perf_table_stat;field:seqTupRead + summary: "Seq tup read reported for this table stat record." +- key: table:db_perf_table_stat;field:idxScan + summary: "Idx scan reported for this table stat record." +- key: table:db_perf_table_stat;field:idxTupFetch + summary: "Idx tup fetch reported for this table stat record." +- key: table:db_perf_table_stat;field:nTupIns + summary: "PostgreSQL n tup ins tuple statistic for the sampled table." +- key: table:db_perf_table_stat;field:nTupUpd + summary: "PostgreSQL n tup upd tuple statistic for the sampled table." +- key: table:db_perf_table_stat;field:nTupDel + summary: "PostgreSQL n tup del tuple statistic for the sampled table." +- key: table:db_perf_table_stat;field:nTupHotUpd + summary: "PostgreSQL n tup hot upd tuple statistic for the sampled table." +- key: table:db_perf_table_stat;field:nLiveTup + summary: "PostgreSQL n live tup tuple statistic for the sampled table." +- key: table:db_perf_table_stat;field:nDeadTup + summary: "PostgreSQL n dead tup tuple statistic for the sampled table." +- key: table:db_perf_table_stat;field:deadRatio + summary: "Dead tuples divided by live plus dead tuples, from 0 to 1; an estimate used for maintenance review." +- key: table:db_perf_table_stat;field:totalBytes + summary: "Total bytes; size measured in bytes." +- key: table:db_perf_table_stat;field:tableBytes + summary: "Table bytes; size measured in bytes." +- key: table:db_perf_table_stat;field:indexBytes + summary: "Index bytes; size measured in bytes." +- key: table:db_perf_table_stat;field:toastBytes + summary: "Toast bytes; size measured in bytes." +- key: table:db_perf_table_stat;field:growthBytes7d + summary: "Change in table size in bytes over seven days, once enough daily history exists." +- key: table:db_perf_table_stat;field:lastVacuum + summary: "Last manual vacuum reported by PostgreSQL." +- key: table:db_perf_table_stat;field:lastAutovacuum + summary: "Last automatic vacuum reported by PostgreSQL." +- key: table:db_perf_table_stat;field:lastAnalyze + summary: "Last manual statistics analyze reported by PostgreSQL." +- key: table:db_perf_table_stat;field:lastAutoanalyze + summary: "Last automatic statistics analyze reported by PostgreSQL." +- key: table:db_perf_table_stat;field:vacuumCount + summary: "Vacuum count reported for this table stat record." +- key: table:db_perf_table_stat;field:autovacuumCount + summary: "Autovacuum count reported for this table stat record." +- key: table:db_perf_table_stat;field:analyzeCount + summary: "Analyze count reported for this table stat record." +- key: table:db_perf_table_stat;field:autoanalyzeCount + summary: "Autoanalyze count reported for this table stat record." +- key: table:db_perf_table_stat;field:freezeAge + summary: "Transaction-ID age used to assess vacuum and wraparound risk." +- key: table:db_perf_table_stat;field:seqScanShare + summary: "Fraction of table scans that are sequential, from 0 to 1; high values merit query and index review." +- key: table:db_perf_table_stat;field:lastSeenAt + summary: "Most recent observation of this statement, table, index or finding." +- key: process:dbPerfCollectSnapshot;slot:help + summary: "Collect database snapshots on the host schedule." + content: | + Runs at the configured snapshot cadence, normally every five minutes. Reads statistics and writes interval deltas and notable activity; table/index passes normally run hourly. Requires manage settings. +- key: process:dbPerfCollectNow;slot:help + summary: "Collect an immediate snapshot and force table and index statistics." + content: | + Run from the Snapshots table. Requires manage settings. The Result screen reports success, partial access or collection failure. The first pass establishes counter baselines. +- key: process:dbPerfRollupAndPurge;slot:help + summary: "Roll up completed days and remove expired details." + content: | + Runs daily at 03:15 by default. Uses the configured time zone, detail retention and rollup retention. Reruns replace daily aggregates instead of duplicating them. +- key: process:dbPerfQueryPlan;slot:help + summary: "Inspect the plan of one statement, with optional read-only execution." + content: | + Choose PLAN (default) or ANALYZE. PLAN uses a generic plan for parameterized SQL on PostgreSQL 16+. ANALYZE requires explain analyze permission and exactly one SELECT with no modifying CTEs, INTO or row locks. Enter parameter values only for ANALYZE, one per line in placeholder order. Values are sent transiently; they and the ANALYZE plan are never stored. Every request is audited. Results include copyable plan text, cost, a SQL Workbench link and a literal-free assistant prompt. No model is called by this qbit. +- key: widget:dbPerfSetup;slot:label + summary: "Check backend, statistics access, collection, attribution and scheduler setup." +- key: widget:dbPerfDatabaseTime;slot:label + summary: "Database execution milliseconds and call counts per snapshot over the last 24 hours." +- key: widget:dbPerfTopQueries;slot:label + summary: "Most expensive retained query shapes over 24 hours; open a query to inspect its history." +- key: widget:dbPerfFindings;slot:label + summary: "Open and acknowledged findings ranked by impact." +- key: widget:dbPerfLiveActivity;slot:label + summary: "Current notable sessions with normalized text, blocking and transaction ages." +- key: widget:dbPerfConnections;slot:label + summary: "Connection capacity followed by a live application and user breakdown." +- key: widget:dbPerfConnectionStatistics;slot:label + summary: "Latest snapshot capacity and cache hit percentage, with 24-hour temp and deadlock totals." +- key: widget:dbPerfConnectionsByApplication;slot:label + summary: "Current connections grouped by application, database user and state." +- key: widget:dbPerfAttribution;slot:label + summary: "QQQ units ranked by database time, followed by repeated-statement patterns." +- key: widget:dbPerfTables;slot:label + summary: "Sequential-scan-heavy tables and tables with the fastest seven-day size growth." +- key: widget:db_perf_snapshot_db_perf_query_snapshot;slot:label + summary: "Related interval deltas for the highest-cost statements in each collection." +- key: widget:db_perf_statement_db_perf_query_snapshot;slot:label + summary: "Related interval deltas for the highest-cost statements in each collection." +- key: widget:db_perf_statement_db_perf_query_daily;slot:label + summary: "Related daily totals from retained statement snapshot details." +- key: widget:db_perf_snapshot_db_perf_table_snapshot;slot:label + summary: "Related historical samples of table scans, tuples, size and maintenance." +- key: widget:db_perf_table_stat_db_perf_table_snapshot;slot:label + summary: "Related historical samples of table scans, tuples, size and maintenance." +- key: widget:db_perf_table_stat_db_perf_table_daily;slot:label + summary: "Related daily table scan deltas and ending size used to track growth." +- key: widget:db_perf_table_stat_db_perf_index_stat;slot:label + summary: "Related current index definitions, usage, size and uniqueness." +- key: widget:db_perf_snapshot_db_perf_activity_sample;slot:label + summary: "Related notable session samples with normalized text, never raw query text." +- key: widget:db_perf_snapshot_db_perf_connection_sample;slot:label + summary: "Related connection counts grouped by application, user and state for each snapshot." +- key: widget:db_perf_rule_db_perf_finding;slot:label + summary: "Related ranked, deduplicated suggestions with supporting evidence and proposed fixes." +- key: widget:db_perf_statement_db_perf_finding;slot:label + summary: "Related ranked, deduplicated suggestions with supporting evidence and proposed fixes." +- key: widget:db_perf_table_stat_db_perf_finding;slot:label + summary: "Related ranked, deduplicated suggestions with supporting evidence and proposed fixes." +- key: widget:db_perf_statement_db_perf_explain;slot:label + summary: "Related audited plan requests, including denied, failed and timed-out attempts." +- key: possibleValueSource:db_perf_ExplainMode;slot:help + summary: "Available values for explainmode." +- key: possibleValueSource:db_perf_FindingStatus;slot:help + summary: "Available values for findingstatus." +- key: possibleValueSource:db_perf_Operation;slot:help + summary: "Available values for operation." +- key: possibleValueSource:db_perf_Outcome;slot:help + summary: "Available values for outcome." +- key: possibleValueSource:db_perf_Provider;slot:help + summary: "Available values for provider." +- key: possibleValueSource:db_perf_Reason;slot:help + summary: "Available values for reason." +- key: possibleValueSource:db_perf_Severity;slot:help + summary: "Available values for severity." +- key: possibleValueSource:db_perf_SnapshotStatus;slot:help + summary: "Available values for snapshotstatus." +- key: possibleValueSource:db_perf_State;slot:help + summary: "Available values for state." +- key: possibleValueSource:db_perf_StatsAccess;slot:help + summary: "Available values for statsaccess." +- key: possibleValueSource:db_perf_UnitKind;slot:help + summary: "Available values for unitkind." +- key: possibleValueSource:db_perf_activity_sample;slot:help + summary: "Available values for activity sample." +- key: possibleValueSource:db_perf_attribution;slot:help + summary: "Available values for attribution." +- key: possibleValueSource:db_perf_attribution_daily;slot:help + summary: "Available values for attribution daily." +- key: possibleValueSource:db_perf_connection_sample;slot:help + summary: "Available values for connection sample." +- key: possibleValueSource:db_perf_explain;slot:help + summary: "Available values for explain." +- key: possibleValueSource:db_perf_finding;slot:help + summary: "Available values for finding." +- key: possibleValueSource:db_perf_index_stat;slot:help + summary: "Available values for index stat." +- key: possibleValueSource:db_perf_query_daily;slot:help + summary: "Available values for query daily." +- key: possibleValueSource:db_perf_query_snapshot;slot:help + summary: "Available values for query snapshot." +- key: possibleValueSource:db_perf_repeat_pattern;slot:help + summary: "Available values for repeat pattern." +- key: possibleValueSource:db_perf_rule;slot:help + summary: "Available values for rule." +- key: possibleValueSource:db_perf_settings;slot:help + summary: "Available values for settings." +- key: possibleValueSource:db_perf_snapshot;slot:help + summary: "Available values for snapshot." +- key: possibleValueSource:db_perf_statement;slot:help + summary: "Available values for statement." +- key: possibleValueSource:db_perf_table_daily;slot:help + summary: "Available values for table daily." +- key: possibleValueSource:db_perf_table_snapshot;slot:help + summary: "Available values for table snapshot." +- key: possibleValueSource:db_perf_table_stat;slot:help + summary: "Available values for table stat." +- key: process:db_perf_settings.bulkEdit;slot:help + summary: Review and apply changes to database performance settings records. + content: | + Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema. +- key: process:db_perf_settings.bulkEditWithFile;slot:help + summary: Review and apply changes to database performance settings records. + content: | + Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema. +- key: process:db_perf_rule.bulkInsert;slot:help + summary: Review and apply changes to database performance rule records. + content: | + Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema. +- key: process:db_perf_rule.bulkEdit;slot:help + summary: Review and apply changes to database performance rule records. + content: | + Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema. +- key: process:db_perf_rule.bulkEditWithFile;slot:help + summary: Review and apply changes to database performance rule records. + content: | + Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema. +- key: process:db_perf_rule.bulkDelete;slot:help + summary: Review and apply changes to database performance rule records. + content: | + Requires manage settings permission. Review every changed value before applying the standard QQQ action. Changes affect subsequent collection or analysis; they do not change the monitored database schema. +- key: possibleValueSource:db_perf_ExplainRequestMode;slot:help + summary: PLAN inspects the query; ANALYZE executes a guarded read-only SELECT with separate permission. diff --git a/src/main/resources/openapi/qqq-v1/db-performance.yaml b/src/main/resources/openapi/qqq-v1/db-performance.yaml new file mode 100644 index 0000000..8472a4e --- /dev/null +++ b/src/main/resources/openapi/qqq-v1/db-performance.yaml @@ -0,0 +1,434 @@ +components: + schemas: + DbPerformanceActivityV1: + properties: + blockingChains: + items: + $ref: "#/components/schemas/DbPerformanceBlockingChainV1" + type: "array" + liveTextRedacted: + type: "boolean" + sessions: + items: + $ref: "#/components/schemas/DbPerformanceSessionV1" + type: "array" + type: "object" + DbPerformanceBlockingChainV1: + properties: + blockedPids: + items: + nullable: true + type: "integer" + type: "array" + rootPid: + nullable: true + type: "integer" + type: "object" + DbPerformanceErrorV1: + properties: + error: + nullable: true + type: "string" + type: "object" + DbPerformanceExplainRequestV1: + description: "Supply exactly one of statementId or sql. Parameters are strings,\ + \ accepted only for ANALYZE and never stored." + properties: + backend: + nullable: true + type: "string" + mode: + enum: + - "PLAN" + - "ANALYZE" + type: "string" + parameters: + items: + nullable: true + type: "string" + type: "array" + sql: + nullable: true + type: "string" + statementId: + nullable: true + type: "integer" + type: "object" + DbPerformanceExplainResultV1: + properties: + actualTotalMs: + nullable: true + type: "number" + assistantPrompt: + nullable: true + type: "string" + explainId: + nullable: true + type: "integer" + mode: + nullable: true + type: "string" + planJson: + nullable: true + type: "string" + planText: + nullable: true + type: "string" + sqlWorkbenchUrl: + nullable: true + type: "string" + totalCost: + nullable: true + type: "number" + warnings: + items: + nullable: true + type: "string" + type: "array" + type: "object" + DbPerformanceFindingV1: + properties: + id: + nullable: true + type: "integer" + score: + nullable: true + type: "number" + severity: + nullable: true + type: "string" + title: + nullable: true + type: "string" + type: "object" + DbPerformanceLookupV1: + properties: + fingerprint: + nullable: true + type: "string" + matches: + items: + $ref: "#/components/schemas/DbPerformanceStatementV1" + type: "array" + normalizedSql: + nullable: true + type: "string" + type: "object" + DbPerformanceOverviewV1: + properties: + findingsBySeverity: + additionalProperties: + nullable: true + type: "integer" + type: "object" + settings: + $ref: "#/components/schemas/DbPerformanceSettingsV1" + snapshot: + $ref: "#/components/schemas/DbPerformanceSnapshotV1" + topFindings: + items: + $ref: "#/components/schemas/DbPerformanceFindingV1" + type: "array" + topStatements: + items: + $ref: "#/components/schemas/DbPerformanceStatementV1" + type: "array" + type: "object" + DbPerformanceSessionV1: + properties: + application: + nullable: true + type: "string" + blockedBy: + items: + nullable: true + type: "integer" + type: "array" + blockingCount: + nullable: true + type: "integer" + fingerprint: + nullable: true + type: "string" + liveTextRedacted: + type: "boolean" + normalizedText: + nullable: true + type: "string" + pid: + nullable: true + type: "integer" + query: + description: "Raw live SQL only with view_live_text; never persisted." + nullable: true + type: "string" + querySeconds: + nullable: true + type: "number" + state: + nullable: true + type: "string" + user: + nullable: true + type: "string" + waitEvent: + nullable: true + type: "string" + xactSeconds: + nullable: true + type: "number" + type: "object" + DbPerformanceSettingsV1: + properties: + lastSnapshotAt: + nullable: true + type: "string" + provider: + nullable: true + type: "string" + statsAccess: + nullable: true + type: "string" + statsAccessMessage: + nullable: true + type: "string" + type: "object" + DbPerformanceSnapshotV1: + properties: + cacheHitRatio: + nullable: true + type: "number" + capturedAt: + nullable: true + type: "string" + connectionsTotal: + nullable: true + type: "integer" + deadlocks: + nullable: true + type: "integer" + id: + nullable: true + type: "integer" + maxConnections: + nullable: true + type: "integer" + statementCalls: + nullable: true + type: "integer" + statementTotalMs: + nullable: true + type: "number" + status: + nullable: true + type: "string" + tempBytes: + nullable: true + type: "integer" + type: "object" + DbPerformanceStatementV1: + properties: + calls24h: + nullable: true + type: "integer" + historyPath: + nullable: true + type: "string" + hitRatio: + nullable: true + type: "number" + id: + nullable: true + type: "integer" + meanMs24h: + nullable: true + type: "number" + queryText: + description: "Normalized SQL, truncated to 300 characters." + nullable: true + type: "string" + shareOfDbTime24h: + nullable: true + type: "number" + totalMs24h: + nullable: true + type: "number" + type: "object" + securitySchemes: + sessionUuidCookie: + in: "cookie" + name: "sessionUUID" + type: "apiKey" +info: + contact: + email: "contact@kingsrook.com" + description: "Database performance, live activity and audited query plans" + title: "QQQ Middleware API extension: db-performance" + version: "1.0.0" +openapi: "3.0.3" +paths: + /qqq/v1/qbit/db-performance/overview: + get: + description: "Requires db_performance.view and uses the caller's session." + operationId: "dbPerformanceOverview" + parameters: + - description: "Configured monitored backend; defaults to the qbit backend." + in: "query" + name: "backend" + required: false + schema: + type: "string" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Sign in required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Permission denied" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceOverviewV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Database Performance overview" + /qqq/v1/qbit/db-performance/activity: + get: + description: "Requires db_performance.view and uses the caller's session." + operationId: "dbPerformanceActivity" + parameters: + - description: "Configured monitored backend; defaults to the qbit backend." + in: "query" + name: "backend" + required: false + schema: + type: "string" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Sign in required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Permission denied" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceActivityV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Database Performance activity" + /qqq/v1/qbit/db-performance/explain: + post: + description: "PLAN requires db_performance.explain; ANALYZE requires db_performance.explain_analyze.\ + \ Values and ANALYZE plans are never persisted." + operationId: "dbPerformanceExplain" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceExplainRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Sign in required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Permission denied" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceExplainResultV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Explain a PostgreSQL query" + /qqq/v1/qbit/db-performance/statements/lookup: + get: + description: "Requires db_performance.view and uses the caller's session." + operationId: "dbPerformanceLookup" + parameters: + - description: "Configured monitored backend; defaults to the qbit backend." + in: "query" + name: "backend" + required: false + schema: + type: "string" + - in: "query" + name: "sql" + required: true + schema: + type: "string" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Sign in required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceErrorV1" + description: "Permission denied" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DbPerformanceLookupV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Database Performance lookup" diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java new file mode 100644 index 0000000..a63aeb3 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfApiContractTest.java @@ -0,0 +1,66 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.nio.file.Files; +import java.nio.file.Path; +import com.kingsrook.qbits.dbperformance.api.DbPerformanceUiApi; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData; +import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; +import com.kingsrook.qqq.middleware.javalin.tools.UiApiContract; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The committed fragment is generated from exactly the registered UI specs. + *******************************************************************************/ +class DbPerfApiContractTest +{ + + + + /******************************************************************************* + ** Explicit regeneration flag; normal builds detect contract drift. + *******************************************************************************/ + @Test + void contract() throws Exception + { + var extension = DbPerformanceUiApi.extension(); + var version = new MiddlewareVersionV1(); + String generated = UiApiContract.generateExtensionYaml(version, extension); + Path path = Path.of("src/main/resources/openapi/qqq-v1/db-performance.yaml"); + if(Boolean.getBoolean("dbperf.openapi.update")) + { + Files.createDirectories(path.getParent()); + Files.writeString(path, generated); + } + assertThat(UiApiContract.check(generated, Files.readString(path), null, 1, "mvnw -Dtest=DbPerfApiContractTest -Ddbperf.openapi.update=true test")).isEmpty(); + var model = version.generateExtensionOpenAPIModel("qqq", extension); + assertThat(model.getComponents().getSchemas().keySet()).allMatch(name -> name.startsWith("DbPerformance") && name.endsWith("V1")); + assertThat(model.getPaths().keySet()).containsExactlyInAnyOrder("/qqq/v1/qbit/db-performance/overview", "/qqq/v1/qbit/db-performance/activity", + "/qqq/v1/qbit/db-performance/explain", "/qqq/v1/qbit/db-performance/statements/lookup"); + assertThat(extension.getSemanticVersion()).isEqualTo("1.0.0"); + assertThat(MiddlewareSpecExtensionsMetaData.of(DbPerfTestSupport.instance()).getExtensions()).containsOnlyKeys("db-performance"); + assertThat(generated).contains("liveTextRedacted", "parameters", "assistantPrompt", "403"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHelpBundleTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHelpBundleTest.java new file mode 100644 index 0000000..5f6be4c --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfHelpBundleTest.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import com.kingsrook.qbits.dbperformance.metadata.DbPerfHelp; +import com.kingsrook.qbits.dbperformance.metadata.DbPerfMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; +import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles; +import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Version, live references, summary quality and complete owned metadata coverage. + *******************************************************************************/ +class DbPerfHelpBundleTest +{ + + + + /******************************************************************************* + ** No host Help tables are needed for registration or empty-state guidance. + *******************************************************************************/ + @Test + void bundleCoverage() throws Exception + { + var instance = DbPerfTestSupport.instance(); + HelpBundleAssertions.forInstance(instance).bundles(DbPerfHelp.RESOURCE) + .owning(target -> target.name().startsWith("db_perf_") || target.name().startsWith("dbPerf")).assertComplete(); + var bundle = HelpBundles.of(instance).getBundles().getFirst(); + assertThat(bundle.getVersion()).isEqualTo(projectVersion()); + var keys = bundle.getEntries().stream().map(entry -> entry.getKey()).toList(); + instance.getWidgets().keySet().forEach(name -> assertThat(keys).contains("widget:" + name + ";slot:label")); + for(String table : DbPerfMetaData.TABLES) + { + assertThat(instance.getTable(table).getHelpContent()).containsKeys("help", "emptyState"); + } + QHelpContent host = new QHelpContent().withContentAsMarkdown("Host-specific help"); + instance.getTable("db_perf_statement").withHelpContent("help", host); + DbPerfHelp.register(instance, DbPerfRuntime.of(instance).getConfig()); + assertThat(instance.getTable("db_perf_statement").getHelpContent().get("help")).containsExactly(host); + } + + + + /******************************************************************************* + ** Compare to the effective Maven project version supplied to this test's resource. + *******************************************************************************/ + private static String projectVersion() throws Exception + { + try(var stream = DbPerfHelpBundleTest.class.getResourceAsStream("/help/project-version.txt")) + { + return (new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).trim()); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java new file mode 100644 index 0000000..f30f509 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/DbPerfPresentationTest.java @@ -0,0 +1,292 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dbperformance.api.ActivitySpec; +import com.kingsrook.qbits.dbperformance.api.DbPerformanceData; +import com.kingsrook.qbits.dbperformance.api.ExplainSpec; +import com.kingsrook.qbits.dbperformance.api.LookupSpec; +import com.kingsrook.qbits.dbperformance.api.OverviewSpec; +import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.explain.ExplainRequest; +import com.kingsrook.qbits.dbperformance.explain.ExplainService; +import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess; +import com.kingsrook.qbits.dbperformance.normalize.SqlNormalizer; +import com.kingsrook.qbits.dbperformance.provider.SessionInfo; +import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgetRenderer; +import com.kingsrook.qbits.dbperformance.widgets.DbPerfWidgets; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiTableData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec.JsonInput; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** H2 fixtures exercise real QQQ reads, typed widgets and the actual spec handlers. + *******************************************************************************/ +class DbPerfPresentationTest +{ + private DbPerfRuntime runtime; + private DbPerformanceData data; + + + + /******************************************************************************* + ** Isolated migrated tables and production producer registration. + *******************************************************************************/ + @BeforeEach + void setup() throws Exception + { + runtime = DbPerfRuntime.of(DbPerfTestSupport.instance()); + data = new DbPerformanceData(runtime); + } + + + + /******************************************************************************* + ** Every home widget renders an honest guided empty state and requires view. + *******************************************************************************/ + @Test + void emptyWidgetsAndPermissions() throws Exception + { + assertThat(runtime.getInstance().getApp("dbPerformance").getWidgets()).containsExactlyElementsOf(DbPerfWidgets.HOME); + for(String name : DbPerfWidgets.HOME) + { + var meta = runtime.getInstance().getWidget(name); + assertThat(meta.getGridColumns()).isEqualTo(12); + assertThat(meta.getPermissionRules().getPermissionBaseName()).isEqualTo(DbPerfPermissions.VIEW); + var output = new DbPerfWidgetRenderer().render(new RenderWidgetInput().withWidgetMetaData(meta)); + assertThat(output.getWidgetData()).isNotNull(); + if(output.getWidgetData() instanceof TableData table) + { + assertThat(table.getFixedHeight()).isNull(); + assertThat(table.getRowsPerPage()).isEqualTo(10); + } + } + assertThat(new DbPerfWidgetRenderer().topQueries(data).getNoRowsFoundHTML()).contains("two snapshots"); + assertThat(new DbPerfWidgetRenderer().liveActivity(data).getNoRowsFoundHTML()).isEqualTo("No long-running or blocked sessions right now"); + assertThat(new DbPerfWidgetRenderer().attribution().getTableDataList().getFirst().getNoRowsFoundHTML()).isEqualTo("Attribution starts after the next flush"); + assertThat(new DbPerfWidgetRenderer().connectionStatistics(data).getType()).isEqualTo("multiStatistics"); + assertThat(new DbPerfWidgetRenderer().connectionsByApplication(runtime).getRows()).isEmpty(); + QContext.setQSession(new QSession()); + for(String name : DbPerfWidgets.HOME) + { + assertThatThrownBy(() -> new DbPerfWidgetRenderer().render(new RenderWidgetInput().withWidgetMetaData(runtime.getInstance().getWidget(name)))).hasMessageContaining("permission"); + } + assertThatThrownBy(() -> new OverviewSpec().handle(new JsonInput())).hasMessageContaining("permission"); + assertThatThrownBy(() -> new ActivitySpec().handle(new JsonInput())).hasMessageContaining("permission"); + assertThatThrownBy(() -> new LookupSpec().handle(new JsonInput())).hasMessageContaining("permission"); + } + + + + /******************************************************************************* + ** Persisted rows change the API and widgets, with correct units and record links. + *******************************************************************************/ + @Test + @SuppressWarnings("unchecked") + void fixtureRows() throws Exception + { + fixture(); + Map overview = (Map) new OverviewSpec().handle(new JsonInput()); + assertThat((List) overview.get("topStatements")).hasSize(1); + assertThat((Map) overview.get("findingsBySeverity")).containsEntry("HIGH", 1); + TableData queries = new DbPerfWidgetRenderer().topQueries(data); + assertThat(queries.getRows()).hasSize(1); + assertThat((BigDecimal) queries.getRows().getFirst().get("% DB time")).isEqualByComparingTo("25"); + assertThat(JsonUtils.toJson(queries)).contains("/app/dbPerformance/db_perf_statement/", "composite").doesNotContain("private-value"); + ChartData chart = new DbPerfWidgetRenderer().databaseTime(data); + assertThat(chart.getChartData().getDatasets()).hasSize(2); + assertThat(chart.getChartData().getDatasets().getFirst().getData()).hasSize(1); + assertThat(new DbPerfWidgetRenderer().findings(data).getRows()).hasSize(1); + assertThat(new DbPerfWidgetRenderer().tables(data).getTableDataList().getFirst().getRows()).hasSize(1); + MultiTableData attribution = new DbPerfWidgetRenderer().attribution(); + assertThat(attribution.getTableDataList().getFirst().getRows().getFirst()).containsEntry("Top table", "patient"); + assertThat(attribution.getTableDataList().get(1).getRows()).hasSize(1); + assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().connectionStatistics(data))).contains("Deadlocks", "Cache hit"); + JsonInput lookup = new JsonInput(); + lookup.getQueryParams().put("sql", "SELECT * FROM patient WHERE name = 'different'"); + assertThat((List) ((Map) new LookupSpec().handle(lookup)).get("matches")).hasSize(1); + runtime.getConfig().withAppName("custom"); + assertThat(data.recordPath("db_perf_statement", 1)).isEqualTo("/app/custom/db_perf_statement/1"); + JsonInput invalid = new JsonInput(); + invalid.getQueryParams().put("backend", "other"); + assertThatThrownBy(() -> new OverviewSpec().handle(invalid)).hasMessageContaining("monitored backend"); + assertThatThrownBy(() -> new LookupSpec().handle(new JsonInput())).hasMessageContaining("Supply SQL"); + } + + + + /******************************************************************************* + ** Raw text is permission gated; transitive blocking includes indirect descendants. + *******************************************************************************/ + @Test + @SuppressWarnings("unchecked") + void liveTextAndBlocking() throws Exception + { + List sessions = List.of(session(10, List.of()), session(20, List.of(10)), session(30, List.of(20))); + QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW)); + Map redacted = data.activity(sessions, true); + assertThat(JsonUtils.toJson(redacted)).doesNotContain("private-value").contains("liveTextRedacted"); + List> chains = (List>) redacted.get("blockingChains"); + assertThat(chains).hasSize(1); + assertThat(chains.getFirst()).containsEntry("rootPid", 10).containsEntry("blockedPids", List.of(20, 30)); + QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.VIEW_LIVE_TEXT)); + assertThat(JsonUtils.toJson(data.activity(sessions, true))).contains("private-value"); + assertThat(JsonUtils.toJson(data.activity(sessions, false))).doesNotContain("private-value"); + assertThat(((Map) new ActivitySpec().handle(new JsonInput())).get("sessions")).isEqualTo(List.of()); + } + + + + /******************************************************************************* + ** The actual home renderer requests redacted live data, even for privileged viewers. + *******************************************************************************/ + @Test + void liveWidgetRedaction() throws Exception + { + List sessions = List.of(session(10, List.of()), session(20, List.of(10))); + DbPerformanceData fixtureData = new DbPerformanceData(runtime) + { + @Override + public Map activity(String backend, boolean allowRawText) throws Exception + { + assertThat(allowRawText).isFalse(); + return (activity(sessions, allowRawText)); + } + }; + QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW)); + TableData table = new DbPerfWidgetRenderer().liveActivity(fixtureData); + assertThat(table.getRows()).hasSize(2); + assertThat(JsonUtils.toJson(table)).doesNotContain("private-value").contains("normalizedText", "blockingCount"); + QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.VIEW_LIVE_TEXT)); + assertThat(JsonUtils.toJson(new DbPerfWidgetRenderer().liveActivity(fixtureData))).doesNotContain("private-value"); + } + + + + /******************************************************************************* + ** The record action requires one statement and removes parameter text on every exit. + *******************************************************************************/ + @Test + void recordAction() throws Exception + { + var process = runtime.getInstance().getProcess(QueryPlanProcess.NAME); + assertThat(process.getTableName()).isEqualTo("db_perf_statement"); + assertThat(process.getMinInputRecords()).isEqualTo(1); + assertThat(process.getMaxInputRecords()).isEqualTo(1); + RunBackendStepInput input = new RunBackendStepInput(); + input.addValue("parameterValues", "private-value"); + assertThatThrownBy(() -> new QueryPlanProcess().run(input, new RunBackendStepOutput())).hasMessageContaining("exactly one"); + assertThat(input.getValues()).doesNotContainKey("parameterValues"); + input.withRecords(List.of(new QRecord().withValue("id", 1))); + input.addValue("mode", "ANALYZE"); + input.addValue("parameterValues", "private-value\nsecond-value"); + QContext.setQSession(new QSession().withPermissions(DbPerfPermissions.VIEW, DbPerfPermissions.EXPLAIN)); + assertThatThrownBy(() -> new QueryPlanProcess().run(input, new RunBackendStepOutput())).hasMessageContaining(DbPerfPermissions.EXPLAIN_ANALYZE); + assertThat(input.getValues()).doesNotContainKey("parameterValues"); + input.addValue("mode", "INVALID"); + assertThatThrownBy(() -> new QueryPlanProcess().run(input, new RunBackendStepOutput())).hasMessageContaining("Choose PLAN"); + assertThat(runtime.getInstance().getProcesses().values()).noneMatch(p -> "db_perf_explain".equals(p.getTableName())); + } + + + + /******************************************************************************* + ** Audit denied, invalid and unsupported-vendor paths using H2; no JDBC execution. + *******************************************************************************/ + @Test + void explainAuditAndHandler() throws Exception + { + ExplainService service = new ExplainService(runtime); + assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, "select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("PostgreSQL"); + assertThatThrownBy(() -> service.explain(null)).hasMessageContaining("Choose PLAN"); + assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, null, ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("either"); + QContext.setQSession(new QSession()); + JsonInput input = new JsonInput(); + input.getBody().put("mode", "ANALYZE"); + input.getBody().put("sql", "select 'private-value'"); + input.getBody().put("parameters", List.of("private-value")); + assertThatThrownBy(() -> new ExplainSpec().handle(input)).hasMessageContaining(DbPerfPermissions.EXPLAIN_ANALYZE); + List rows = runtime.asSystem(() -> DbPerfStore.query("db_perf_explain", new QQueryFilter())); + assertThat(rows).hasSize(4); + assertThat(rows).anyMatch(row -> "DENIED".equals(row.getValueString("outcome"))); + assertThat(JsonUtils.toJson(rows)).doesNotContain("private-value"); + input.getBody().put("parameters", List.of(5)); + assertThatThrownBy(() -> new ExplainSpec().handle(input)).hasMessageContaining("array of strings"); + input.getBody().put("statementId", "bad"); + assertThatThrownBy(() -> new ExplainSpec().handle(input)).hasMessageContaining("integer"); + } + + + + /******************************************************************************* + ** A realistic session whose raw text must not reach a home widget or storage. + *******************************************************************************/ + private static SessionInfo session(int pid, List blockedBy) + { + return (new SessionInfo(pid, "app", "QQQ", null, "active", "Lock", "transactionid", "client backend", Instant.now(), Instant.now(), Instant.now(), + BigDecimal.valueOf(60), BigDecimal.valueOf(60), blockedBy, 1L, "select * from patient where name = 'private-value'")); + } + + + + /******************************************************************************* + ** One row in every presentation source proves the renderer consumes persisted data. + *******************************************************************************/ + private void fixture() throws Exception + { + new DbPerfSettingsService(runtime).ensure(runtime.getConfig().getBackendName()); + QRecord statement = DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("queryId", 1L) + .withValue("queryText", "select * from patient where name = ?").withValue("fingerprint", SqlNormalizer.fingerprint(SqlNormalizer.normalize("select * from patient where name = ?"))) + .withValue("calls24h", 5L).withValue("totalMs24h", BigDecimal.valueOf(50)).withValue("meanMs24h", BigDecimal.TEN).withValue("shareOfDbTime24h", BigDecimal.valueOf(25)))).getFirst(); + QRecord snapshot = DbPerfStore.insert("db_perf_snapshot", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("capturedAt", Instant.now()) + .withValue("status", "OK").withValue("statementTotalMs", BigDecimal.valueOf(200)).withValue("statementCalls", 20L).withValue("connectionsTotal", 5).withValue("maxConnections", 100) + .withValue("cacheHitRatio", new BigDecimal("0.95")).withValue("deadlocks", 2L).withValue("tempBytes", 1048576L))).getFirst(); + DbPerfStore.insert("db_perf_query_snapshot", List.of(new QRecord().withValue("statementId", statement.getValueInteger("id")).withValue("snapshotId", snapshot.getValueInteger("id")) + .withValue("capturedAt", Instant.now()).withValue("sharedBlksHit", 90L).withValue("sharedBlksRead", 10L))); + DbPerfStore.insert("db_perf_finding", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("dedupeKey", "test").withValue("status", "OPEN") + .withValue("severity", "HIGH").withValue("title", "Slow query").withValue("ruleCode", "SLOW_QUERY").withValue("score", BigDecimal.TEN))); + DbPerfStore.insert("db_perf_table_stat", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()).withValue("schemaName", "public").withValue("tableName", "patient").withValue("relid", 1L))); + DbPerfStore.insert("db_perf_attribution", List.of(new QRecord().withValue("windowStart", Instant.now().minusSeconds(60)).withValue("windowEnd", Instant.now()).withValue("unitKind", "PROCESS") + .withValue("unitName", "sync").withValue("tableName", "patient").withValue("statements", 3L).withValue("totalMs", BigDecimal.TEN))); + DbPerfStore.insert("db_perf_repeat_pattern", List.of(new QRecord().withValue("windowStart", Instant.now().minusSeconds(60)).withValue("windowEnd", Instant.now()).withValue("unitKind", "PROCESS") + .withValue("unitName", "sync").withValue("maxRepeats", 12L))); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/ExplainConnectionTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/ExplainConnectionTest.java new file mode 100644 index 0000000..6f2664d --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/ExplainConnectionTest.java @@ -0,0 +1,322 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.explain.ExplainRequest; +import com.kingsrook.qbits.dbperformance.explain.ExplainResult; +import com.kingsrook.qbits.dbperformance.explain.ExplainService; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.postgresql.core.QueryExecutor; +import org.postgresql.jdbc.PgStatement; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.withSettings; + + +/******************************************************************************* + ** Driver contract tests: cleanup, state restoration, unknown binding and audit isolation. + ** These supplement, but do not replace, the real PostgreSQL acceptance tests. + *******************************************************************************/ +class ExplainConnectionTest +{ + private DbPerfRuntime runtime; + private final List events = new ArrayList<>(); + private int version = 17; + private String failureState; + private boolean autoCommit = true; + private boolean readOnly = false; + private String json = "[{\"Plan\":{\"Node Type\":\"Result\",\"Total Cost\":1.25,\"Actual Total Time\":0.1}}]"; + + + + /******************************************************************************* + ** Real H2 storage for the audit and settings; only plan execution JDBC is simulated. + *******************************************************************************/ + @BeforeEach + void setup() throws Exception + { + runtime = DbPerfRuntime.of(DbPerfTestSupport.instance()); + } + + + + /******************************************************************************* + ** Parameters reach setObject as unknown types; success never commits the plan transaction. + *******************************************************************************/ + @Test + void analyzeCleanupAndBinding() throws Exception + { + ExplainResult result = service().explain(request("select $1::int", ExplainRequest.Mode.ANALYZE, List.of("876543"))); + assertThat(result.mode()).isEqualTo("ANALYZE"); + assertThat(result.explainId()).isNotNull(); + assertThat(result.actualTotalMs()).isEqualByComparingTo("0.1"); + assertThat(events).anyMatch(event -> event.startsWith("PREPARE dbperf_") && event.endsWith("AS select $1::int")); + assertThat(events).anyMatch(event -> event.startsWith("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE dbperf_") && event.endsWith("(?)")); + assertThat(events).contains("bind:1:" + Types.OTHER).doesNotContain("commit"); + assertThat(events).noneMatch(event -> event.contains("876543")); + assertThat(events.indexOf("rollback")).isLessThan(index("DEALLOCATE")); + assertThat(events.getLast()).isEqualTo("close"); + assertThat(autoCommit).isTrue(); + assertThat(readOnly).isFalse(); + assertThat(audit().getValueString("planJson")).isNull(); + assertThat(audit().getValueString("outcome")).isEqualTo("OK"); + } + + + + /******************************************************************************* + ** Driver cancellation is classified by SQLSTATE, not a potentially sensitive message. + *******************************************************************************/ + @Test + void timeoutAndErrorCleanup() throws Exception + { + for(String state : List.of("57014", "25006")) + { + failureState = state; + events.clear(); + assertThatThrownBy(() -> service().explain(request("select $1::int", ExplainRequest.Mode.ANALYZE, List.of("876543")))) + .hasMessageContaining(state.equals("57014") ? "timed out" : "Writes are not allowed").hasNoCause(); + assertThat(audit().getValueString("outcome")).isEqualTo(state.equals("57014") ? "TIMEOUT" : "ERROR"); + assertThat(audit().getValueString("errorMessage")).doesNotContain("private"); + assertThat(events.indexOf("rollback")).isLessThan(index("DEALLOCATE")); + assertThat(events).doesNotContain("commit"); + assertThat(autoCommit).isTrue(); + } + } + + + + /******************************************************************************* + ** Generic version guard, literal-free storage, and normalized statement-source linkage. + *******************************************************************************/ + @Test + void planStorageAndVersion() throws Exception + { + assertThat(service().explain(request("select current_date", ExplainRequest.Mode.PLAN, List.of())).mode()).isEqualTo("PLAN"); + assertThat(audit().getValueString("planJson")).isEqualTo(json); + assertThat(events).contains("SET TRANSACTION READ ONLY", "SET LOCAL statement_timeout = 10000", "rollback"); + version = 15; + assertThatThrownBy(() -> service().explain(request("select $1::int", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("PostgreSQL 16"); + version = 17; + QRecord statement = DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()) + .withValue("queryId", 42L).withValue("queryText", "select $1::int"))).getFirst(); + ExplainResult generic = service().explain(new ExplainRequest(runtime.getConfig().getBackendName(), statement.getValueInteger("id"), null, ExplainRequest.Mode.PLAN, List.of())); + assertThat(generic.mode()).isEqualTo("GENERIC"); + assertThat(audit().getValueInteger("statementId")).isEqualTo(statement.getValueInteger("id")); + assertThat(audit().getValueString("planJson")).isEqualTo(json); + assertThat(events).contains("EXPLAIN (GENERIC_PLAN, FORMAT JSON) select $1::int"); + service().explain(request("SELECT 'private'", ExplainRequest.Mode.PLAN, List.of())); + assertThat(audit().getValueString("planJson")).isNull(); + assertThat(audit().getValueString("sqlText")).isEqualTo("select ?"); + } + + + + /******************************************************************************* + ** Pool state can start non-default; no result can escape without valid JSON. + *******************************************************************************/ + @Test + void restoreNonDefaultAndBadPlan() throws Exception + { + autoCommit = false; + readOnly = true; + json = "{}"; + assertThatThrownBy(() -> service().explain(request("select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("Query plan failed"); + assertThat(autoCommit).isFalse(); + assertThat(readOnly).isTrue(); + assertThat(events).contains("rollback", "close"); + assertThat(audit().getValueString("outcome")).isEqualTo("ERROR"); + } + + + + /******************************************************************************* + ** A separate connection boundary keeps real QQQ settings and audits untouched. + *******************************************************************************/ + private ExplainService service() + { + return (new ExplainService(runtime) + { + @Override + protected Connection openConnection() + { + return (connection()); + } + }); + } + + + + /******************************************************************************* + ** JDBC protocol recorder. Unexpected calls fail instead of silently succeeding. + *******************************************************************************/ + private Connection connection() + { + return (proxy(Connection.class, (proxy, method, args) -> + { + switch(method.getName()) + { + case "getMetaData": + return (proxy(DatabaseMetaData.class, (p, m, a) -> m.getName().equals("getDatabaseProductName") ? "PostgreSQL" : version)); + case "getAutoCommit": + return (autoCommit); + case "isReadOnly": + return (readOnly); + case "setAutoCommit": + autoCommit = (Boolean) args[0]; + events.add("autoCommit:" + autoCommit); + return (null); + case "setReadOnly": + readOnly = (Boolean) args[0]; + events.add("readOnly:" + readOnly); + return (null); + case "createStatement": + return (statement(false)); + case "prepareStatement": + events.add((String) args[0]); + return (statement(true)); + case "rollback", "commit", "close": + events.add(method.getName()); + return (null); + default: + throw (new AssertionError("Unexpected connection call: " + method.getName())); + } + })); + } + + + + /******************************************************************************* + ** Record only bind position and type, never the synthetic sensitive fixture value. + *******************************************************************************/ + private Statement statement(boolean prepared) + { + InvocationHandler handler = (proxy, method, args) -> + { + switch(method.getName()) + { + case "execute": + events.add((String) args[0]); + return (false); + case "unwrap": + return (proxy); + case "executeWithFlags": + assertThat(args[args.length - 1]).isEqualTo(QueryExecutor.QUERY_EXECUTE_AS_SIMPLE); + if(args.length == 2) + { + events.add((String) args[0]); + } + if(failureState != null) + { + throw (new SQLException("private-value in driver error", failureState)); + } + return (true); + case "getResultSet": + return (proxy(ResultSet.class, (p, m, a) -> switch(m.getName()) + { + case "next" -> true; + case "getString" -> json; + case "close" -> null; + default -> throw new AssertionError("Unexpected result call: " + m.getName()); + })); + case "setObject": + assertThat(args[1]).isEqualTo("876543"); + events.add("bind:" + args[0] + ":" + args[2]); + return (null); + case "close": + return (null); + default: + throw (new AssertionError("Unexpected statement call: " + method.getName())); + } + }; + var settings = withSettings().defaultAnswer(invocation -> handler.invoke(invocation.getMock(), invocation.getMethod(), invocation.getArguments())); + if(prepared) + { + settings.extraInterfaces(PreparedStatement.class); + } + return (mock(PgStatement.class, settings)); + } + + + + /******************************************************************************* + ** Type-safe interface proxy for the small JDBC contract exercised above. + *******************************************************************************/ + private static T proxy(Class type, InvocationHandler handler) + { + return (type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, handler))); + } + + + + /******************************************************************************* + ** Position of a protocol event for ordering assertions. + *******************************************************************************/ + private int index(String prefix) + { + for(int i = 0; i < events.size(); i++) + { + if(events.get(i).startsWith(prefix)) + { + return (i); + } + } + throw (new AssertionError("Missing event " + prefix)); + } + + + + /******************************************************************************* + ** Current backend request. + *******************************************************************************/ + private ExplainRequest request(String sql, ExplainRequest.Mode mode, List parameters) + { + return (new ExplainRequest(runtime.getConfig().getBackendName(), null, sql, mode, parameters)); + } + + + + /******************************************************************************* + ** Most recent persisted audit. + *******************************************************************************/ + private QRecord audit() throws Exception + { + return (DbPerfStore.first("db_perf_explain", new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)))); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/ExplainServiceTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/ExplainServiceTest.java new file mode 100644 index 0000000..f50ab53 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/ExplainServiceTest.java @@ -0,0 +1,239 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import java.sql.Connection; +import java.sql.Statement; +import java.util.List; +import com.kingsrook.qbits.dbperformance.collect.DbPerfSettingsService; +import com.kingsrook.qbits.dbperformance.collect.DbPerfStore; +import com.kingsrook.qbits.dbperformance.explain.ExplainRequest; +import com.kingsrook.qbits.dbperformance.explain.ExplainResult; +import com.kingsrook.qbits.dbperformance.explain.ExplainService; +import com.kingsrook.qbits.dbperformance.explain.QueryPlanProcess; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** PostgreSQL 17 acceptance tests for permission, PHI and READ ONLY boundaries. + *******************************************************************************/ +@Testcontainers +class ExplainServiceTest +{ + @Container + static final PostgreSQLContainer POSTGRES = DbPerfPostgresSupport.container(); + private DbPerfRuntime runtime; + private ExplainService service; + + + + /******************************************************************************* + ** A volatile function attempts a write even though its caller is a SELECT. + *******************************************************************************/ + @BeforeAll + static void fixture() throws Exception + { + try(Connection connection = DbPerfPostgresSupport.connect(POSTGRES); Statement statement = connection.createStatement()) + { + statement.execute("CREATE TABLE explain_probe (id integer)"); + statement.execute("CREATE FUNCTION explain_write() RETURNS integer LANGUAGE plpgsql VOLATILE AS $$ BEGIN INSERT INTO explain_probe VALUES (1); RETURN 1; END $$"); + } + } + + + + /******************************************************************************* + ** Fresh QQQ runtime, sharing the container's migrated schema. + *******************************************************************************/ + @BeforeEach + void setup() throws Exception + { + runtime = DbPerfRuntime.of(DbPerfTestSupport.instance(DbPerfPostgresSupport.backend(POSTGRES))); + service = new ExplainService(runtime); + } + + + + /******************************************************************************* + ** Literal-free PLAN is persisted, while literals and ANALYZE are transient. + *******************************************************************************/ + @Test + void planAndGeneric() throws Exception + { + ExplainResult plain = explain("select * from explain_probe", ExplainRequest.Mode.PLAN, List.of()); + assertThat(plain.mode()).isEqualTo("PLAN"); + assertThat(plain.explainId()).isNotNull(); + assertThat(audit().getValueString("planJson")).isEqualTo(plain.planJson()); + assertThat(plain.planText()).contains("Seq Scan", "explain_probe", "Total Cost", "Plan Rows"); + assertThat(plain.sqlWorkbenchUrl()).contains("?sql=EXPLAIN+", "&backend="); + ExplainResult generic = explain("select * from explain_probe where id = $1", ExplainRequest.Mode.PLAN, List.of()); + assertThat(generic.mode()).isEqualTo("GENERIC"); + assertThat(explain("select * from explain_probe where id = ?", ExplainRequest.Mode.PLAN, List.of()).mode()).isEqualTo("GENERIC"); + ExplainResult literal = explain("select * from explain_probe where id = 987654", ExplainRequest.Mode.PLAN, List.of()); + assertThat(audit().getValueString("planJson")).isNull(); + assertThat(audit().getValueString("sqlText")).doesNotContain("987654"); + assertThat(literal.assistantPrompt()).doesNotContain("987654"); + } + + + + /******************************************************************************* + ** Bind values as unknown JDBC values, never in SQL interpolation or persistent rows. + *******************************************************************************/ + @Test + void analyze() throws Exception + { + ExplainResult result = explain("select * from explain_probe where id = $1", ExplainRequest.Mode.ANALYZE, List.of("987654")); + assertThat(result.actualTotalMs()).isNotNull(); + assertThat(result.mode()).isEqualTo("ANALYZE"); + assertThat(result.assistantPrompt()).doesNotContain("987654"); + assertThat(audit().getValueString("outcome")).isEqualTo("OK"); + assertThat(audit().getValueString("planJson")).isNull(); + assertThat(audit().getValueString("planText")).isNull(); + assertThat(audit().getValueInteger("parameterCount")).isEqualTo(1); + assertThat(explain("select $1::text", ExplainRequest.Mode.ANALYZE, List.of("'; DELETE FROM explain_probe; --")).actualTotalMs()).isNotNull(); + assertThat(explain("select current_date", ExplainRequest.Mode.ANALYZE, List.of()).actualTotalMs()).isNotNull(); + } + + + + /******************************************************************************* + ** Rejected requests still receive sanitized audit rows. + *******************************************************************************/ + @Test + void rejectUnsafeSql() throws Exception + { + for(String sql : List.of("DELETE FROM explain_probe", "WITH d AS (DELETE FROM explain_probe RETURNING *) SELECT * FROM d", + "SELECT * FROM explain_probe; DELETE FROM explain_probe", "SELECT * INTO probe_copy FROM explain_probe", "SELECT * FROM explain_probe FOR UPDATE", + "SELECT * FROM explain_probe FOR SHARE", "WITH d AS (SELECT 1) DELETE FROM explain_probe")) + { + assertThatThrownBy(() -> explain(sql, ExplainRequest.Mode.ANALYZE, List.of())).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class).satisfies(e -> assertThat(e.getCause()).isNull()); + assertThat(audit().getValueString("outcome")).isEqualTo("ERROR"); + } + assertThatThrownBy(() -> explain("select $1::int", ExplainRequest.Mode.ANALYZE, List.of())).hasMessageContaining("exactly 1"); + assertThatThrownBy(() -> explain("select current_date", ExplainRequest.Mode.PLAN, List.of("secret"))).hasMessageContaining("only for ANALYZE"); + } + + + + /******************************************************************************* + ** A write hidden in a volatile function is stopped by the database itself. + *******************************************************************************/ + @Test + void readOnlyAndTimeout() throws Exception + { + assertThatThrownBy(() -> explain("select explain_write()", ExplainRequest.Mode.ANALYZE, List.of())).hasMessageContaining("Writes are not allowed"); + try(Connection connection = DbPerfPostgresSupport.connect(POSTGRES); Statement statement = connection.createStatement(); var rows = statement.executeQuery("SELECT count(*) FROM explain_probe")) + { + rows.next(); + assertThat(rows.getInt(1)).isZero(); + } + QRecord settings = new DbPerfSettingsService(runtime).ensure(runtime.getConfig().getBackendName()); + DbPerfStore.update("db_perf_settings", List.of(new QRecord().withValue("id", settings.getValueInteger("id")).withValue("explainTimeoutMs", 10))); + assertThatThrownBy(() -> explain("select pg_sleep($1)", ExplainRequest.Mode.ANALYZE, List.of("1"))).hasMessageContaining("timed out"); + assertThat(audit().getValueString("outcome")).isEqualTo("TIMEOUT"); + assertThat(explain("select current_date", ExplainRequest.Mode.PLAN, List.of()).mode()).isEqualTo("PLAN"); + } + + + + /******************************************************************************* + ** Denial is audited as system but caller identity and context are retained. + *******************************************************************************/ + @Test + void deniedAndValidation() throws Exception + { + QSession caller = new QSession().withUser(new QUser().withIdReference("viewer")); + QContext.setQSession(caller); + assertThatThrownBy(() -> explain("select 'private'", ExplainRequest.Mode.PLAN, List.of())).hasMessageContaining(DbPerfPermissions.EXPLAIN); + assertThat(QContext.getQSession()).isSameAs(caller); + QRecord denied = runtime.asSystem(this::audit); + assertThat(denied.getValueString("outcome")).isEqualTo("DENIED"); + assertThat(denied.getValueString("requestedBy")).isEqualTo("viewer"); + assertThat(denied.getValueString("sqlText")).doesNotContain("private"); + caller.withPermissions(DbPerfPermissions.EXPLAIN); + assertThatThrownBy(() -> explain("select current_date", ExplainRequest.Mode.ANALYZE, List.of())).hasMessageContaining(DbPerfPermissions.EXPLAIN_ANALYZE); + assertThatThrownBy(() -> service.explain(new ExplainRequest("wrong", null, "select current_date", ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("monitored backend"); + assertThatThrownBy(() -> service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), 999999, null, ExplainRequest.Mode.PLAN, List.of()))).hasMessageContaining("not found"); + } + + + + /******************************************************************************* + ** The record action carries a real PostgreSQL plan into copyable result fields. + *******************************************************************************/ + @Test + void recordActionResult() throws Exception + { + QRecord statement = DbPerfStore.insert("db_perf_statement", List.of(new QRecord().withValue("backendName", runtime.getConfig().getBackendName()) + .withValue("queryId", 123456L).withValue("queryText", "select * from explain_probe where id = ?"))).getFirst(); + RunBackendStepInput input = new RunBackendStepInput().withRecords(List.of(statement)); + RunBackendStepOutput output = new RunBackendStepOutput(); + new QueryPlanProcess().run(input, output); + assertThat(output.getValueString("planText")).contains("explain_probe"); + assertThat(output.getValueBigDecimal("totalCost")).isNotNull(); + assertThat(output.getValueString("sqlWorkbenchUrl")).contains("?sql=EXPLAIN"); + assertThat(output.getValueString("assistantPrompt")).contains("Query:", "Plan:"); + assertThat(audit().getValueString("planJson")).isNotNull(); + input.addValue("mode", "ANALYZE"); + input.addValue("parameterValues", "987654"); + new QueryPlanProcess().run(input, output); + assertThat(input.getValues()).doesNotContainKey("parameterValues"); + assertThat(audit().getValueString("planJson")).isNull(); + assertThat(output.getValueString("assistantPrompt")).doesNotContain("987654"); + } + + + + /******************************************************************************* + ** Use the configured backend for each direct SQL request. + *******************************************************************************/ + private ExplainResult explain(String sql, ExplainRequest.Mode mode, List parameters) throws Exception + { + return (service.explain(new ExplainRequest(runtime.getConfig().getBackendName(), null, sql, mode, parameters))); + } + + + + /******************************************************************************* + ** Last audit across the shared container. + *******************************************************************************/ + private QRecord audit() throws Exception + { + return (DbPerfStore.first("db_perf_explain", new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)))); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dbperformance/ExplainSqlTest.java b/src/test/java/com/kingsrook/qbits/dbperformance/ExplainSqlTest.java new file mode 100644 index 0000000..cf9a8ea --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dbperformance/ExplainSqlTest.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dbperformance; + + +import com.kingsrook.qbits.dbperformance.explain.ExplainSql; +import com.kingsrook.qbits.dbperformance.explain.PlanTextRenderer; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Lexical security, bind-slot identity and readable plan rendering without JDBC. + *******************************************************************************/ +class ExplainSqlTest +{ + + + + /******************************************************************************* + ** Quotes, nested comments and repeated parameter references retain their meaning. + *******************************************************************************/ + @Test + void lexicalBoundaries() throws Exception + { + var parsed = ExplainSql.parse("/* UPDATE /* DELETE */ */ SELECT ?, '?', E'foo\\'UPDATE', $$DELETE;$$, \"INTO\" -- DELETE\n ;"); + parsed.requireSelect(); + assertThat(parsed.parameterCount()).isEqualTo(1); + assertThat(parsed.sql()).contains("SELECT $1", "'?'").doesNotContain("/*"); + assertThat(ExplainSql.parse("SELECT $1::int + $1, $2").parameterCount()).isEqualTo(2); + assertThat(ExplainSql.parse("SELECT col ?| array['a'], col ?& array['b'] FROM example").parameterCount()).isZero(); + for(String sql : new String[] {"select $0", "select $999999999999999999", "select $2", "select ?, $1", "select 'bad", "select $$bad", "/*bad", "select 1; select 2", "", " ", "select 1;;"}) + { + assertThatThrownBy(() -> ExplainSql.parse(sql)).isInstanceOf(Exception.class); + } + assertThatThrownBy(() -> ExplainSql.parse(null)).hasMessageContaining("Supply"); + assertThatThrownBy(() -> ExplainSql.parse("x".repeat(100001))).hasMessageContaining("100000"); + for(String sql : new String[] {"delete from x", "with x as (insert into t values (1)) select * from x", "select * from t for no key update", "select * from t for key share", "select * into t from x"}) + { + assertThatThrownBy(() -> ExplainSql.parse(sql).requireSelect()).hasMessageContaining("single SELECT"); + } + ExplainSql.parse("with x as (select 'UPDATE') select * from x").requireSelect(); + } + + + + /******************************************************************************* + ** Render hierarchy and index detail; sanitize plan expressions in assistant prompts. + *******************************************************************************/ + @Test + void planTree() throws Exception + { + var plan = PlanTextRenderer.render(""" + [{"Plan":{"Node Type":"Nested Loop","Startup Cost":0.1,"Total Cost":42.5,"Plan Rows":2, + "Plans":[{"Node Type":"Index Scan","Relation Name":"patient","Index Name":"patient_pk", + "Index Cond":"(id = 987654)","Filter":"(name = 'private-name'::text)","Actual Rows":1}]},"Execution Time":1.25}] + """); + assertThat(plan.text()).contains("Nested Loop", "\n Index Scan", "patient_pk", "Total Cost: 42.5", "Plan Rows: 2", "private-name"); + assertThat(plan.safeText()).doesNotContain("private-name", "987654").contains("id = ?", "name = ?"); + assertThat(plan.totalCost()).isEqualByComparingTo("42.5"); + assertThat(plan.actualTotalMs()).isEqualByComparingTo("1.25"); + assertThat(PlanTextRenderer.render("{\"Plan\":{\"Node Type\":\"Result\"}}").actualTotalMs()).isNull(); + assertThat(PlanTextRenderer.render("{\"Plan\":{\"Actual Total Time\":2}}").actualTotalMs()).isEqualByComparingTo("2"); + assertThatThrownBy(() -> PlanTextRenderer.render("[]")).hasMessageContaining("plan tree"); + assertThatThrownBy(() -> PlanTextRenderer.render("broken")).isInstanceOf(Exception.class); + } +} -- 2.54.0